Bump versions.

https://bugs.launchpad.net/calibre/+bug/1216398

app.yaml

@@ -1,6 +1,6 @@
 # ffd-retief-hrd fanfictiondownloader
 application: fanfictiondownloader
-version: 4-4-36
+version: 4-4-78
 runtime: python27
 api_version: 1
 threadsafe: true

calibre-plugin/__init__.py

@@ -1,13 +1,22 @@
 #!/usr/bin/env python
-# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
 # -*- coding: utf-8 -*-
 from __future__ import (unicode_literals, division, absolute_import,
                         print_function)
 
 __license__   = 'GPL v3'
-__copyright__ = '2011, Jim Miller'
+__copyright__ = '2013, Jim Miller'
 __docformat__ = 'restructuredtext en'
 
+import sys
+if sys.version_info >= (2, 7):
+    import logging
+    logger = logging.getLogger(__name__)
+    loghandler=logging.StreamHandler()
+    loghandler.setFormatter(logging.Formatter("FFDL:%(levelname)s:%(filename)s(%(lineno)d):%(message)s"))
+    logger.addHandler(loghandler)
+    loghandler.setLevel(logging.DEBUG)
+    logger.setLevel(logging.DEBUG)
+
 # The class that all Interface Action plugin wrappers must inherit from
 from calibre.customize import InterfaceActionBase
 
@@ -27,7 +36,7 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
     description         = 'UI plugin to download FanFiction stories from various sites.'
     supported_platforms = ['windows', 'osx', 'linux']
     author              = 'Jim Miller'
-    version             = (1, 7, 2)
+    version             = (1, 7, 49)
     minimum_calibre_version = (0, 8, 57)
 
     #: This field defines the GUI plugin class that contains all the code
@@ -80,11 +89,29 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
         if ac is not None:
             ac.apply_settings()
 
-# For testing, run from command line with this:
-# calibre-debug -e __init__.py
-# 
-if __name__ == '__main__':
-    from PyQt4.Qt import QApplication
-    from calibre.gui2.preferences import test_widget
-    app = QApplication([])
-    test_widget('Advanced', 'Plugins')
+    def cli_main(self,argv):
+        # I believe there's no performance hit loading these here when
+        # CLI--it would load everytime anyway.
+        from StringIO import StringIO
+        from calibre.library import db
+        from calibre_plugins.fanfictiondownloader_plugin.downloader import main as ffdl_main
+        from calibre_plugins.fanfictiondownloader_plugin.prefs import PrefsFacade
+        from calibre.utils.config import prefs as calibre_prefs
+        from optparse import OptionParser      
+
+        parser = OptionParser('%prog --run-plugin '+self.name+' -- [options] <storyurl>')
+        parser.add_option('--library-path', '--with-library', default=None, help=_('Path to the calibre library. Default is to use the path stored in the settings.'))
+        # parser.add_option('--dont-notify-gui', default=False, action='store_true',
+        #               help=_('Do not notify the running calibre GUI (if any) that the database has'
+        #                      ' changed. Use with care, as it can lead to database corruption!'))
+
+        pargs = [x for x in argv if x.startswith('--with-library') or x.startswith('--library-path')
+                 or not x.startswith('-')]
+        opts, args = parser.parse_args(pargs)
+
+        ffdl_prefs = PrefsFacade(db(path=opts.library_path,
+                                    read_only=True))
+        ffdl_main(argv[1:],
+                  parser=parser,
+                  passed_defaultsini=StringIO(get_resources("defaults.ini")),
+                  passed_personalini=StringIO(ffdl_prefs["personal.ini"]))

calibre-plugin/about.txt

@@ -4,7 +4,7 @@
 '<a href="http://www.mobileread.com/forums/showthread.php?t=134856">Reading List</a>',
 '<a href="http://www.mobileread.com/forums/showthread.php?t=126727">Extract ISBN</a>' and
 '<a href="http://www.mobileread.com/forums/showthread.php?t=134000">Count Pages</a>'
-plugins.  bbcodeutils code contributed by Pau Sanchez.</p>
+plugins.</p>
 
 <p>
 Calibre officially distributes plugins from the mobileread.com forum site.

calibre-plugin/common_utils.py

@@ -239,8 +239,8 @@ class SizePersistedDialog(QDialog):
             self.restoreGeometry(self.geom)
 
     def dialog_closing(self, result):
-        geom = bytearray(self.saveGeometry())
-        gprefs[self.unique_pref_name] = geom
+        self.geom = bytearray(self.saveGeometry())
+        gprefs[self.unique_pref_name] = self.geom
 
 
 class ReadOnlyTableWidgetItem(QTableWidgetItem):

calibre-plugin/config.py

@@ -7,168 +7,50 @@ __license__   = 'GPL v3'
 __copyright__ = '2012, Jim Miller'
 __docformat__ = 'restructuredtext en'
 
+import logging
+logger = logging.getLogger(__name__)
+
 import traceback, copy, threading
 from collections import OrderedDict
 
 from PyQt4.Qt import (QDialog, QWidget, QVBoxLayout, QHBoxLayout, QLabel,
                       QLineEdit, QFont, QWidget, QTextEdit, QComboBox,
                       QCheckBox, QPushButton, QTabWidget, QVariant, QScrollArea,
-                      QDialogButtonBox )
+                      QDialogButtonBox, QGroupBox )
 
-from calibre.gui2 import dynamic, info_dialog
-from calibre.utils.config import JSONConfig
 from calibre.gui2.ui import get_gui
+from calibre.gui2 import dynamic, info_dialog
+from calibre.constants import numeric_version as calibre_version
 
+from calibre_plugins.fanfictiondownloader_plugin.prefs import prefs, PREFS_NAMESPACE
 from calibre_plugins.fanfictiondownloader_plugin.dialogs \
     import (UPDATE, UPDATEALWAYS, OVERWRITE, collision_order, RejectListDialog,
-            EditTextDialog)
+            EditTextDialog, RejectUrlEntry)
     
 from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.adapters \
     import (getConfigSections, getNormalStoryURL)
 
 from calibre_plugins.fanfictiondownloader_plugin.common_utils \
-    import ( get_library_uuid, KeyboardConfigDialog, PrefsViewerDialog )
+    import ( KeyboardConfigDialog, PrefsViewerDialog )
 
 from calibre.gui2.complete import MultiCompleteLineEdit
 
-PREFS_NAMESPACE = 'FanFictionDownLoaderPlugin'
-PREFS_KEY_SETTINGS = 'settings'
-
-# Set defaults used by all.  Library specific settings continue to
-# take from here.
-default_prefs = {}
-default_prefs['personal.ini'] = get_resources('plugin-example.ini')
-default_prefs['rejecturls'] = ''
-default_prefs['rejectreasons'] = '''Sucked
-Boring
-Dup from another site'''
-
-default_prefs['updatemeta'] = True
-default_prefs['updatecover'] = False
-default_prefs['updateepubcover'] = False
-default_prefs['keeptags'] = False
-default_prefs['urlsfromclip'] = True
-default_prefs['updatedefault'] = True
-default_prefs['fileform'] = 'epub'
-default_prefs['collision'] = OVERWRITE
-default_prefs['deleteotherforms'] = False
-default_prefs['adddialogstaysontop'] = False
-default_prefs['includeimages'] = False
-default_prefs['lookforurlinhtml'] = False
-default_prefs['injectseries'] = False
-
-default_prefs['send_lists'] = ''
-default_prefs['read_lists'] = ''
-default_prefs['addtolists'] = False
-default_prefs['addtoreadlists'] = False
-default_prefs['addtolistsonread'] = False
-
-default_prefs['gcnewonly'] = False
-default_prefs['gc_site_settings'] = {}
-default_prefs['allow_gc_from_ini'] = True
-
-default_prefs['countpagesstats'] = []
-
-default_prefs['errorcol'] = ''
-default_prefs['custom_cols'] = {}
-default_prefs['custom_cols_newonly'] = {}
-default_prefs['allow_custcol_from_ini'] = True
-
-default_prefs['std_cols_newonly'] = {}
-
-def set_library_config(library_config):
-    get_gui().current_db.prefs.set_namespaced(PREFS_NAMESPACE,
-                                              PREFS_KEY_SETTINGS,
-                                              library_config)
-    
-def get_library_config():
-    db = get_gui().current_db
-    library_id = get_library_uuid(db)
-    library_config = None
-    # Check whether this is a configuration needing to be migrated
-    # from json into database.  If so: get it, set it, rename it in json.
-    if library_id in old_prefs:
-        #print("get prefs from old_prefs")
-        library_config = old_prefs[library_id]
-        set_library_config(library_config)
-        old_prefs["migrated to library db %s"%library_id] = old_prefs[library_id]
-        del old_prefs[library_id]
-
-    if library_config is None:
-        #print("get prefs from db")
-        library_config = db.prefs.get_namespaced(PREFS_NAMESPACE, PREFS_KEY_SETTINGS,
-                                                 copy.deepcopy(default_prefs))
-    return library_config
-
-# This is where all preferences for this plugin *were* stored
-# Remember that this name (i.e. plugins/fanfictiondownloader_plugin) is also
-# in a global namespace, so make it as unique as possible.
-# You should always prefix your config file name with plugins/,
-# so as to ensure you dont accidentally clobber a calibre config file
-old_prefs = JSONConfig('plugins/fanfictiondownloader_plugin')
-
-# fake out so I don't have to change the prefs calls anywhere.  The
-# Java programmer in me is offended by op-overloading, but it's very
-# tidy.
-class PrefsFacade():
-    def __init__(self,default_prefs):
-        self.default_prefs = default_prefs
-        self.libraryid = None
-        self.current_prefs = None
-        
-    def _get_prefs(self):
-        libraryid = get_library_uuid(get_gui().current_db)
-        if self.current_prefs == None or self.libraryid != libraryid:
-            #print("self.current_prefs == None(%s) or self.libraryid != libraryid(%s)"%(self.current_prefs == None,self.libraryid != libraryid))
-            self.libraryid = libraryid
-            self.current_prefs = get_library_config()
-        return self.current_prefs
-        
-    def __getitem__(self,k):            
-        prefs = self._get_prefs()
-        if k not in prefs:
-            # pulls from default_prefs.defaults automatically if not set
-            # in default_prefs
-            return self.default_prefs[k]
-        return prefs[k]
-
-    def __setitem__(self,k,v):
-        prefs = self._get_prefs()
-        prefs[k]=v
-        # self._save_prefs(prefs)
-
-    def __delitem__(self,k):
-        prefs = self._get_prefs()
-        if k in prefs:
-            del prefs[k]
-
-    def save_to_db(self):
-        set_library_config(self._get_prefs())
-        
-prefs = PrefsFacade(default_prefs)
-
 class RejectURLList:
     def __init__(self,prefs):
         self.prefs = prefs
         self.sync_lock = threading.RLock()
         self.listcache = None
 
-    def _read_list_from_text(self,text,addreasontext=None):
-        cache = {}
+    def _read_list_from_text(self,text,addreasontext=''):
+        cache = OrderedDict()
+
+        #print("_read_list_from_text")
         for line in text.splitlines():
-            if ',' in line:
-                (rejurl,note) = line.split(',',1)
-            else:
-                (rejurl,note) = (line,'')
-            rejurl = getNormalStoryURL(rejurl)
-            if rejurl:
-                if addreasontext and note:
-                    note = note +" - "+addreasontext
-                elif addreasontext:
-                    note = addreasontext
-                cache[rejurl] = note
-        return cache
-        
+            rue = RejectUrlEntry(line,addreasontext=addreasontext,fromline=True)
+            #print("rue.url:%s"%rue.url)
+            if rue.valid:
+                cache[rue.url] = rue
+        return cache        
 
     def _get_listcache(self):
         if self.listcache == None:
@@ -176,26 +58,35 @@ class RejectURLList:
         return self.listcache
 
     def _save_list(self,listcache):
-        rejectlist = []
-        for url in listcache:
-            rejectlist.append("%s,%s"%(url,listcache[url]))
-            
-        self.prefs['rejecturls'] = '\n'.join(rejectlist)
+        #print("_save_list")
+        self.prefs['rejecturls'] = '\n'.join([x.to_line() for x in listcache.values()])
         self.prefs.save_to_db()
         self.listcache = None
         
     def clear_cache(self):
         self.listcache = None
 
+    # true if url is in list.
     def check(self,url):
+        with self.sync_lock:
+            listcache = self._get_listcache()
+            return url in listcache
+        
+    def get_note(self,url):
         with self.sync_lock:
             listcache = self._get_listcache()
             if url in listcache:
-                note = listcache[url]
-                return note
-            
+                return listcache[url].note
             # not found
-            return None
+            return ''
+
+    def get_full_note(self,url):
+        with self.sync_lock:
+            listcache = self._get_listcache()
+            if url in listcache:
+                return listcache[url].fullnote()
+            # not found
+            return ''
 
     def remove(self,url):
         with self.sync_lock:
@@ -205,27 +96,26 @@ class RejectURLList:
                 self._save_list(listcache)
 
     def add_text(self,rejecttext,addreasontext):
-        self.add(self._read_list_from_text(rejecttext,addreasontext).items())
+        self.add(self._read_list_from_text(rejecttext,addreasontext).values())
             
     def add(self,rejectlist,clear=False):
-        # rejectlist=list of (url,note) tuples.
         with self.sync_lock:
             if clear:
-                listcache={}
+                listcache=OrderedDict()
             else:
                 listcache = self._get_listcache()
-            for (url,note) in rejectlist:
-                listcache[url]=note
+            for l in rejectlist:
+                listcache[l.url]=l
             self._save_list(listcache)
 
     def get_list(self):
-        return copy.deepcopy(self._get_listcache())
+        return self._get_listcache().values()
             
     def get_reject_reasons(self):
         return self.prefs['rejectreasons'].splitlines()
 
 rejecturllist = RejectURLList(prefs)
-    
+
 class ConfigWidget(QWidget):
 
     def __init__(self, plugin_action):
@@ -239,9 +129,15 @@ class ConfigWidget(QWidget):
         label.setOpenExternalLinks(True)
         self.l.addWidget(label)
 
-        tab_widget = QTabWidget(self)
-        self.l.addWidget(tab_widget)
 
+        self.scroll_area = QScrollArea(self)
+        self.scroll_area.setFrameShape(QScrollArea.NoFrame)
+        self.scroll_area.setWidgetResizable(True)
+        self.l.addWidget(self.scroll_area)
+
+        tab_widget = QTabWidget(self)
+        self.scroll_area.setWidget(tab_widget)
+        
         self.basic_tab = BasicTab(self, plugin_action)
         tab_widget.addTab(self.basic_tab, 'Basic')
 
@@ -282,13 +178,20 @@ class ConfigWidget(QWidget):
         prefs['updatecover'] = self.basic_tab.updatecover.isChecked()
         prefs['updateepubcover'] = self.basic_tab.updateepubcover.isChecked()
         prefs['keeptags'] = self.basic_tab.keeptags.isChecked()
+        prefs['suppressauthorsort'] = self.basic_tab.suppressauthorsort.isChecked()
+        prefs['suppresstitlesort'] = self.basic_tab.suppresstitlesort.isChecked()
+        prefs['mark'] = self.basic_tab.mark.isChecked()
+        prefs['showmarked'] = self.basic_tab.showmarked.isChecked()
         prefs['urlsfromclip'] = self.basic_tab.urlsfromclip.isChecked()
         prefs['updatedefault'] = self.basic_tab.updatedefault.isChecked()
         prefs['deleteotherforms'] = self.basic_tab.deleteotherforms.isChecked()
         prefs['adddialogstaysontop'] = self.basic_tab.adddialogstaysontop.isChecked()
         prefs['includeimages'] = self.basic_tab.includeimages.isChecked()
         prefs['lookforurlinhtml'] = self.basic_tab.lookforurlinhtml.isChecked()
+        prefs['checkforseriesurlid'] = self.basic_tab.checkforseriesurlid.isChecked()
+        prefs['checkforurlchange'] = self.basic_tab.checkforurlchange.isChecked()
         prefs['injectseries'] = self.basic_tab.injectseries.isChecked()
+        prefs['smarten_punctuation'] = self.basic_tab.smarten_punctuation.isChecked()
 
         if self.readinglist_tab:
             # lists
@@ -378,13 +281,16 @@ class BasicTab(QWidget):
         self.plugin_action = plugin_action
         QWidget.__init__(self)
         
-        self.l = QVBoxLayout()
-        self.setLayout(self.l)
+        topl = QVBoxLayout()
+        self.setLayout(topl)
 
         label = QLabel('These settings control the basic features of the plugin--downloading FanFiction.')
         label.setWordWrap(True)
-        self.l.addWidget(label)
-        self.l.addSpacing(5)
+        topl.addWidget(label)
+
+        defs_gb = groupbox = QGroupBox("Defaults Options on Download")
+        self.l = QVBoxLayout()
+        groupbox.setLayout(self.l)
 
         tooltip = "On each download, FFDL offers an option to select the output format. <br />This sets what that option will default to."
         horz = QHBoxLayout()
@@ -429,7 +335,15 @@ class BasicTab(QWidget):
         self.updateepubcover.setChecked(prefs['updateepubcover'])
         self.l.addWidget(self.updateepubcover)
 
-        self.l.addSpacing(10)        
+        self.smarten_punctuation = QCheckBox('Smarten Punctuation (EPUB only)',self)
+        self.smarten_punctuation.setToolTip("Run Smarten Punctuation from Calibre's Polish Book feature on each EPUB download and update.")
+        self.smarten_punctuation.setChecked(prefs['smarten_punctuation'])
+        if calibre_version >= (0, 9, 39):
+            self.l.addWidget(self.smarten_punctuation)
+
+        cali_gb = groupbox = QGroupBox("Updating Calibre Options")
+        self.l = QVBoxLayout()
+        groupbox.setLayout(self.l)
 
         self.deleteotherforms = QCheckBox('Delete other existing formats?',self)
         self.deleteotherforms.setToolTip('Check this to automatically delete all other ebook formats when updating an existing book.\nHandy if you have both a Nook(epub) and Kindle(mobi), for example.')
@@ -446,7 +360,44 @@ class BasicTab(QWidget):
         self.keeptags.setChecked(prefs['keeptags'])
         self.l.addWidget(self.keeptags)
 
-        self.l.addSpacing(10)        
+        self.suppressauthorsort = QCheckBox('Force Author into Author Sort?',self)
+        self.suppressauthorsort.setToolTip("If checked, the author(s) as given will be used for the Author Sort, too.\nIf not checked, calibre will apply it's built in algorithm which makes 'Bob Smith' sort as 'Smith, Bob', etc.")
+        self.suppressauthorsort.setChecked(prefs['suppressauthorsort'])
+        self.l.addWidget(self.suppressauthorsort)
+
+        self.suppresstitlesort = QCheckBox('Force Title into Title Sort?',self)
+        self.suppresstitlesort.setToolTip("If checked, the title as given will be used for the Title Sort, too.\nIf not checked, calibre will apply it's built in algorithm which makes 'The Title' sort as 'Title, The', etc.")
+        self.suppresstitlesort.setChecked(prefs['suppresstitlesort'])
+        self.l.addWidget(self.suppresstitlesort)
+
+        self.checkforseriesurlid = QCheckBox("Check for existing Series Anthology books?",self)
+        self.checkforseriesurlid.setToolTip("Check for existings Series Anthology books using each new story's series URL before downloading.\nOffer to skip downloading if a Series Anthology is found.")
+        self.checkforseriesurlid.setChecked(prefs['checkforseriesurlid'])
+        self.l.addWidget(self.checkforseriesurlid)
+
+        self.checkforurlchange = QCheckBox("Check for changed Story URL?",self)
+        self.checkforurlchange.setToolTip("Warn you if an update will change the URL of an existing book.")
+        self.checkforurlchange.setChecked(prefs['checkforurlchange'])
+        self.l.addWidget(self.checkforurlchange)
+
+        self.lookforurlinhtml = QCheckBox("Search EPUB text for Story URL?",self)
+        self.lookforurlinhtml.setToolTip("Look for first valid story URL inside EPUB text if not found in metadata.\nSomewhat risky, could find wrong URL depending on EPUB content.\nAlso finds and corrects bad ffnet URLs from ficsaver.com files.")
+        self.lookforurlinhtml.setChecked(prefs['lookforurlinhtml'])
+        self.l.addWidget(self.lookforurlinhtml)
+
+        self.mark = QCheckBox("Mark added/updated books when finished?",self)
+        self.mark.setToolTip("Mark added/updated books when finished.  Use with option below.\nYou can also manually search for 'marked:ffdl_success'.\n'marked:ffdl_failed' is also available, or search 'marked:ffdl' for both.")
+        self.mark.setChecked(prefs['mark'])
+        self.l.addWidget(self.mark)
+
+        self.showmarked = QCheckBox("Show Marked books when finished?",self)
+        self.showmarked.setToolTip("Show Marked added/updated books only when finished.\nYou can also manually search for 'marked:ffdl_success'.\n'marked:ffdl_failed' is also available, or search 'marked:ffdl' for both.")
+        self.showmarked.setChecked(prefs['showmarked'])
+        self.l.addWidget(self.showmarked)
+
+        gui_gb = groupbox = QGroupBox("GUI Options")
+        self.l = QVBoxLayout()
+        groupbox.setLayout(self.l)
 
         self.urlsfromclip = QCheckBox('Take URLs from Clipboard?',self)
         self.urlsfromclip.setToolTip('Prefill URLs from valid URLs in Clipboard when Adding New.')
@@ -464,7 +415,9 @@ class BasicTab(QWidget):
         self.adddialogstaysontop.setChecked(prefs['adddialogstaysontop'])
         self.l.addWidget(self.adddialogstaysontop)
 
-        self.l.addSpacing(10)        
+        misc_gb = groupbox = QGroupBox("Misc Options")
+        self.l = QVBoxLayout()
+        groupbox.setLayout(self.l)
 
         # this is a cheat to make it easier for users to realize there's a new include_images features.
         self.includeimages = QCheckBox("Include images in EPUBs?",self)
@@ -472,38 +425,45 @@ class BasicTab(QWidget):
         self.includeimages.setChecked(prefs['includeimages'])
         self.l.addWidget(self.includeimages)
 
-        self.lookforurlinhtml = QCheckBox("Search EPUB text for Story URL?",self)
-        self.lookforurlinhtml.setToolTip("Look for first valid story URL inside EPUB text if not found in metadata.\nSomewhat risky, could find wrong URL depending on EPUB content.\nAlso finds and corrects bad ffnet URLs from ficsaver.com files.")
-        self.lookforurlinhtml.setChecked(prefs['lookforurlinhtml'])
-        self.l.addWidget(self.lookforurlinhtml)
-
         self.injectseries = QCheckBox("Inject calibre Series when none found?",self)
         self.injectseries.setToolTip("If no series is found, inject the calibre series (if there is one) so it appears on the FFDL title page(not cover).")
         self.injectseries.setChecked(prefs['injectseries'])
         self.l.addWidget(self.injectseries)
 
-        self.l.addSpacing(10)        
+        rej_gb = groupbox = QGroupBox("Reject List")
+        self.l = QVBoxLayout()
+        groupbox.setLayout(self.l)
 
-        horz = QHBoxLayout()
-        
         self.rejectlist = QPushButton('Edit Reject URL List', self)
         self.rejectlist.setToolTip("Edit list of URLs FFDL will automatically Reject.")
         self.rejectlist.clicked.connect(self.show_rejectlist)
-        horz.addWidget(self.rejectlist)
+        self.l.addWidget(self.rejectlist)
         
         self.reject_urls = QPushButton('Add Reject URLs', self)
         self.reject_urls.setToolTip("Add additional URLs to Reject as text.")
         self.reject_urls.clicked.connect(self.add_reject_urls)
-        horz.addWidget(self.reject_urls)
+        self.l.addWidget(self.reject_urls)
         
         self.reject_reasons = QPushButton('Edit Reject Reasons List', self)
         self.reject_reasons.setToolTip("Customize the Reasons presented when Rejecting URLs")
         self.reject_reasons.clicked.connect(self.show_reject_reasons)
-        horz.addWidget(self.reject_reasons)
+        self.l.addWidget(self.reject_reasons)
+
+        topl.addWidget(defs_gb)
+
+        horz = QHBoxLayout()
+
+        horz.addWidget(cali_gb)
+
+        vert = QVBoxLayout()
+        vert.addWidget(gui_gb)
+        vert.addWidget(misc_gb)
+        vert.addWidget(rej_gb)
         
-        self.l.addLayout(horz)
+        horz.addLayout(vert)
         
-        self.l.insertStretch(-1)
+        topl.addLayout(horz)
+        topl.insertStretch(-1)
         
     def set_collisions(self):
         prev=self.collision.currentText()
@@ -520,12 +480,8 @@ class BasicTab(QWidget):
         ShowDefaultsIniDialog(self.windowIcon(),text,self).exec_()
 
     def show_rejectlist(self):
-        rejectlist = []
-        for (url,note) in rejecturllist.get_list().items():
-            rejectlist.append((None,url,note,note))
-            
         d = RejectListDialog(self,
-                             rejectlist,
+                             rejecturllist.get_list(),
                              rejectreasons=rejecturllist.get_reject_reasons(),
                              header="Edit Reject URLs List",
                              show_delete=False,
@@ -535,11 +491,7 @@ class BasicTab(QWidget):
         if d.result() != d.Accepted:
             return
         
-        rejectlist=[]
-        for (bookid,url,note) in d.get_reject_list():
-            rejectlist.append((url,note))
-
-        rejecturllist.add(rejectlist,clear=True)
+        rejecturllist.add(d.get_reject_list(),clear=True)
         
     def show_reject_reasons(self):
         d = EditTextDialog(self,
@@ -554,11 +506,11 @@ class BasicTab(QWidget):
                 
     def add_reject_urls(self):
         d = EditTextDialog(self,
-                           "http://example.com?story.php?sid=5,Reason why I rejected it",
+                           "http://example.com/story.php?sid=5,Reason why I rejected it\nhttp://example.com/story.php?sid=6,Title by Author - Reason why I rejected it",
                            icon=self.windowIcon(),
                            title="Add Reject URLs",
-                           label="Add Reject URLs. Use: <b>http://...,note</b><br>Invalid story URLs will be ignored.",
-                           tooltip="One URL per line, everything after <b>,</b> will be put in the note.",
+                           label="Add Reject URLs. Use: <b>http://...,note</b> or <b>http://...,title by author - note</b><br>Invalid story URLs will be ignored.",
+                           tooltip="One URL per line:\n<b>http://...,note</b>\n<b>http://...,title by author - note</b>",
                            rejectreasons=rejecturllist.get_reject_reasons(),
                            reasonslabel='Add this reason to all URLs added:')
         d.exec_()
@@ -586,9 +538,9 @@ class PersonalIniTab(QWidget):
         self.ini = QTextEdit(self)
         try:
             self.ini.setFont(QFont("Courier",
-                                   self.plugin_action.gui.font().pointSize()+1));
+                                   self.plugin_action.gui.font().pointSize()+1))
         except Exception as e:
-            print("Couldn't get font: %s"%e)
+            logger.error("Couldn't get font: %s"%e)
         self.ini.setLineWrapMode(QTextEdit.NoWrap)
         self.ini.setText(prefs['personal.ini'])
         self.l.addWidget(self.ini)
@@ -622,9 +574,9 @@ class ShowDefaultsIniDialog(QDialog):
         self.ini.setToolTip("These are all of the plugin's configurable options\nand their default settings.")
         try:
             self.ini.setFont(QFont("Courier",
-                                   get_gui().font().pointSize()+1));
+                                   get_gui().font().pointSize()+1))
         except Exception as e:
-            print("Couldn't get font: %s"%e)
+            logger.error("Couldn't get font: %s"%e)
         self.ini.setLineWrapMode(QTextEdit.NoWrap)
         self.ini.setText(text)
         self.ini.setReadOnly(True)
@@ -911,7 +863,7 @@ titleLabels = {
     'ships':'Relationships',
     'datePublished':'Published',
     'dateUpdated':'Updated',
-    'dateCreated':'Packaged',
+    'dateCreated':'Created',
     'rating':'Rating',
     'warnings':'Warnings',
     'numChapters':'Chapters',
@@ -922,7 +874,7 @@ titleLabels = {
     'extratags':'Extra Tags',
     'title':'Title',
     'storyUrl':'Story URL',
-    'description':'Summary',
+    'description':'Description',
     'author':'Author',
     'authorUrl':'Author URL',
     'formatname':'File Format',

calibre-plugin/dialogs.py

@@ -7,36 +7,42 @@ __license__   = 'GPL v3'
 __copyright__ = '2011, Jim Miller'
 __docformat__ = 'restructuredtext en'
 
-import traceback
+import logging
+logger = logging.getLogger(__name__)
+
+import traceback, re
 from functools import partial
 
+import logging
+logger = logging.getLogger(__name__)
+
+import urllib
+import email
+
 from PyQt4 import QtGui
-from PyQt4.Qt import (QDialog, QTableWidget, QMessageBox, QVBoxLayout, QHBoxLayout,
-                      QGridLayout, QPushButton, QProgressDialog, QString, QLabel,
-                      QCheckBox, QIcon, QTextCursor, QTextEdit, QLineEdit, QInputDialog,
-                      QComboBox, QClipboard, QVariant, QProgressDialog, QTimer,
-                      QDialogButtonBox, QPixmap, Qt, QAbstractItemView, SIGNAL,
-                      QTableWidgetItem )
+from PyQt4.Qt import (QDialog, QTableWidget, QVBoxLayout, QHBoxLayout, QGridLayout,
+                      QPushButton, QString, QLabel, QCheckBox, QIcon, QLineEdit,
+                      QComboBox, QVariant, QProgressDialog, QTimer, QDialogButtonBox,
+                      QPixmap, Qt, QAbstractItemView, SIGNAL, QTextEdit, pyqtSignal,
+                      QGroupBox, QFrame)
 
-from calibre.gui2 import error_dialog, warning_dialog, question_dialog, info_dialog
 from calibre.gui2.dialogs.confirm_delete import confirm
 from calibre.gui2.complete2 import EditWithComplete
 
-from calibre import confirm_config_name
-from calibre.gui2 import dynamic
-
-from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapters,writers,exceptions
 from calibre_plugins.fanfictiondownloader_plugin.common_utils \
     import (ReadOnlyTableWidgetItem, ReadOnlyTextIconWidgetItem, SizePersistedDialog,
             ImageTitleLayout, get_icon)
 
-SKIP='Skip'
-ADDNEW='Add New Book'
-UPDATE='Update EPUB if New Chapters'
-UPDATEALWAYS='Update EPUB Always'
-OVERWRITE='Overwrite if Newer'
-OVERWRITEALWAYS='Overwrite Always'
-CALIBREONLY='Update Calibre Metadata Only'
+from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.geturls import get_urls_from_html, get_urls_from_text
+from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.adapters import getNormalStoryURL
+
+SKIP=u'Skip'
+ADDNEW=u'Add New Book'
+UPDATE=u'Update EPUB if New Chapters'
+UPDATEALWAYS=u'Update EPUB Always'
+OVERWRITE=u'Overwrite if Newer'
+OVERWRITEALWAYS=u'Overwrite Always'
+CALIBREONLY=u'Update Calibre Metadata Only'
 collision_order=[SKIP,
                  ADDNEW,
                  UPDATE,
@@ -44,7 +50,69 @@ collision_order=[SKIP,
                  OVERWRITE,
                  OVERWRITEALWAYS,
                  CALIBREONLY,]
+
+anthology_collision_order=[UPDATE,
+                           UPDATEALWAYS,
+                           OVERWRITEALWAYS]
+
+gpstyle='QGroupBox {border:0; padding-top:10px; padding-bottom:0px; margin-bottom:0px;}' #  background-color:red;
+
+class RejectUrlEntry:
+
+    matchpat=re.compile(r"^(?P<url>[^,]+)(,(?P<fullnote>(((?P<title>.+) by (?P<auth>.+?)( - (?P<note>.+))?)|.*)))?$")
+    
+    def __init__(self,url_or_line,note=None,title=None,auth=None,
+                 addreasontext=None,fromline=False,book_id=None):
+
+        self.url=url_or_line
+        self.note=note
+        self.title=title
+        self.auth=auth
+        self.valid=False
+        self.book_id=book_id
+
+        if fromline:
+            mc = re.match(self.matchpat,url_or_line)
+            if mc:
+                #print("mc:%s"%mc.groupdict())
+                (url,title,auth,note) = mc.group('url','title','auth','note')
+                if not mc.group('title'):
+                    title=''
+                    auth=''
+                    note=mc.group('fullnote')
+                self.url=url
+                self.note=note
+                self.title=title
+                self.auth=auth
         
+        if not self.note:
+            if addreasontext:
+                self.note = addreasontext
+            else:
+                self.note = ''
+        else:
+            if addreasontext:
+                self.note = self.note + ' - ' + addreasontext
+                
+        self.url = getNormalStoryURL(self.url)
+        self.valid = self.url != None
+                
+    def to_line(self):
+        # always 'url,'
+        return self.url+","+self.fullnote()
+        
+    def fullnote(self):
+        retval = ""
+        if self.title and self.auth:
+            retval = retval + "%s by %s"%(self.title,self.auth)
+            if self.note:
+                retval = retval + " - "
+                
+        if self.note:
+            retval = retval + self.note
+            
+        return retval
+
 # This is a more than slightly kludgey way to get
 # EditWithComplete to *not* alpha-order the reasons, but leave
 # them in the order entered.  If
@@ -69,10 +137,40 @@ class NotGoingToDownload(Exception):
 class DroppableQTextEdit(QTextEdit):
     def __init__(self,parent):
         QTextEdit.__init__(self,parent)
+
+    def dropEvent(self,event):
+        # print("event:%s"%event)
+
+        mimetype='text/uri-list'
+
+        urllist=[]
+        filelist="%s"%event.mimeData().data(mimetype)
+        for f in filelist.splitlines():
+            #print("filename:%s"%f)
+            if f.endswith(".eml"):
+                fhandle = urllib.urlopen(f)
+                #print("file:\n%s\n\n"%fhandle.read())
+                msg = email.message_from_file(fhandle)
+                if msg.is_multipart():
+                    for part in msg.walk():
+                        #print("part type:%s"%part.get_content_type())
+                        if part.get_content_type() == "text/html":
+                            #print("URL list:%s"%get_urls_from_data(part.get_payload(decode=True)))
+                            urllist.extend(get_urls_from_html(part.get_payload(decode=True)))
+                        if part.get_content_type() == "text/plain":
+                            #print("part content:text/plain")
+                            # print("part content:%s"%part.get_payload(decode=True))
+                            urllist.extend(get_urls_from_text(part.get_payload(decode=True)))
+                else:
+                    urllist.extend(get_urls_from_text("%s"%msg))
+        
+        if urllist:
+            self.append("\n".join(urllist))
+        return QTextEdit.dropEvent(self,event)
         
     def canInsertFromMimeData(self, source):
         if source.hasUrls():
-            return True;
+            return True
         else:
             return QTextEdit.canInsertFromMimeData(self,source)
 
@@ -84,12 +182,11 @@ class DroppableQTextEdit(QTextEdit):
                             
 class AddNewDialog(SizePersistedDialog):
 
-    def __init__(self, gui, prefs, icon, url_list_text):
-        SizePersistedDialog.__init__(self, gui, 'FanFictionDownLoader plugin:add new dialog')
-        self.gui = gui
+    go_signal = pyqtSignal(object, object, object, object)
 
-        if prefs['adddialogstaysontop']:
-            QDialog.setWindowFlags ( self, Qt.Dialog|Qt.WindowStaysOnTopHint )
+    def __init__(self, gui, prefs, icon):
+        SizePersistedDialog.__init__(self, gui, 'FanFictionDownLoader plugin:add new dialog')
+        self.prefs = prefs
         
         self.setMinimumWidth(300)
         self.l = QVBoxLayout()
@@ -98,85 +195,211 @@ class AddNewDialog(SizePersistedDialog):
         self.setWindowTitle('FanFictionDownLoader')
         self.setWindowIcon(icon)
 
-        self.l.addWidget(QLabel('Story URL(s), one per line:'))
+        self.toplabel=QLabel("Toplabel")
+        self.l.addWidget(self.toplabel)
         self.url = DroppableQTextEdit(self)
-        self.url.setToolTip('URLs for stories, one per line.\nWill take URLs from clipboard, but only valid URLs.\nAdd [1,5] after the URL to limit the download to chapters 1-5.')
+        self.url.setToolTip("UrlTooltip")
         self.url.setLineWrapMode(QTextEdit.NoWrap)
-        self.url.setText(url_list_text)
         self.l.addWidget(self.url)
+
+        self.merge = self.newmerge = False
+        
+        # elements to hide when doing merge.
+        self.mergehide = []
+        # elements to show again when doing *update* merge
+        self.mergeupdateshow = []
+
+        self.groupbox = QGroupBox("Show Download Options")
+        self.groupbox.setCheckable(True)
+        self.groupbox.setChecked(False)
+        self.groupbox.setFlat(True)
+        #print("style:%s"%self.groupbox.styleSheet())
+        self.groupbox.setStyleSheet(gpstyle)
+
+        self.gbf = QFrame()
+        self.gbl = QVBoxLayout()
+        self.gbl.addWidget(self.gbf)
+        self.groupbox.setLayout(self.gbl)
+        self.gbl = QVBoxLayout()
+        self.gbf.setLayout(self.gbl)
+        self.l.addWidget(self.groupbox)
+
+        self.gbf.setVisible(False)
+        self.groupbox.toggled.connect(self.gbf.setVisible)
         
         horz = QHBoxLayout()
         label = QLabel('Output &Format:')
-        horz.addWidget(label)
+        self.mergehide.append(label)
+        
         self.fileform = QComboBox(self)
         self.fileform.addItem('epub')
         self.fileform.addItem('mobi')
         self.fileform.addItem('html')
         self.fileform.addItem('txt')
-        self.fileform.setCurrentIndex(self.fileform.findText(prefs['fileform']))
         self.fileform.setToolTip('Choose output format to create.  May set default from plugin configuration.')
         self.fileform.activated.connect(self.set_collisions)
         
+        horz.addWidget(label)
         label.setBuddy(self.fileform)
         horz.addWidget(self.fileform)
-        self.l.addLayout(horz)
+        self.gbl.addLayout(horz)
+        self.mergehide.append(self.fileform)
 
         horz = QHBoxLayout()
-        label = QLabel('If Story Already Exists?')
-        horz.addWidget(label)
+        self.collisionlabel = QLabel("CollisionLabel")
+        horz.addWidget(self.collisionlabel)
         self.collision = QComboBox(self)
-        self.collision.setToolTip("What to do if there's already an existing story with the same URL or title and author.")
+        self.collision.setToolTip("CollisionToolTip")
         # add collision options
         self.set_collisions()
         i = self.collision.findText(prefs['collision'])
         if i > -1:
             self.collision.setCurrentIndex(i)
-        label.setBuddy(self.collision)
+        self.collisionlabel.setBuddy(self.collision)
         horz.addWidget(self.collision)
-        self.l.addLayout(horz)
+        self.gbl.addLayout(horz)
+        self.mergehide.append(self.collisionlabel)
+        self.mergehide.append(self.collision)
+        self.mergeupdateshow.append(self.collisionlabel)
+        self.mergeupdateshow.append(self.collision)
 
         horz = QHBoxLayout()
         self.updatemeta = QCheckBox('Update Calibre &Metadata?',self)
         self.updatemeta.setToolTip("Update metadata for existing stories in Calibre from web site?\n(Columns set to 'New Only' in the column tabs will only be set for new books.)")
         self.updatemeta.setChecked(prefs['updatemeta'])
         horz.addWidget(self.updatemeta)
+        self.mergehide.append(self.updatemeta)
+        self.mergeupdateshow.append(self.updatemeta)
 
         self.updateepubcover = QCheckBox('Update EPUB Cover?',self)
         self.updateepubcover.setToolTip('Update book cover image from site or defaults (if found) <i>inside</i> the EPUB when EPUB is updated.')
         self.updateepubcover.setChecked(prefs['updateepubcover'])
         horz.addWidget(self.updateepubcover)
+        self.mergehide.append(self.updateepubcover)
         
-        self.l.addLayout(horz)
+        self.gbl.addLayout(horz)
 
-        button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
-        button_box.accepted.connect(self.accept)
-        button_box.rejected.connect(self.reject)
-        self.l.addWidget(button_box)
+        self.button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
+        self.button_box.accepted.connect(self.ok_clicked)
+        self.button_box.rejected.connect(self.reject)
+        self.l.addWidget(self.button_box)
+
+    # invoke the 
+    def ok_clicked(self):
+        self.dialog_closing(None) # save persistent size.
+        self.hide()
+        self.go_signal.emit( self.get_ffdl_options(),
+                             self.get_urlstext(),
+                             self.merge,
+                             self.extrapayload )
+
+    def show_dialog(self,
+                    url_list_text,
+                    callback,
+                    show=True,
+                    merge=False,
+                    newmerge=True,
+                    extraoptions={},
+                    extrapayload=None):
+        # rather than mutex in ffdl_plugin, just bail here if it's
+        # already in use.
+        if self.isVisible(): return
+
+        try:
+            self.go_signal.disconnect()
+        except:
+            pass # if not already connected.
+        self.go_signal.connect(callback)
+
+        self.merge = merge
+        self.newmerge = newmerge
+        self.extraoptions = extraoptions
+        self.extrapayload = extrapayload
+
+        self.groupbox.setVisible(not(self.merge and self.newmerge))
         
+        if self.merge:
+            self.toplabel.setText('Story URL(s) for anthology, one per line:')
+            self.url.setToolTip('URLs for stories to include in the anthology, one per line.\nWill take URLs from clipboard, but only valid URLs.')
+            self.collisionlabel.setText('If Story Already Exists in Anthology?')
+            self.collision.setToolTip("What to do if there's already an existing story with the same URL in the anthology.")
+            for widget in self.mergehide:
+                widget.setVisible(False)
+            if not self.newmerge:
+                for widget in self.mergeupdateshow:
+                    widget.setVisible(True)
+        else:
+            for widget in self.mergehide:
+                widget.setVisible(True)
+            self.toplabel.setText('Story URL(s), one per line:')
+            self.url.setToolTip('URLs for stories, one per line.\nWill take URLs from clipboard, but only valid URLs.\nAdd [1,5] after the URL to limit the download to chapters 1-5.')
+            self.collisionlabel.setText('If Story Already Exists?')
+            self.collision.setToolTip("What to do if there's already an existing story with the same URL or title and author.")
+
+        # Need to re-able after hiding/showing
+        self.setAcceptDrops(True)
+        self.url.setFocus()
+            
+        if self.prefs['adddialogstaysontop']:
+            QDialog.setWindowFlags ( self, Qt.Dialog | Qt.WindowStaysOnTopHint )
+        else:
+            QDialog.setWindowFlags ( self, Qt.Dialog )
+
+        if not self.merge:
+            self.fileform.setCurrentIndex(self.fileform.findText(self.prefs['fileform']))
+
+        # add collision options
+        self.set_collisions()
+        
+        i = self.collision.findText(self.prefs['collision'])
+        if i > -1:
+            self.collision.setCurrentIndex(i)
+        self.updatemeta.setChecked(self.prefs['updatemeta'])
+            
+        if not self.merge:
+            self.updateepubcover.setChecked(self.prefs['updateepubcover'])
+                
+        self.url.setText(url_list_text)
         if url_list_text:
-            button_box.button(QDialogButtonBox.Ok).setFocus()
-
+            self.button_box.button(QDialogButtonBox.Ok).setFocus()
         # restore saved size.
         self.resize_dialog()
+        
+        if show: # so anthology update can be modal still.
+            self.show()
         #self.resize(self.sizeHint())
 
     def set_collisions(self):
         prev=self.collision.currentText()
         self.collision.clear()
-        for o in collision_order:
-            if self.fileform.currentText() == 'epub' or o not in [UPDATE,UPDATEALWAYS]:
+        if self.merge:
+            order = anthology_collision_order
+        else:
+            order = collision_order
+        for o in order:
+            if self.merge or self.fileform.currentText() == 'epub' or o not in [UPDATE,UPDATEALWAYS]:
                 self.collision.addItem(o)
         i = self.collision.findText(prev)
         if i > -1:
             self.collision.setCurrentIndex(i)
         
     def get_ffdl_options(self):
-        return {
+        retval =  {
             'fileform': unicode(self.fileform.currentText()),
             'collision': unicode(self.collision.currentText()),
             'updatemeta': self.updatemeta.isChecked(),
             'updateepubcover': self.updateepubcover.isChecked(),
-            }
+            'smarten_punctuation':self.prefs['smarten_punctuation']
+                }
+        
+        if self.merge:
+            retval['fileform']=='epub'
+            retval['updateepubcover']=True
+            if self.newmerge:
+                retval['updatemeta']=True
+                retval['collision']=ADDNEW
+            
+        return dict(retval.items() + self.extraoptions.items() )
 
     def get_urlstext(self):
         return unicode(self.url.toPlainText())
@@ -193,10 +416,10 @@ class CollectURLDialog(SizePersistedDialog):
     '''
     Collect single url for get urls.
     '''
-    def __init__(self, gui, title, url_text): 
+    def __init__(self, gui, title, url_text, epubmerge_plugin=None): 
         SizePersistedDialog.__init__(self, gui, 'FanFictionDownLoader plugin:get story urls')
-        self.gui = gui
         self.status=False
+        self.anthology=False
 
         self.setMinimumWidth(300)
         
@@ -204,28 +427,40 @@ class CollectURLDialog(SizePersistedDialog):
         self.setLayout(self.l)
 
         self.setWindowTitle(title)
-        self.l.addWidget(QLabel(title),0,0,1,2)
+        self.l.addWidget(QLabel(title),0,0,1,3)
         
         self.l.addWidget(QLabel("URL:"),1,0)
         self.url = QLineEdit(self)
         self.url.setText(url_text)
-        self.l.addWidget(self.url,1,1)
+        self.l.addWidget(self.url,1,1,1,2)
    
-        self.ok_button = QPushButton('OK', self)
-        self.ok_button.clicked.connect(self.ok)
-        self.l.addWidget(self.ok_button,2,0)
+        self.indiv_button = QPushButton('For Individual Books', self)
+        self.indiv_button.setToolTip('Get URLs and go to dialog for individual story downloads.')
+        self.indiv_button.clicked.connect(self.indiv)
+        self.l.addWidget(self.indiv_button,2,0)
+
+        self.merge_button = QPushButton('For Anthology Epub', self)
+        self.merge_button.setToolTip('Get URLs and go to dialog for Anthology download.\nRequires EpubMerge 1.3.1+ plugin.')
+        self.merge_button.clicked.connect(self.merge)
+        self.l.addWidget(self.merge_button,2,1)
+        self.merge_button.setEnabled(epubmerge_plugin!=None)
 
         self.cancel_button = QPushButton('Cancel', self)
         self.cancel_button.clicked.connect(self.cancel)
-        self.l.addWidget(self.cancel_button,2,1)
+        self.l.addWidget(self.cancel_button,2,2)
 
         # restore saved size.
         self.resize_dialog()
 
-    def ok(self):
+    def indiv(self):
         self.status=True
         self.accept()
 
+    def merge(self):
+        self.status=True
+        self.anthology=True
+        self.accept()
+
     def cancel(self):
         self.status=False
         self.reject()
@@ -236,7 +471,6 @@ class UserPassDialog(QDialog):
     '''
     def __init__(self, gui, site, exception=None):
         QDialog.__init__(self, gui)
-        self.gui = gui
         self.status=False
 
         self.l = QGridLayout()
@@ -295,7 +529,6 @@ class LoopProgressDialog(QProgressDialog):
                                  QString(), 0, len(book_list), gui)
         self.setWindowTitle(win_title)
         self.setMinimumWidth(500)
-        self.gui = gui
         self.book_list = book_list
         self.foreach_function = foreach_function
         self.finish_function = finish_function
@@ -310,7 +543,7 @@ class LoopProgressDialog(QProgressDialog):
     def updateStatus(self):
         self.setLabelText("%s %d of %d"%(self.status_prefix,self.i+1,len(self.book_list)))
         self.setValue(self.i+1)
-        print(self.labelText())
+        #print(self.labelText())
 
     def do_loop(self):
 
@@ -331,7 +564,7 @@ class LoopProgressDialog(QProgressDialog):
         except Exception as e:
             book['good']=False
             book['comment']=unicode(e)
-            print("Exception: %s:%s"%(book,unicode(e)))
+            logger.error("Exception: %s:%s"%(book,unicode(e)))
             traceback.print_exc()
             
         self.updateStatus()
@@ -344,7 +577,6 @@ class LoopProgressDialog(QProgressDialog):
 
     def do_when_finished(self):
         self.hide()
-        self.gui = None        
         # Queues a job to process these books in the background.
         self.finish_function(self.book_list)
 
@@ -393,8 +625,8 @@ class UpdateExistingDialog(SizePersistedDialog):
     def __init__(self, gui, header, prefs, icon, books,
                  save_size_name='fanfictiondownloader_plugin:update list dialog'):
         SizePersistedDialog.__init__(self, gui, save_size_name)
-        self.gui = gui
-        
+
+        self.prefs = prefs
         self.setWindowTitle(header)
         self.setWindowIcon(icon)
         
@@ -411,11 +643,7 @@ class UpdateExistingDialog(SizePersistedDialog):
 
         button_layout = QVBoxLayout()
         books_layout.addLayout(button_layout)
-        # self.move_up_button = QtGui.QToolButton(self)
-        # self.move_up_button.setToolTip('Move selected books up the list')
-        # self.move_up_button.setIcon(QIcon(I('arrow-up.png')))
-        # self.move_up_button.clicked.connect(self.books_table.move_rows_up)
-        # button_layout.addWidget(self.move_up_button)
+
         spacerItem = QtGui.QSpacerItem(20, 40, QtGui.QSizePolicy.Minimum, QtGui.QSizePolicy.Expanding)
         button_layout.addItem(spacerItem)
         self.remove_button = QtGui.QToolButton(self)
@@ -425,16 +653,28 @@ class UpdateExistingDialog(SizePersistedDialog):
         button_layout.addWidget(self.remove_button)
         spacerItem1 = QtGui.QSpacerItem(20, 40, QtGui.QSizePolicy.Minimum, QtGui.QSizePolicy.Expanding)
         button_layout.addItem(spacerItem1)
-        # self.move_down_button = QtGui.QToolButton(self)
-        # self.move_down_button.setToolTip('Move selected books down the list')
-        # self.move_down_button.setIcon(QIcon(I('arrow-down.png')))
-        # self.move_down_button.clicked.connect(self.books_table.move_rows_down)
-        # button_layout.addWidget(self.move_down_button)
 
         options_layout = QHBoxLayout()
 
+        groupbox = QGroupBox("Show Download Options")
+        groupbox.setCheckable(True)
+        groupbox.setChecked(False)
+        groupbox.setFlat(True)
+        groupbox.setStyleSheet(gpstyle)
+
+        gbf = QFrame()
+        gbl = QVBoxLayout()
+        gbl.addWidget(gbf)
+        groupbox.setLayout(gbl)
+        gbl = QHBoxLayout()
+        gbf.setLayout(gbl)
+        options_layout.addWidget(groupbox)
+
+        gbf.setVisible(False)
+        groupbox.toggled.connect(gbf.setVisible)
+        
         label = QLabel('Output &Format:')
-        options_layout.addWidget(label)
+        gbl.addWidget(label)
         self.fileform = QComboBox(self)
         self.fileform.addItem('epub')
         self.fileform.addItem('mobi')
@@ -444,10 +684,10 @@ class UpdateExistingDialog(SizePersistedDialog):
         self.fileform.setToolTip('Choose output format to create.  May set default from plugin configuration.')
         self.fileform.activated.connect(self.set_collisions)
         label.setBuddy(self.fileform)
-        options_layout.addWidget(self.fileform)
+        gbl.addWidget(self.fileform)
         
         label = QLabel('Update Mode:')
-        options_layout.addWidget(label)
+        gbl.addWidget(label)
         self.collision = QComboBox(self)
         self.collision.setToolTip("What sort of update to perform.  May set default from plugin configuration.")
         # add collision options
@@ -455,19 +695,20 @@ class UpdateExistingDialog(SizePersistedDialog):
         i = self.collision.findText(prefs['collision'])
         if i > -1:
             self.collision.setCurrentIndex(i)
-        # self.collision.setToolTip('Overwrite will replace the existing story.  Add New will create a new story with the same title and author.')
         label.setBuddy(self.collision)
-        options_layout.addWidget(self.collision)
+        gbl.addWidget(self.collision)
 
         self.updatemeta = QCheckBox('Update Calibre &Metadata?',self)
         self.updatemeta.setToolTip("Update metadata for existing stories in Calibre from web site?\n(Columns set to 'New Only' in the column tabs will only be set for new books.)")
         self.updatemeta.setChecked(prefs['updatemeta'])
-        options_layout.addWidget(self.updatemeta)
+        gbl.addWidget(self.updatemeta)
                 
         self.updateepubcover = QCheckBox('Update EPUB Cover?',self)
         self.updateepubcover.setToolTip('Update book cover image from site or defaults (if found) <i>inside</i> the EPUB when EPUB is updated.')
         self.updateepubcover.setChecked(prefs['updateepubcover'])
-        options_layout.addWidget(self.updateepubcover)
+        gbl.addWidget(self.updateepubcover)
+
+
         
         button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
         button_box.accepted.connect(self.accept)
@@ -503,86 +744,9 @@ class UpdateExistingDialog(SizePersistedDialog):
             'collision': unicode(self.collision.currentText()),
             'updatemeta': self.updatemeta.isChecked(),
             'updateepubcover': self.updateepubcover.isChecked(),
+            'smarten_punctuation':self.prefs['smarten_punctuation']
             }
 
-def display_story_list(gui, header, prefs, icon, books,
-                       label_text='',
-                       save_size_name='fanfictiondownloader_plugin:display list dialog',
-                       offer_skip=False):
-    all_good = True
-    for b in books:
-        if not b['good']:
-            all_good=False
-            break
-        
-    ## 
-    if all_good and not dynamic.get(confirm_config_name(save_size_name), True):
-        return True
-        pass
-            ## fake accept?
-    d = DisplayStoryListDialog(gui, header, prefs, icon, books,
-                               label_text,
-                               save_size_name,
-                               offer_skip and all_good)
-    d.exec_()
-    return d.result() == d.Accepted
-
-class DisplayStoryListDialog(SizePersistedDialog):
-    def __init__(self, gui, header, prefs, icon, books,
-                 label_text='',
-                 save_size_name='fanfictiondownloader_plugin:display list dialog',
-                 offer_skip=False):
-        SizePersistedDialog.__init__(self, gui, save_size_name)
-        self.name = save_size_name
-        self.gui = gui
-        
-        self.setWindowTitle(header)
-        self.setWindowIcon(icon)
-        
-        layout = QVBoxLayout(self)
-        self.setLayout(layout)
-        title_layout = ImageTitleLayout(self, 'images/icon.png',
-                                        header)
-        layout.addLayout(title_layout)
-
-        self.books_table = StoryListTableWidget(self)
-        layout.addWidget(self.books_table)
-
-        options_layout = QHBoxLayout()
-        self.label = QLabel(label_text)
-        #self.label.setOpenExternalLinks(True)
-        #self.label.setWordWrap(True)
-        options_layout.addWidget(self.label)
-        
-        if offer_skip:
-            spacerItem1 = QtGui.QSpacerItem(2, 4, QtGui.QSizePolicy.Expanding, QtGui.QSizePolicy.Minimum)
-            options_layout.addItem(spacerItem1)
-            self.again = QCheckBox('Show this again?',self)
-            self.again.setChecked(True)
-            self.again.stateChanged.connect(self.toggle)
-            self.again.setToolTip('Uncheck to skip review and update stories immediately when no problems.')
-            options_layout.addWidget(self.again)
-        
-        button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
-        button_box.accepted.connect(self.accept)
-        button_box.rejected.connect(self.reject)
-
-        options_layout.addWidget(button_box)
-        
-        layout.addLayout(options_layout)
-        
-        # Cause our dialog size to be restored from prefs or created on first usage
-        self.resize_dialog()
-        self.books_table.populate_table(books)
-
-    def get_books(self):
-        return self.books_table.get_books()
-
-    def toggle(self, *args):
-        dynamic[confirm_config_name(self.name)] = self.again.isChecked()
-
-    
-
 class StoryListTableWidget(QTableWidget):
 
     def __init__(self, parent):
@@ -642,11 +806,9 @@ class StoryListTableWidget(QTableWidget):
         self.setItem(row, 2, AuthorTableWidgetItem(", ".join(book['author']), ", ".join(book['author_sort'])))
         
         url_cell = ReadOnlyTableWidgetItem(book['url'])
-        #url_cell.setData(Qt.UserRole, QVariant(book['url']))
         self.setItem(row, 3, url_cell)
         
         comment_cell = ReadOnlyTableWidgetItem(book['comment'])
-        #comment_cell.setData(Qt.UserRole, QVariant(book))
         self.setItem(row, 4, comment_cell)
 
     def get_books(self):
@@ -680,56 +842,6 @@ class StoryListTableWidget(QTableWidget):
         self.selectRow(row)
         self.scrollToItem(self.currentItem())
 
-    def move_rows_up(self):
-        self.setFocus()
-        rows = self.selectionModel().selectedRows()
-        if len(rows) == 0:
-            return
-        first_sel_row = rows[0].row()
-        if first_sel_row <= 0:
-            return
-        # Workaround for strange selection bug in Qt which "alters" the selection
-        # in certain circumstances which meant move down only worked properly "once"
-        selrows = []
-        for row in rows:
-            selrows.append(row.row())
-        selrows.sort()
-        for selrow in selrows:
-            self.swap_row_widgets(selrow - 1, selrow + 1)
-        scroll_to_row = first_sel_row - 1
-        if scroll_to_row > 0:
-            scroll_to_row = scroll_to_row - 1
-        self.scrollToItem(self.item(scroll_to_row, 0))
-
-    def move_rows_down(self):
-        self.setFocus()
-        rows = self.selectionModel().selectedRows()
-        if len(rows) == 0:
-            return
-        last_sel_row = rows[-1].row()
-        if last_sel_row == self.rowCount() - 1:
-            return
-        # Workaround for strange selection bug in Qt which "alters" the selection
-        # in certain circumstances which meant move down only worked properly "once"
-        selrows = []
-        for row in rows:
-            selrows.append(row.row())
-        selrows.sort()
-        for selrow in reversed(selrows):
-            self.swap_row_widgets(selrow + 2, selrow)
-        scroll_to_row = last_sel_row + 1
-        if scroll_to_row < self.rowCount() - 1:
-            scroll_to_row = scroll_to_row + 1
-        self.scrollToItem(self.item(scroll_to_row, 0))
-
-    def swap_row_widgets(self, src_row, dest_row):
-        self.blockSignals(True)
-        self.insertRow(dest_row)
-        for col in range(0, self.columnCount()):
-            self.setItem(dest_row, col, self.takeItem(src_row, col))
-        self.removeRow(src_row)
-        self.blockSignals(False)
-
 class RejectListTableWidget(QTableWidget):
 
     def __init__(self, parent,rejectreasons=[]):
@@ -737,85 +849,55 @@ class RejectListTableWidget(QTableWidget):
         self.setSelectionBehavior(QAbstractItemView.SelectRows)
         self.rejectreasons = rejectreasons
 
-    def on_headersection_clicked(self):
-        self.setSortingEnabled(True)
-        
     def populate_table(self, reject_list):
         self.clear()
         self.setAlternatingRowColors(True)
         self.setRowCount(len(reject_list))
-        header_labels = ['URL', 'Note']
+        header_labels = ['URL', 'Title', 'Author', 'Note']
         self.setColumnCount(len(header_labels))
         self.setHorizontalHeaderLabels(header_labels)
         self.horizontalHeader().setStretchLastSection(True)
         #self.verticalHeader().setDefaultSectionSize(24)
         self.verticalHeader().hide()
 
-        # need sortingEnbled to sort, but off to up & down.
-        self.connect(self.horizontalHeader(),
-                     SIGNAL('sectionClicked(int)'),
-                     self.on_headersection_clicked)
-
+        # it's generally recommended to enable sort after pop, not
+        # before.  But then it needs to be sorted on a column and I'd
+        # rather keep the order given.
+        self.setSortingEnabled(True)
         # row is just row number.
         for row, rejectrow in enumerate(reject_list):
+            #print("populating table:%s"%rejectrow.to_line())
             self.populate_table_row(row,rejectrow)
 
         self.resizeColumnsToContents()
-        self.setMinimumColumnWidth(1, 100)
-        self.setMinimumColumnWidth(2, 100)
+        self.setMinimumColumnWidth(0, 100)
+        self.setMinimumColumnWidth(3, 100)
         self.setMinimumSize(300, 0)
 
     def setMinimumColumnWidth(self, col, minimum):
         if self.columnWidth(col) < minimum:
             self.setColumnWidth(col, minimum)
 
-    def populate_table_row(self, row, rejectrow):
+    def populate_table_row(self, row, rej):
 
-        (bookid,url,titleauth,oldrejnote) = rejectrow
-        if oldrejnote:
-            noteprefix = note = oldrejnote
-            # incase the existing note ends with one of the known reasons.
-            for reason in self.rejectreasons:
-                if noteprefix.endswith(' - '+reason):
-                    noteprefix = noteprefix[:-len(' - '+reason)]
-                    break
-        else:
-            noteprefix = note = titleauth
-            
-        if len(noteprefix) > 0:
-            noteprefix = noteprefix+' - '
-            
-        url_cell = ReadOnlyTableWidgetItem(url)
-        url_cell.setData(Qt.UserRole, QVariant(bookid))
-        url_cell.setToolTip('URL to add to the Reject List.')
+        url_cell = ReadOnlyTableWidgetItem(rej.url)
+        url_cell.setData(Qt.UserRole, QVariant(rej.book_id))
         self.setItem(row, 0, url_cell)
-
+        self.setItem(row, 1, ReadOnlyTableWidgetItem(rej.title))
+        self.setItem(row, 2, ReadOnlyTableWidgetItem(rej.auth))
+        
         note_cell = EditWithComplete(self)
-
         note_cell.lineEdit().mcompleter.model().set_items = \
             partial(complete_model_set_items_kludge,
                     note_cell.lineEdit().mcompleter.model())
         
-        items = [note]+[ noteprefix+x for x in self.rejectreasons ]
+        items = [rej.note]+self.rejectreasons
         note_cell.update_items_cache(items)
-        note_cell.show_initial_value(note)
+        note_cell.show_initial_value(rej.note)
         note_cell.set_separator(None)
         note_cell.setToolTip('Select or Edit Reject Note.')
-        self.setCellWidget(row, 1, note_cell)
+        self.setCellWidget(row, 3, note_cell)
         
-        # note_cell = QTableWidgetItem(note)
-        # note_cell.setToolTip('Double-click to edit note.')
-        # self.setItem(row, 1, note_cell)
-      
-    def get_reject_list(self):
-        rejectrows = []
-        for row in range(self.rowCount()):
-            bookid = self.item(row, 0).data(Qt.UserRole).toPyObject()
-            url = unicode(self.item(row, 0).text())
-            note = unicode(self.cellWidget(row, 1).currentText()).strip()
-            rejectrows.append((bookid,url,note))
-        return rejectrows
-
     def remove_selected_rows(self):
         self.setFocus()
         rows = self.selectionModel().selectedRows()
@@ -838,57 +920,6 @@ class RejectListTableWidget(QTableWidget):
         self.selectRow(row)
         self.scrollToItem(self.currentItem())
 
-    def move_rows_up(self):
-        self.setFocus()
-        rows = self.selectionModel().selectedRows()
-        if len(rows) == 0:
-            return
-        first_sel_row = rows[0].row()
-        if first_sel_row <= 0:
-            return
-        # Workaround for strange selection bug in Qt which "alters" the selection
-        # in certain circumstances which meant move down only worked properly "once"
-        selrows = []
-        for row in rows:
-            selrows.append(row.row())
-        selrows.sort()
-        for selrow in selrows:
-            self.swap_row_widgets(selrow - 1, selrow + 1)
-        scroll_to_row = first_sel_row - 1
-        if scroll_to_row > 0:
-            scroll_to_row = scroll_to_row - 1
-        self.scrollToItem(self.item(scroll_to_row, 0))
-
-    def move_rows_down(self):
-        self.setFocus()
-        rows = self.selectionModel().selectedRows()
-        if len(rows) == 0:
-            return
-        last_sel_row = rows[-1].row()
-        if last_sel_row == self.rowCount() - 1:
-            return
-        # Workaround for strange selection bug in Qt which "alters" the selection
-        # in certain circumstances which meant move down only worked properly "once"
-        selrows = []
-        for row in rows:
-            selrows.append(row.row())
-        selrows.sort()
-        for selrow in reversed(selrows):
-            self.swap_row_widgets(selrow + 2, selrow)
-        scroll_to_row = last_sel_row + 1
-        if scroll_to_row < self.rowCount() - 1:
-            scroll_to_row = scroll_to_row + 1
-        self.scrollToItem(self.item(scroll_to_row, 0))
-
-    def swap_row_widgets(self, src_row, dest_row):
-        self.blockSignals(True)
-        self.setSortingEnabled(False)
-        self.insertRow(dest_row)
-        for col in range(0, self.columnCount()):
-            self.setItem(dest_row, col, self.takeItem(src_row, col))
-        self.removeRow(src_row)
-        self.blockSignals(False)
-
 class RejectListDialog(SizePersistedDialog):
     def __init__(self, gui, reject_list,
                  rejectreasons=[],
@@ -898,7 +929,6 @@ class RejectListDialog(SizePersistedDialog):
                  show_all_reasons=True,
                  save_size_name='ffdl:reject list dialog'):
         SizePersistedDialog.__init__(self, gui, save_size_name)
-        self.gui = gui
       
         self.setWindowTitle(header)
         self.setWindowIcon(get_icon(icon))
@@ -918,21 +948,13 @@ class RejectListDialog(SizePersistedDialog):
         rejects_layout.addLayout(button_layout)
         spacerItem = QtGui.QSpacerItem(20, 40, QtGui.QSizePolicy.Minimum, QtGui.QSizePolicy.Expanding)
         button_layout.addItem(spacerItem)
-        # self.move_up_button = QtGui.QToolButton(self)
-        # self.move_up_button.setToolTip('Move selected books up the list')
-        # self.move_up_button.setIcon(QIcon(I('arrow-up.png')))
-        # self.move_up_button.clicked.connect(self.books_table.move_rows_up)
-        # button_layout.addWidget(self.move_up_button)
+        
         self.remove_button = QtGui.QToolButton(self)
         self.remove_button.setToolTip('Remove selected URL(s) from the list')
         self.remove_button.setIcon(get_icon('list_remove.png'))
         self.remove_button.clicked.connect(self.remove_from_list)
         button_layout.addWidget(self.remove_button)
-        # self.move_down_button = QtGui.QToolButton(self)
-        # self.move_down_button.setToolTip('Move selected books down the list')
-        # self.move_down_button.setIcon(QIcon(I('arrow-down.png')))
-        # self.move_down_button.clicked.connect(self.books_table.move_rows_down)
-        # button_layout.addWidget(self.move_down_button)
+
         spacerItem1 = QtGui.QSpacerItem(20, 40, QtGui.QSizePolicy.Minimum, QtGui.QSizePolicy.Expanding)
         button_layout.addItem(spacerItem1)
 
@@ -979,10 +1001,30 @@ class RejectListDialog(SizePersistedDialog):
         self.rejects_table.remove_selected_rows()
 
     def get_reject_list(self):
-        return self.rejects_table.get_reject_list()
+        rejectrows = []
+        for row in range(self.rejects_table.rowCount()):
+            url = unicode(self.rejects_table.item(row, 0).text()).strip()
+            book_id = self.rejects_table.item(row, 0).data(Qt.UserRole).toPyObject()
+            title = unicode(self.rejects_table.item(row, 1).text()).strip()
+            auth = unicode(self.rejects_table.item(row, 2).text()).strip()
+            note = unicode(self.rejects_table.cellWidget(row, 3).currentText()).strip()
+            rejectrows.append(RejectUrlEntry(url,note,title,auth,self.get_reason_text(),book_id=book_id))
+        return rejectrows
+
+    def get_reject_list_ids(self):
+        rejectrows = []
+        for row in range(self.rejects_table.rowCount()):
+            book_id = self.rejects_table.item(row, 0).data(Qt.UserRole).toPyObject()
+            if book_id:
+                rejectrows.append(book_id)
+        return rejectrows
 
     def get_reason_text(self):
-        return unicode(self.reason_edit.currentText()).strip()
+        try:
+            return unicode(self.reason_edit.currentText()).strip()
+        except:
+            # doesn't have self.reason_edit when editing existing list.
+            return None
     
     def get_deletebooks(self):
         return self.deletebooks.isChecked()

calibre-plugin/ffdl_util.py

@@ -0,0 +1,43 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2013, Jim Miller'
+__docformat__ = 'restructuredtext en'
+
+from StringIO import StringIO
+
+from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapters, exceptions
+from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.configurable import Configuration
+from calibre_plugins.fanfictiondownloader_plugin.prefs import (prefs)
+
+def get_ffdl_personalini():
+    if prefs['includeimages']:
+        # this is a cheat to make it easier for users.
+        return '''[epub]
+include_images:true
+keep_summary_html:true
+make_firstimage_cover:true
+''' + prefs['personal.ini']
+    else:
+        return prefs['personal.ini']
+
+def get_ffdl_config(url,fileform="epub",personalini=None):
+    if not personalini:
+        personalini = get_ffdl_personalini()
+    site='unknown'
+    try:
+        site = adapters.getConfigSectionFor(url)
+    except Exception as e:
+        print("Failed trying to get ini config for url(%s): %s, using section [%s] instead"%(url,e,site))
+    configuration = Configuration(site,fileform)
+    configuration.readfp(StringIO(get_resources("plugin-defaults.ini")))
+    configuration.readfp(StringIO(personalini))
+
+    return configuration
+
+def get_ffdl_adapter(url,fileform="epub",personalini=None):
+    return adapters.getAdapter(get_ffdl_config(url,fileform,personalini),url)
+    

calibre-plugin/jobs.py

@@ -8,19 +8,23 @@ __copyright__ = '2012, Jim Miller'
 __copyright__ = '2011, Grant Drake <grant.drake@gmail.com>'
 __docformat__ = 'restructuredtext en'
 
+import logging
+logger = logging.getLogger(__name__)
+
 import time, os, traceback
 
 from StringIO import StringIO
 
 from calibre.utils.ipc.server import Server
 from calibre.utils.ipc.job import ParallelJob
+from calibre.constants import numeric_version as calibre_version
 
 from calibre_plugins.fanfictiondownloader_plugin.dialogs import (NotGoingToDownload,
     OVERWRITE, OVERWRITEALWAYS, UPDATE, UPDATEALWAYS, ADDNEW, SKIP, CALIBREONLY)
 from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapters, writers, exceptions
-from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.configurable import Configuration
 from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.epubutils import get_update_data
 
+from calibre_plugins.fanfictiondownloader_plugin.ffdl_util import (get_ffdl_adapter, get_ffdl_config)
 # ------------------------------------------------------------------------------
 #
 #              Functions to perform downloads using worker jobs
@@ -37,18 +41,19 @@ def do_download_worker(book_list, options,
     '''
     server = Server(pool_size=cpus)
 
-    print(options['version'])
+    logger.info(options['version'])
     total = 0
+    alreadybad = []
     # Queue all the jobs
-    print("Adding jobs for URLs:")
+    logger.info("Adding jobs for URLs:")
     for book in book_list:
+        logger.info("%s"%book['url'])
         if book['good']:
-            print("%s"%book['url'])
             total += 1
             args = ['calibre_plugins.fanfictiondownloader_plugin.jobs',
                     'do_download_for_worker',
                     (book,options)]
-            job = ParallelJob('arbitrary',
+            job = ParallelJob('arbitrary_n',
                               "url:(%s) id:(%s)"%(book['url'],book['calibre_id']),
                               done=None,
                               args=args)
@@ -58,6 +63,9 @@ def do_download_worker(book_list, options,
             # job._modified_date = modified_date
             # job._existing_isbn = existing_isbn
             server.add_job(job)
+        else:
+            # was already bad before the subprocess ever started.
+            alreadybad.append(book)
     
     # This server is an arbitrary_n job, so there is a notifier available.
     # Set the % complete to a small number to avoid the 'unavailable' indicator
@@ -80,21 +88,21 @@ def do_download_worker(book_list, options,
         book_id = job._book['calibre_id']
         #title = job._title
         count = count + 1
-        notification(float(count)/total, 'Downloaded Story')
+        notification(float(count)/total, '%d of %d stories finished downloading'%(count,total))
         # Add this job's output to the current log
-        print('Logfile for book ID %s (%s)'%(book_id, job._book['title']))
-        print(job.details)
+        logger.info('Logfile for book ID %s (%s)'%(book_id, job._book['title']))
+        logger.info(job.details)
 
         if count >= total:
             # All done!  Output some lists for convenience of some users.
-            print("Successfully downloaded:")
+            logger.info("Successfully downloaded:")
             for book in book_list:
                 if book['good']:
-                    print(book['title'])
-            print("\nUnsuccessful:")
+                    logger.info("%s %s"%(book['title'],book['url']))
+            logger.info("\nUnsuccessful:")
             for book in book_list:
                 if not book['good']:
-                    print(book['title'])
+                    logger.info("%s %s"%(book['title'],book['url']))
             break
 
     server.close()
@@ -102,7 +110,7 @@ def do_download_worker(book_list, options,
     # return the book list as the job result
     return book_list
 
-def do_download_for_worker(book,options):
+def do_download_for_worker(book,options,notification=lambda x,y:x):
     '''
     Child job, to extract isbn from formats for this specific book,
     when run as a worker job
@@ -110,9 +118,9 @@ def do_download_for_worker(book,options):
     try:
         book['comment'] = 'Download started...'
 
-        configuration = Configuration(adapters.getConfigSectionFor(book['url']),options['fileform'])
-        configuration.readfp(StringIO(get_resources("plugin-defaults.ini")))
-        configuration.readfp(StringIO(options['personal.ini']))
+        configuration = get_ffdl_config(book['url'],
+                                        options['fileform'],
+                                        options['personal.ini'])
         
         if not options['updateepubcover'] and 'epub_for_update' in book and options['collision'] in (UPDATE, UPDATEALWAYS):
             configuration.set("overrides","never_make_cover","true")
@@ -142,14 +150,30 @@ def do_download_for_worker(book,options):
 
         ## No need to download at all.  Shouldn't ever get down here.
         if options['collision'] in (CALIBREONLY):
-            print("Skipping CALIBREONLY 'update' down inside worker--this shouldn't be happening...")
+            logger.info("Skipping CALIBREONLY 'update' down inside worker--this shouldn't be happening...")
             book['comment'] = 'Metadata collected.'
             
         ## checks were done earlier, it's new or not dup or newer--just write it.
         elif options['collision'] in (ADDNEW, SKIP, OVERWRITE, OVERWRITEALWAYS) or \
                 ('epub_for_update' not in book and options['collision'] in (UPDATE, UPDATEALWAYS)):
 
-            print("write to %s"%outfile)
+            # preserve logfile even on overwrite.
+            if 'epub_for_update' in book:
+                (urlignore,
+                 chaptercountignore,
+                 oldchaptersignore,
+                 oldimgsignore,
+                 oldcoverignore,
+                 calibrebookmarkignore,
+                 # only logfile set in adapter, so others aren't used.
+                 adapter.logfile) = get_update_data(book['epub_for_update'])
+
+                # change the existing entries id to notid so
+                # write_epub writes a whole new set to indicate overwrite.
+                if adapter.logfile:
+                    adapter.logfile = adapter.logfile.replace("span id","span notid")
+            
+            logger.info("write to %s"%outfile)
             writer.writeStory(outfilename=outfile, forceOverwrite=True)
             book['comment'] = 'Download %s completed, %s chapters.'%(options['fileform'],story.getMetadata("numChapters"))
             
@@ -167,14 +191,46 @@ def do_download_for_worker(book,options):
              adapter.calibrebookmark,
              adapter.logfile) = get_update_data(book['epub_for_update'])
 
-            print("Do update - epub(%d) vs url(%d)" % (chaptercount, urlchaptercount))
-            print("write to %s"%outfile)
+            # dup handling from ffdl_plugin needed for anthology updates.
+            if options['collision'] == UPDATE:
+                if chaptercount == urlchaptercount:
+                    book['comment']="Already contains %d chapters.  Reuse as is."%chaptercount
+                    book['outfile'] = book['epub_for_update'] # for anthology merge ops.
+                    return book
+
+            # dup handling from ffdl_plugin needed for anthology updates.
+            if chaptercount > urlchaptercount:
+                raise NotGoingToDownload("Existing epub contains %d chapters, web site only has %d. Use Overwrite to force update." % (chaptercount,urlchaptercount),'dialog_error.png')
+
+            if not (options['collision'] == UPDATEALWAYS and chaptercount == urlchaptercount) \
+                    and adapter.getConfig("do_update_hook"):
+                chaptercount = adapter.hookForUpdates(chaptercount)
+
+            logger.info("Do update - epub(%d) vs url(%d)" % (chaptercount, urlchaptercount))
+            logger.info("write to %s"%outfile)
 
             writer.writeStory(outfilename=outfile, forceOverwrite=True)
             
             book['comment'] = 'Update %s completed, added %s chapters for %s total.'%\
                 (options['fileform'],(urlchaptercount-chaptercount),urlchaptercount)
         
+        if options['smarten_punctuation'] and options['fileform'] == "epub" \
+                and calibre_version >= (0, 9, 39):
+            # do smarten_punctuation from calibre's polish feature
+            from calibre.ebooks.oeb.polish.main import polish, ALL_OPTS
+            from calibre.utils.logging import Log
+            from collections import namedtuple
+        
+            data = {'smarten_punctuation':True}
+            opts = ALL_OPTS.copy()
+            opts.update(data)
+            O = namedtuple('Options', ' '.join(ALL_OPTS.iterkeys()))
+            opts = O(**opts)
+            
+            log = Log(level=Log.DEBUG)
+            # report = []
+            polish({outfile:outfile}, opts, log, logger.info) # report.append
+        
     except NotGoingToDownload as d:
         book['good']=False
         book['comment']=unicode(d)
@@ -185,8 +241,8 @@ def do_download_for_worker(book,options):
         book['comment']=unicode(e)
         book['icon']='dialog_error.png'
         book['status'] = 'Error'
-        print("Exception: %s:%s"%(book,unicode(e)))
+        logger.info("Exception: %s:%s"%(book,unicode(e)))
         traceback.print_exc()
-
+        
     #time.sleep(10)
     return book

calibre-plugin/prefs.py

@@ -0,0 +1,147 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2013, Jim Miller'
+__docformat__ = 'restructuredtext en'
+
+import copy
+
+from calibre.utils.config import JSONConfig
+from calibre.gui2.ui import get_gui
+
+from calibre_plugins.fanfictiondownloader_plugin.dialogs import OVERWRITE
+from calibre_plugins.fanfictiondownloader_plugin.common_utils import get_library_uuid
+PREFS_NAMESPACE = 'FanFictionDownLoaderPlugin'
+PREFS_KEY_SETTINGS = 'settings'
+
+# Set defaults used by all.  Library specific settings continue to
+# take from here.
+default_prefs = {}
+default_prefs['personal.ini'] = get_resources('plugin-example.ini')
+default_prefs['rejecturls'] = ''
+default_prefs['rejectreasons'] = '''Sucked
+Boring
+Dup from another site'''
+
+default_prefs['updatemeta'] = True
+default_prefs['updatecover'] = False
+default_prefs['updateepubcover'] = False
+default_prefs['keeptags'] = False
+default_prefs['suppressauthorsort'] = False
+default_prefs['suppresstitlesort'] = False
+default_prefs['mark'] = False
+default_prefs['showmarked'] = False
+default_prefs['urlsfromclip'] = True
+default_prefs['updatedefault'] = True
+default_prefs['fileform'] = 'epub'
+default_prefs['collision'] = OVERWRITE
+default_prefs['deleteotherforms'] = False
+default_prefs['adddialogstaysontop'] = False
+default_prefs['includeimages'] = False
+default_prefs['lookforurlinhtml'] = False
+default_prefs['checkforseriesurlid'] = True
+default_prefs['checkforurlchange'] = True
+default_prefs['injectseries'] = False
+default_prefs['smarten_punctuation'] = False
+
+default_prefs['send_lists'] = ''
+default_prefs['read_lists'] = ''
+default_prefs['addtolists'] = False
+default_prefs['addtoreadlists'] = False
+default_prefs['addtolistsonread'] = False
+
+default_prefs['gcnewonly'] = False
+default_prefs['gc_site_settings'] = {}
+default_prefs['allow_gc_from_ini'] = True
+
+default_prefs['countpagesstats'] = []
+
+default_prefs['errorcol'] = ''
+default_prefs['custom_cols'] = {}
+default_prefs['custom_cols_newonly'] = {}
+default_prefs['allow_custcol_from_ini'] = True
+
+default_prefs['std_cols_newonly'] = {}
+
+# This is where all preferences for this plugin *were* stored
+# Remember that this name (i.e. plugins/fanfictiondownloader_plugin) is also
+# in a global namespace, so make it as unique as possible.
+# You should always prefix your config file name with plugins/,
+# so as to ensure you dont accidentally clobber a calibre config file
+old_prefs = JSONConfig('plugins/fanfictiondownloader_plugin')
+
+def set_library_config(library_config,db):
+    db.prefs.set_namespaced(PREFS_NAMESPACE,
+                            PREFS_KEY_SETTINGS,
+                            library_config)
+
+def get_library_config(db):
+    library_id = get_library_uuid(db)
+    library_config = None
+    # Check whether this is a configuration needing to be migrated
+    # from json into database.  If so: get it, set it, rename it in json.
+    if library_id in old_prefs:
+        #print("get prefs from old_prefs")
+        library_config = old_prefs[library_id]
+        set_library_config(library_config,db)
+        old_prefs["migrated to library db %s"%library_id] = old_prefs[library_id]
+        del old_prefs[library_id]
+
+    if library_config is None:
+        #print("get prefs from db")
+        library_config = db.prefs.get_namespaced(PREFS_NAMESPACE, PREFS_KEY_SETTINGS,
+                                                 copy.deepcopy(default_prefs))
+    return library_config
+
+# fake out so I don't have to change the prefs calls anywhere.  The
+# Java programmer in me is offended by op-overloading, but it's very
+# tidy.
+class PrefsFacade():
+    def _get_db(self):
+        if self.passed_db:
+            return self.passed_db
+        else:
+            # In the GUI plugin we want current db so we detect when
+            # it's changed.  CLI plugin calls need to pass db in.
+            return get_gui().current_db
+    
+    def __init__(self,passed_db=None):
+        self.default_prefs = default_prefs
+        self.libraryid = None
+        self.current_prefs = None
+        self.passed_db=passed_db
+        
+    def _get_prefs(self):
+        libraryid = get_library_uuid(self._get_db())
+        if self.current_prefs == None or self.libraryid != libraryid:
+            #print("self.current_prefs == None(%s) or self.libraryid != libraryid(%s)"%(self.current_prefs == None,self.libraryid != libraryid))
+            self.libraryid = libraryid
+            self.current_prefs = get_library_config(self._get_db())
+        return self.current_prefs
+        
+    def __getitem__(self,k):            
+        prefs = self._get_prefs()
+        if k not in prefs:
+            # pulls from default_prefs.defaults automatically if not set
+            # in default_prefs
+            return self.default_prefs[k]
+        return prefs[k]
+
+    def __setitem__(self,k,v):
+        prefs = self._get_prefs()
+        prefs[k]=v
+        # self._save_prefs(prefs)
+
+    def __delitem__(self,k):
+        prefs = self._get_prefs()
+        if k in prefs:
+            del prefs[k]
+
+    def save_to_db(self):
+        set_library_config(self._get_prefs(),self._get_db())
+        
+prefs = PrefsFacade()
+

defaults.ini

@@ -1,4 +1,4 @@
-# Copyright 2012 Fanficdownloader team
+# Copyright 2013 Fanficdownloader team
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -16,7 +16,22 @@
 [defaults]
 
 ## [defaults] section applies to all formats and sites but may be
-## overridden at several levels
+## overridden at several levels.  Example:
+
+## [defaults]
+## titlepage_entries: category,genre, status
+## [www.whofic.com]
+## # overrides defaults.
+## titlepage_entries: category,genre, status,dateUpdated,rating
+## [epub]
+## # overrides defaults & site section
+## titlepage_entries: category,genre, status,datePublished,dateUpdated,dateCreated
+## [www.whofic.com:epub]
+## # overrides defaults, site section & format section
+## titlepage_entries: category,genre, status,datePublished
+## [overrides]
+## # overrides all other sections
+## titlepage_entries: category
 
 ## Some sites also require the user to confirm they are adult for
 ## adult content.  Uncomment by removing '#' in front of is_adult.
@@ -44,6 +59,9 @@ language_label:Language
 characters_label:Characters
 ships_label:Relationships
 series_label:Series
+seriesUrl_label:Series URL
+## seriesHTML is series as a link to seriesUrl.
+seriesHTML_label:Series
 ## Completed/In-Progress
 status_label:Status
 ## Dates story first published, last updated, and downloaded(last with time).
@@ -81,7 +99,7 @@ dateUpdated_format:%%Y-%%m-%%d
 ## You can include extra text or HTML that will be included as-is in
 ## the title page. Eg: titlepage_entries: ...,<br />,summary,<br />,...
 ## All current formats already include title and author.
-titlepage_entries: series,category,genre,language,characters,ships,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,description
+titlepage_entries: seriesHTML,category,genre,language,characters,ships,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,description
 
 ## Try to collect series name and number of this story in series.
 ## Some sites (ab)use 'series' for reading lists and personal
@@ -130,6 +148,9 @@ zip_filename: ${title}-${siteabbrev}_${storyId}${formatext}.zip
 ## zip_filename.
 allow_unsafe_filename: false
 
+## The regex pattern of 'unsafe' filename chars for above.
+#output_filename_safepattern:[^a-zA-Z0-9_\. \[\]\(\)&'-]+
+
 ## entries to make epub subjects and calibre tags
 ## lastupdate creates two tags: "Last Update Year/Month: %Y/%m" and "Last Update: %Y/%m/%d"
 include_subject_tags: extratags, genre, category, characters, ships, lastupdate, status
@@ -146,6 +167,17 @@ extratags: FanFiction
 #extraships:
 #extrawarnings:
 
+## Add this to genre if there's more than one category.
+#add_genre_when_multi_category: Crossover
+
+## default_value_(entry) can be used to set the value for a metadata
+## entry when no value has been found on the site.  For example, some
+## sites doesn't have a status metadatum.  If uncommented, this will
+## use 'Unknown' for status when no status is found.
+#default_value_status:Unknown
+## Can also be used for other metadata values
+#default_value_category:FanFiction
+
 ## number of seconds to sleep between calls to the story site.  May by
 ## useful if pulling large numbers of stories or if the site is slow.
 #slow_down_sleep_time:0.5
@@ -166,6 +198,13 @@ extratags: FanFiction
 ## *Five* part lines.  Effect only when trailing conditional key=>regexp matches
 ## metakey[,metakey]=>pattern=>replacement[&&conditionalkey=>regexp]
 ## Note that if metakey == conditionalkey the conditional is ignored.
+## You can use \s in the replacement to add explicit spaces.  (The config parser
+## tends to discard trailing spaces.)
+## replace_metadata <entry>_LIST options: FFDL replace_metadata lines
+## operate on individual list items for list entries. But if you
+## want to do a replacement on the joined string for the whole list,
+## you can by using <entry>_LIST. Example, if you added
+## calibre_author: calibre_author_LIST=>^(.{,100}).*$=>\1
 #replace_metadata:
 # genre,category=>Sci-Fi=>SF
 # Puella Magi Madoka Magica.* => Madoka
@@ -186,6 +225,10 @@ extratags: FanFiction
 ## summary.
 keep_summary_html:true
 
+## If set true, any style attributes on tags in the story HTML will be
+## kept.  Useful for keeping extra colors & formatting from original.
+#keep_style_attr: false
+
 ## Don't like the numbers at the start of chapter titles on some
 ## sites?  You can use strip_chapter_numbers to strip them off.  Just
 ## want to make them all look the same?  Strip them off, then add them
@@ -224,6 +267,26 @@ chapter_title_strip_pattern:^[0-9]+[\.: -]+
 ## "The Beginning" => "1. The Beginning" 
 chapter_title_add_pattern:${index}. ${title}
 
+## Reorder ships so b/a and c/b/a become a/b and a/b/c. Only separates
+## on '/', so use replace_metadata to change separator first if
+## needed. Something like: ships=>[ ]*(/|&amp;|&)[ ]*=>/ You can use
+## ships_LIST to change the / back to something else if you want.
+sort_ships:false
+
+## join_string_<entry> options -- FFDL list entries are comma
+## separated by default. You can use this to change that. For example,
+## if you want authors separated with ' & ' instead, use
+## join_string_calibre_author:\s&\s. (\s == space)
+#join_string_author:,\s
+
+## keep_in_order_<entry> options: FFDL sorts list entries by default
+## (except for author/authorUrl/authorId). But if you want to use an
+## extra entry derived from author, it ends up sorted. For example, if
+## you added calibre_author: keep_in_order_calibre_author:true
+#keep_in_order_author:true
+
+## User-agent
+user_agent:FFDL/1.7
 
 ## Each output format has a section that overrides [defaults]
 [html]
@@ -270,7 +333,7 @@ output_css:
 
 [txt]
 ## Add URLs since there aren't links.
-titlepage_entries: series,category,genre,language,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,storyUrl, authorUrl, description
+titlepage_entries: series,seriesUrl,category,genre,language,characters,ships,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,storyUrl, authorUrl, description
 
 ## Width to word wrap text output.  0 indicates no wrapping.
 wrap_width: 78
@@ -304,13 +367,13 @@ include_logpage: false
 ## if in the list.  You can include extra text or HTML that will be
 ## included as-is in each log entry. Eg: logpage_entries: ...,<br />,
 ## summary,<br />,...
-logpage_entries: dateCreated,datePublished,dateUpdated,numChapters,numWords,status,title,author,description,category,genre,rating,warnings
+logpage_entries: dateCreated,datePublished,dateUpdated,numChapters,numWords,status,series,title,author,description,category,genre,rating,warnings
 
 ## epub->mobi conversions typically don't like tables.
 titlepage_use_table: false
 
 ## When using tables, make these span both columns.
-wide_titlepage_entries: description, storyUrl, author URL
+wide_titlepage_entries: description, storyUrl, authorUrl, seriesUrl
 
 ## output background color--only used by html and epub (and ignored in
 ## epub by many readers).  Included below in output_css--will be
@@ -320,12 +383,13 @@ background_color: ffffff
 ## Allow customization of CSS.  Make sure to keep at least one space
 ## at the start of each line and to escape % to %%.  Also need
 ## background_color to be in the same section, if included in CSS.
-## 'adobe-text-layout: optimizeSpeed;' prevents hyphenation on newer Nooks
+## 'adobe-hyphenate: none;' prevents hyphenation on newer Nooks
+## STR(wG) (1.2.1+ for sure)
 output_css: 
  body { background-color: #%(background_color)s;
         text-align: justify;
         margin: 2%%;
-	adobe-text-layout: optimizeSpeed; }
+	adobe-hyphenate: none; }
  pre { font-size: x-small; }
  sml { font-size: small; }
  h1 { text-align: center; }
@@ -368,7 +432,7 @@ output_css:
 ## It can be either a 'file:' or 'http:' url.
 ## Note that if you enable make_firstimage_cover in [epub], but want
 ## to use default_cover_image for a specific site, use the site:format
-## section, for example: [www.ficwad.com:epub]
+## section, for example: [ficwad.com:epub]
 ## default_cover_image is a python string Template string with
 ## ${title}, ${author} etc, same as titlepage_entries. Unless
 ## allow_unsafe_filename is true, invalid filename chars will be
@@ -411,8 +475,8 @@ nook_img_fix:true
 ## mobi TOC cannot be turned off right now.
 #include_tocpage: true
 
-## Each site has a section that overrides [defaults] *and* the format
-## sections test1.com specifically is not a real story site.  Instead,
+## Each site has a section that overrides [defaults].
+## test1.com specifically is not a real story site.  Instead,
 ## it is a fake site for testing configuration and output.  It uses
 ## URLs like: http://test1.com?sid=12345
 [test1.com]
@@ -469,7 +533,8 @@ extratags: FanFiction,Testing,HTML
 #is_adult:true
 
 ## AO3 adapter defines a few extra metadata entries.
-extra_valid_entries:fandoms,freeformtags,freefromtags,ao3categories,comments,kudos,hits,bookmarks,collections
+## If there's ever more than 4 series, add series04,series04Url etc.
+extra_valid_entries:fandoms,freeformtags,freefromtags,ao3categories,comments,kudos,hits,bookmarks,collections,series00,series01,series02,series03,series00Url,series01Url,series02Url,series03Url
 fandoms_label:Fandoms
 freeformtags_label:Freeform Tags
 freefromtags_label:Freeform Tags
@@ -485,7 +550,7 @@ bookmarks_label:Bookmarks
 include_in_freefromtags:freeformtags
 
 ## adds to titlepage_entries instead of replacing it.
-#extra_titlepage_entries: fandoms,freeformtags,ao3categories,comments,kudos,hits,bookmarks
+#extra_titlepage_entries: fandoms,freeformtags,ao3categories,comments,kudos,hits,bookmarks,series00,series01,series02,series03,series00Url,series01Url,series02Url,series03Url
 
 ## adds to include_subject_tags instead of replacing it.
 #extra_subject_tags:fandoms,freeformtags,ao3categories
@@ -503,6 +568,15 @@ extraships:Severus Snape/Hermione Granger
 #username:YourName
 #password:yourpassword
 
+[asr3.slashzone.org]
+## Site dedicated to these categories/characters/ships
+extracategories:The Sentinel
+
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
 [bloodties-fans.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:Blood Ties
@@ -519,6 +593,15 @@ extracategories:Blood Ties
 ## personal.ini, not defaults.ini.
 #is_adult:true
 
+[buffynfaith.net]
+## Site dedicated to these categories/characters/ships
+extracategories:Buffy: The Vampire Slayer
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
 [castlefans.org]
 ## Site dedicated to these categories/characters/ships
 extracategories:Castle
@@ -558,6 +641,11 @@ extraships:Spike/Buffy
 #username:YourName
 #password:yourpassword
 
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
 [dramione.org]
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter
@@ -589,7 +677,7 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 ## composite metadata entries.  dramione.org, for example, adds
 ## 'cliches' and then defines as the composite of hermiones,dracos in
 ## include_in_cliches.
-extra_valid_entries:themes,hermiones,dracos,timeline,cliches
+extra_valid_entries:themes,hermiones,dracos,timeline,cliches,read,reviews
 include_in_cliches:hermiones,dracos
 
 ## For another example, you could, by uncommenting this line, include
@@ -613,6 +701,10 @@ cliches_label:Character Cliches
 #extra_logpage_entries: themes,timeline,cliches
 #extra_subject_tags: themes,timeline,cliches
 
+[efiction.esteliel.de]
+## Site dedicated to these categories/characters/ships
+extracategories:Lord of the Rings
+
 [erosnsappho.sycophanthex.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter
@@ -630,6 +722,11 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter
 
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -637,11 +734,22 @@ extracategories:Harry Potter
 #username:YourName
 #password:yourpassword
 
-## Some sites also require the user to confirm they are adult for
-## adult content.  In commandline version, this should go in your
-## personal.ini, not defaults.ini.
+[fanfic.potterheadsanonymous.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
 [fanfiction.portkey.org]
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter
@@ -660,6 +768,23 @@ extraships:Harry Potter/Hermione Granger
 #username:YourName
 #password:yourpassword
 
+[fictionpad.com]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+extra_valid_entries:followers,comments,views,likes,dislikes
+#extra_titlepage_entries:followers,comments,views,likes,dislikes
+
+followers_label:Followers
+comments_label:Comments
+views_label:Views
+likes_label:Likes
+dislikes_label:Dislikes
+
 [finestories.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -685,14 +810,35 @@ extracharacters:Hermione Granger
 ## personal.ini, not defaults.ini.
 #is_adult:true
 
+## Extra metadata that this adapter knows about.  See [dramione.org]
+## for examples of how to use them.
+extra_valid_entries:read,reviews
+
 [hlfiction.net]
 ## Site dedicated to these categories/characters/ships
 extracategories:Highlander
 
+[imagine.e-fic.com]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
 [indeath.net]
 ## Site dedicated to these categories/characters/ships
 extracategories:In Death
 
+## some sites include images that we don't ever want becoming the
+## cover image.  This lets you exclude them.
+cover_exclusion_regexp:/public/style_emoticons/.*
+
 [ksarchive.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:Star Trek
@@ -732,6 +878,8 @@ extracategories:West Wing
 ## Site dedicated to these categories/characters/ships
 extracategories:NCIS
 
+[netraptor.org]
+
 [nfacommunity.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:NCIS
@@ -774,6 +922,22 @@ extracategories:One Direction
 ## personal.ini, not defaults.ini.
 #is_adult:true
 
+[pommedesang.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Anita Blake Vampire Hunter 
+
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
 [ponyfictionarchive.net]
 ## Site dedicated to these categories/characters/ships
 extracategories:My Little Pony: Friendship is Magic
@@ -798,6 +962,22 @@ extracategories:Supernatural
 extracharacters:Sam,Dean
 extraships:Sam/Dean
 
+[scarhead.net]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
 [sg1-heliopolis.com]
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
@@ -872,6 +1052,15 @@ extracategories:Harry Potter
 ## personal.ini, not defaults.ini.
 #is_adult:true
 
+[tokra.fandomnet.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Stargate: SG-1
+
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
 [www.adastrafanfic.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:Star Trek
@@ -926,6 +1115,21 @@ extracategories:InuYasha
 extracharacters:Sesshoumaru,Kagome
 extraships:Sesshoumaru/Kagome
 
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+[www.dotmoon.net]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
 [www.efpfanfic.net]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -942,6 +1146,7 @@ context_label:Context
 type_label:Type of Couple
 
 [www.fanfiction.net]
+user_agent:
 ## fanfiction.net's 'cover' images are really just tiny thumbnails.
 ## Change this to false to use them anyway.
 never_make_cover: true
@@ -972,7 +1177,12 @@ extracategories:Harry Potter
 ## fictionally.org storyIds are not unique.  Combine with authorId.
 output_filename: ${title}-${siteabbrev}_${authorId}_${storyId}${formatext}
 
+## fictionalley.org doesn't have a status metadatum.  If uncommented,
+## this will be used for status.
+#default_value_status:Unknown
+
 [www.fictionpress.com]
+user_agent:
 ## Clear FanFiction from defaults, fictionpress.com is original fiction.
 extratags:
 
@@ -980,7 +1190,7 @@ extratags:
 ## for examples of how to use them.
 extra_valid_entries:reviews,favs,follows
 
-[www.ficwad.com]
+[ficwad.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -994,12 +1204,13 @@ extracategories:My Little Pony: Friendship is Magic
 
 ## Extra metadata that this adapter knows about.  See [dramione.org]
 ## for examples of how to use them.
-extra_valid_entries:likes,dislikes,views,total_views,short_description
+extra_valid_entries:likes,dislikes,views,total_views,short_description,groups
 likes_label:Likes
 dislikes_label:Dislikes
 views_label:Highest Single Chapter Views
 total_views_label:Total Views
 short_description_label:Short Summary
+groups_label:Groups
 
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
@@ -1011,6 +1222,13 @@ short_description_label:Short Summary
 ## when a password is required rather than prompting every time.
 #fail_on_password: false
 
+## fimfiction.net stories allow chapters to be added out of order.  So
+## the newest chapter may not be the last one.  FFDL update doesn't
+## like that.  If do_update_hook is uncommented and set true, the
+## adapter will discard all existing chapters from the newest one on
+## when updating to enforce accurate chapters.
+#do_update_hook:false
+
 [www.harrypotterfanfiction.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter
@@ -1020,6 +1238,10 @@ extracategories:Harry Potter
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+[www.henneth-annun.net]
+## Site dedicated to these categories/characters/ships
+extracategories:The Hobbit
+
 [www.hpfandom.net]
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter
@@ -1083,6 +1305,11 @@ cover_exclusion_regexp:/stories/999/images/.*?_trophy.png
 ## Site dedicated to these categories/characters/ships
 extracategories:NCIS
 
+[www.nickandgreg.net]
+## Site dedicated to these categories/characters/ships
+extracategories:CSI
+extraships:Nick Stokes/Greg Sanders
+
 [www.phoenixsong.net]
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter
@@ -1095,6 +1322,12 @@ extraships:Harry Potter/Ginny Weasley
 #username:YourName
 #password:yourpassword
 
+## phoenixsong.net, oddly, can have high rated chapters (login
+## required) in the middle of a lower rated story.  Use this to force
+## FFDL to always login to phoenixsong.net so those stories download
+## correctly.
+#force_login:true
+
 [www.potionsandsnitches.net]
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter
@@ -1107,6 +1340,15 @@ extracategories:Harry Potter
 ## Site dedicated to these categories/characters/ships
 extracategories:Prison Break
 
+[www.psychfic.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Psych
+
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
 [www.qaf-fic.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:Queer as Folk
@@ -1116,6 +1358,16 @@ extracategories:Queer as Folk
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+[www.restrictedsection.org]
+extracategories:Harry Potter
+extragenres:Erotica
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
 [www.scarvesandcoffee.net]
 ## Site dedicated to these categories/characters/ships
 extracategories:Glee
@@ -1126,6 +1378,17 @@ extracharacters:Kurt Hummel,Blaine Anderson
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+[www.simplyundeniable.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
 [www.sinful-desire.org]
 ## Site dedicated to these categories/characters/ships
 extracategories:Supernatural
@@ -1145,17 +1408,22 @@ extraships:Harry Potter/Ginny Weasley
 # www.squidge.org/peja calls it Fandom <shrug>
 category_label:Fandom
 # Remove numWords -- www.squidge.org/peja word counts are inaccurate
-titlepage_entries: series,category,genre,language,characters,ships,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,site,description
+titlepage_entries: seriesHTML,category,genre,language,characters,ships,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,site,description
 
 [www.squidge.org/peja:txt]
 ## Add URLs since there aren't links.
 # Remove numWords -- www.squidge.org/peja word counts are inaccurate
-titlepage_entries: series,category,genre,language,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,site,storyUrl, authorUrl, description
+titlepage_entries: series,seriesUrl,category,genre,language,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,site,storyUrl, authorUrl, description
 
 [www.storiesofarda.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:Lord of the Rings
 
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
 [www.thepetulantpoetess.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -1272,3 +1540,48 @@ extracategories:Stargate: Atlantis
 ## extratags params in all other sections.  Only commandline options
 ## beat overrides.
 #extratags:fanficdownloader
+
+
+[teststory:defaults]
+valid_entries:title,author_list,authorId_list,authorUrl_list,storyUrl,
+ datePublished,dateUpdated,numWords,status,language,series,seriesUrl,
+ rating,category_list,genre_list,warnings_list,characters_list,ships_list,
+ description,site,extratags
+
+# {{storyId}} is a special case--it's the only one that works.
+title:Test Story Title {{storyId}}
+author_list:Test Author aa
+authorId_list:1
+authorUrl_list:http://test1.com?authid=1
+storyUrl:http://test1.com?sid={{storyId}}
+datePublished:1975-03-15
+dateUpdated:1975-04-15
+numWords:123,456
+status:In-Progress
+language:English
+
+chaptertitles:Prologue
+
+## Add additional sections with different numbers to get different
+## parameters for different story urls.
+## test1.com?sid=1000
+[teststory:1000]
+# note the leading commas when doing add_to_ with valid_entries and *_list
+add_to_valid_entries:,favs
+title:Testing New Feature {{storyId}}
+author_list:Bob Smith
+authorId_list:45
+authorUrl_list:http://test1.com?authid=45
+datePublished:2013-03-15
+dateUpdated:2013-04-15
+numWords:1456
+favs:56
+series:The Great Test [4]
+seriesUrl:http://test1.com?seriesid=1
+rating:Tweenie
+category_list:Harry Potter,Furbie,Crossover,Puella Magi Madoka Magica/魔法少女まどか★マギカ,Magical Girl Lyrical Nanoha
+genre_list:Fantasy,Comedy,Sci-Fi,Noir
+warnings_list:Swearing,Violence
+characters_list:Bob Smith,George Johnson,Fred Smythe
+
+chaptertitles:Prologue,Chapter 1\, Xenos on Cinnabar,Chapter 2\, Sinmay on Kintikin,3. Chapter 3

downloader.py

@@ -32,10 +32,25 @@ if sys.version_info >= (2, 7):
     loghandler.setFormatter(logging.Formatter("(=====)(levelname)s:%(message)s"))
     rootlogger.addHandler(loghandler)
 
-from fanficdownloader import adapters,writers,exceptions
-from fanficdownloader.configurable import Configuration
-from fanficdownloader.epubutils import get_dcsource_chaptercount, get_update_data
-from fanficdownloader.geturls import get_urls_from_page
+try:
+    from calibre.constants import numeric_version as calibre_version
+    is_calibre = True
+except:
+    is_calibre = False
+
+# using try/except directly was masking errors during development.
+if is_calibre:
+    # running under calibre
+    from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapters,writers,exceptions
+    from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.configurable import Configuration
+    from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.epubutils import get_dcsource_chaptercount, get_update_data
+    from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.geturls import get_urls_from_page
+else:
+    from fanficdownloader import adapters,writers,exceptions
+    from fanficdownloader.configurable import Configuration
+    from fanficdownloader.epubutils import get_dcsource_chaptercount, get_update_data
+    from fanficdownloader.geturls import get_urls_from_page
+    
 
 if sys.version_info < (2, 5):
     print "This program requires Python 2.5 or newer."
@@ -48,15 +63,23 @@ def writeStory(config,adapter,writeformat,metaonly=False,outstream=None):
     del writer
     return output_filename
 
-def main():
+def main(argv,
+         parser=None,
+         passed_defaultsini=None,
+         passed_personalini=None):
    # read in args, anything starting with -- will be treated as --<varible>=<value>
-   usage = "usage: %prog [options] storyurl"
-   parser = OptionParser(usage)
+   if not parser:
+       parser = OptionParser("usage: %prog [options] storyurl")
    parser.add_option("-f", "--format", dest="format", default="epub",
-                     help="write story as FORMAT, epub(default), text or html", metavar="FORMAT")
+                     help="write story as FORMAT, epub(default), mobi, text or html", metavar="FORMAT")
+
+   if passed_defaultsini:
+       config_help="read config from specified file(s) in addition to calibre plugin personal.ini, ~/.fanficdownloader/personal.ini, and ./personal.ini"
+   else:
+       config_help="read config from specified file(s) in addition to ~/.fanficdownloader/defaults.ini, ~/.fanficdownloader/personal.ini, ./defaults.ini, and ./personal.ini"
    parser.add_option("-c", "--config",
                      action="append", dest="configfile", default=None,
-                     help="read config from specified file(s) in addition to ~/.fanficdownloader/defaults.ini, ~/.fanficdownloader/personal.ini, ./defaults.ini, ./personal.ini", metavar="CONFIG")
+                     help=config_help, metavar="CONFIG")
    parser.add_option("-b", "--begin", dest="begin", default=None,
                      help="Begin with Chapter START", metavar="START")
    parser.add_option("-e", "--end", dest="end", default=None,
@@ -69,50 +92,86 @@ def main():
                      help="Retrieve metadata and stop.  Or, if --update-epub, update metadata title page only.",)
    parser.add_option("-u", "--update-epub",
                      action="store_true", dest="update",
-                     help="Update an existing epub with new chapter, give epub filename instead of storyurl.",)
+                     help="Update an existing epub with new chapters, give epub filename instead of storyurl.",)
    parser.add_option("--update-cover",
                      action="store_true", dest="updatecover",
                      help="Update cover in an existing epub, otherwise existing cover (if any) is used on update.  Only valid with --update-epub.",)
    parser.add_option("--force",
                      action="store_true", dest="force",
-                     help="Force overwrite or update of an existing epub, download and overwrite all chapters.",)
+                     help="Force overwrite of an existing epub, download and overwrite all chapters.",)
    parser.add_option("-l", "--list",
                      action="store_true", dest="list",
                      help="Get list of valid story URLs from page given.",)
+   parser.add_option("-n", "--normalize-list",
+                     action="store_true", dest="normalize",default=False,
+                     help="Get list of valid story URLs from page given, but normalized to standard forms.",)
+   parser.add_option("-s", "--sites-list",
+                     action="store_true", dest="siteslist",default=False,
+                     help="Get list of valid story URLs examples.",)
    parser.add_option("-d", "--debug",
                      action="store_true", dest="debug",
                      help="Show debug output while downloading.",)
    
-   (options, args) = parser.parse_args()
+   (options, args) = parser.parse_args(argv)
 
    if not options.debug:
        logger = logging.getLogger("fanficdownloader")
        logger.setLevel(logging.INFO)
    
-   if len(args) != 1:
+   if not options.siteslist and len(args) != 1:
        parser.error("incorrect number of arguments")
 
+   if options.siteslist:
+       for (site,examples) in adapters.getSiteExamples():
+           print("\n====%s====\n\nExample URLs:"%site)
+           for u in examples:
+               print("  * %s"%u)
+       return
+
    if options.update and options.format != 'epub':
        parser.error("-u/--update-epub only works with epub")
 
    ## Attempt to update an existing epub.
+   chaptercount = None
+   output_filename = None
    if options.update:
-       (url,chaptercount) = get_dcsource_chaptercount(args[0])
-       print "Updating %s, URL: %s" % (args[0],url)
-       output_filename = args[0]
+       try:
+           (url,chaptercount) = get_dcsource_chaptercount(args[0])
+           if not url:
+               print "No story URL found in epub to update."
+               return
+           print "Updating %s, URL: %s" % (args[0],url)
+           output_filename = args[0]
+       except:
+           # if there's an error reading the update file, maybe it's a URL?
+           # we'll look for an existing outputfile down below.
+           url = args[0]
    else:
        url = args[0]
-           
-   configuration = Configuration(adapters.getConfigSectionFor(url),options.format)
+
+   try:
+       configuration = Configuration(adapters.getConfigSectionFor(url),options.format)
+   except exceptions.UnknownSite, e:
+       if options.list or options.normalize:
+           # list for page doesn't have to be a supported site.
+           configuration = Configuration("test1.com",options.format)
+       else:
+           raise e
 
    conflist = []
    homepath = join(expanduser("~"),".fanficdownloader")
+
+   if passed_defaultsini:
+       configuration.readfp(passed_defaultsini)
    
    if isfile(join(homepath,"defaults.ini")):
        conflist.append(join(homepath,"defaults.ini"))
    if isfile("defaults.ini"):
        conflist.append("defaults.ini")
        
+   if passed_personalini:
+       configuration.readfp(passed_personalini)
+   
    if isfile(join(homepath,"personal.ini")):
        conflist.append(join(homepath,"personal.ini"))
    if isfile("personal.ini"):
@@ -132,8 +191,8 @@ def main():
    if options.force:
        configuration.set("overrides","always_overwrite","true")
 
-   if options.update:
-       configuration.set("overrides","output_filename",args[0])
+   if options.update and chaptercount:
+       configuration.set("overrides","output_filename",output_filename)
        
    if options.update and not options.updatecover:
        configuration.set("overrides","never_make_cover","true")
@@ -148,18 +207,26 @@ def main():
            (var,val) = opt.split('=')
            configuration.set("overrides",var,val)
 
-   if options.list:
-       retlist = get_urls_from_page(args[0], configuration)
-       print "\n".join(retlist)
-               
+   if options.list or options.normalize:
+       retlist = get_urls_from_page(args[0], configuration, normalize=options.normalize)
+       print "\n".join(retlist)               
        return
 
    try:
-
        adapter = adapters.getAdapter(configuration,url)
-
        adapter.setChaptersRange(options.begin,options.end)
-           
+
+       # check for updating from URL (vs from file)
+       if options.update and not chaptercount:
+           try:
+               writer = writers.getWriter("epub",configuration,adapter)
+               output_filename=writer.getOutputFileName()
+               (noturl,chaptercount) = get_dcsource_chaptercount(output_filename)
+               print "Updating %s, URL: %s" % (output_filename,url)
+           except:
+               options.update = False
+               pass
+       
        ## Check for include_images and absence of PIL, give warning.
        if adapter.getConfig('include_images'):
            try:
@@ -173,7 +240,6 @@ def main():
                    print "You have include_images enabled, but Python Image Library(PIL) isn't found.\nImages will be included full size in original format.\nContinue? (y/n)?"
                    if not sys.stdin.readline().strip().lower().startswith('y'):
                        return
-               
 
        ## three tries, that's enough if both user/pass & is_adult needed,
        ## or a couple tries of one or the other
@@ -196,13 +262,14 @@ def main():
 
        if options.update and not options.force:
            urlchaptercount = int(adapter.getStoryMetadataOnly().getMetadata('numChapters'))
-           
+
            if chaptercount == urlchaptercount and not options.metaonly:
-               print "%s already contains %d chapters." % (args[0],chaptercount)
+               print "%s already contains %d chapters." % (output_filename,chaptercount)
            elif chaptercount > urlchaptercount:
-               print "%s contains %d chapters, more than source: %d." % (args[0],chaptercount,urlchaptercount)
+               print "%s contains %d chapters, more than source: %d." % (output_filename,chaptercount,urlchaptercount)
+           elif chaptercount == 0:
+               print "%s doesn't contain any recognizable chapters, probably from a different source.  Not updating." % (output_filename)
            else:
-               print "Do update - epub(%d) vs url(%d)" % (chaptercount, urlchaptercount)
                if not options.metaonly:
 
                    # update now handled by pre-populating the old
@@ -214,7 +281,13 @@ def main():
                     adapter.oldimgs,
                     adapter.oldcover,
                     adapter.calibrebookmark,
-                    adapter.logfile) = get_update_data(args[0])
+                    adapter.logfile) = get_update_data(output_filename)
+
+                   print "Do update - epub(%d) vs url(%d)" % (chaptercount, urlchaptercount)
+
+                   if not (options.update and chaptercount == urlchaptercount) \
+                           and adapter.getConfig("do_update_hook"):
+                       chaptercount = adapter.hookForUpdates(chaptercount)
 
                    writeStory(configuration,adapter,"epub")
                    
@@ -243,5 +316,5 @@ def main():
 if __name__ == "__main__":
     #import time
     #start = time.time()
-    main()
+    main(sys.argv[1:])
     #print("Total time seconds:%f"%(time.time()-start))

fanficdownloader/__init__.py

@@ -4,13 +4,16 @@ try:
     # just a way to switch between web service and CLI/PI
     import google.appengine.api 
 except:
-    import sys
-    if sys.version_info >= (2, 7):
-        import logging
-        logger = logging.getLogger(__name__)
-        loghandler=logging.StreamHandler()
-        loghandler.setFormatter(logging.Formatter("FFDL:%(levelname)s:%(filename)s(%(lineno)d):%(message)s"))
-        logger.addHandler(loghandler)
-        loghandler.setLevel(logging.DEBUG)
-        logger.setLevel(logging.DEBUG)
+    try: # just a way to switch between CLI and PI
+        import calibre.constants
+    except:
+        import sys
+        if sys.version_info >= (2, 7):
+            import logging
+            logger = logging.getLogger(__name__)
+            loghandler=logging.StreamHandler()
+            loghandler.setFormatter(logging.Formatter("FFDL:%(levelname)s:%(filename)s(%(lineno)d):%(message)s"))
+            logger.addHandler(loghandler)
+            loghandler.setLevel(logging.DEBUG)
+            logger.setLevel(logging.DEBUG)
 

fanficdownloader/adapters/__init__.py

@@ -71,7 +71,6 @@ import adapter_thehexfilesnet
 import adapter_dokugacom
 import adapter_iketernalnet
 import adapter_onedirectionfanfictioncom
-import adapter_prisonbreakficnet
 import adapter_storiesofardacom
 import adapter_samdeanarchivenu
 import adapter_destinysgatewaycom
@@ -88,7 +87,6 @@ import adapter_pretendercentrecom
 import adapter_darksolaceorg
 import adapter_finestoriescom
 import adapter_hpfanficarchivecom
-import adapter_svufictioncom
 import adapter_twilightarchivescom
 import adapter_wizardtalesnet
 import adapter_nhamagicalworldsus
@@ -103,11 +101,25 @@ import adapter_merlinficdtwinscouk
 import adapter_thehookupzonenet
 import adapter_bloodtiesfancom
 import adapter_indeathnet
-import adapter_jlaunlimitedcom
 import adapter_qafficcom
 import adapter_efpfanficnet
-#import adapter_buffynfaithnet
 import adapter_potterficscom
+import adapter_efictionestelielde
+import adapter_dotmoonnet
+import adapter_pommedesangcom
+import adapter_restrictedsectionorg
+import adapter_imagineeficcom
+import adapter_buffynfaithnet
+import adapter_psychficcom
+import adapter_hennethannunnet
+import adapter_tokrafandomnetcom
+import adapter_netraptororg
+import adapter_asr3slashzoneorg
+import adapter_nickandgregnet
+import adapter_potterheadsanonymouscom
+import adapter_simplyundeniablecom
+import adapter_scarheadnet
+import adapter_fictionpadcom
 
 ## This bit of complexity allows adapters to be added by just adding
 ## importing.  It eliminates the long if/else clauses we used to need
@@ -115,6 +127,7 @@ import adapter_potterficscom
     
 ## List of registered site adapters.
 __class_list = []
+__domain_map = {}
 
 def imports():
     for name, val in globals().items():
@@ -124,7 +137,10 @@ def imports():
 for x in imports():
     if "fanficdownloader.adapters.adapter_" in x:
         #print x
-        __class_list.append(sys.modules[x].getClass())
+        cls = sys.modules[x].getClass()
+        __class_list.append(cls)
+        for site in cls.getAcceptDomains():
+            __domain_map[site]=cls
 
 def getNormalStoryURL(url):
     if not getNormalStoryURL.__dummyconfig:
@@ -142,12 +158,14 @@ def getNormalStoryURL(url):
 # kludgey function static/singleton
 getNormalStoryURL.__dummyconfig = None
 
-def getAdapter(config,url):
+def getAdapter(config,url,anyurl=False):
 
-    logger.debug("trying url:"+url)
+    #logger.debug("trying url:"+url)
     (cls,fixedurl) = getClassFor(url)
-    logger.debug("fixedurl:"+fixedurl)
+    #logger.debug("fixedurl:"+fixedurl)
     if cls:
+        if anyurl:
+            fixedurl = cls.getSiteExampleURLs().split()[0]
         adapter = cls(config,fixedurl) # raises InvalidStoryURL
         return adapter
     # No adapter found.
@@ -156,6 +174,12 @@ def getAdapter(config,url):
 def getConfigSections():
     return [cls.getConfigSection() for cls in __class_list]
 
+def getSiteExamples():
+    l=[]
+    for cls in sorted(__class_list, key=lambda x : x.getConfigSection()):
+        l.append((cls.getConfigSection(),cls.getSiteExampleURLs().split()))
+    return l
+
 def getConfigSectionFor(url):
     (cls,fixedurl) = getClassFor(url)
     if cls:
@@ -180,19 +204,21 @@ def getClassFor(url):
     cls = getClassFromList(domain)
     if not cls and domain.startswith("www."):
         domain = domain.replace("www.","")
-        logger.debug("trying site:without www: "+domain)
+        #logger.debug("trying site:without www: "+domain)
         cls = getClassFromList(domain)
         fixedurl = fixedurl.replace("http://www.","http://")
     if not cls:
-        logger.debug("trying site:www."+domain)
+        #logger.debug("trying site:www."+domain)
         cls = getClassFromList("www."+domain)
         fixedurl = fixedurl.replace("http://","http://www.")
-        
-    fixedurl = cls.stripURLParameters(fixedurl)
+
+    if cls:
+        fixedurl = cls.stripURLParameters(fixedurl)
     
     return (cls,fixedurl)
     
 def getClassFromList(domain):
-    for cls in __class_list:
-        if cls.matchesSite(domain):
-            return cls
+    try:
+        return __domain_map[domain]
+    except KeyError:
+        pass # return none.

fanficdownloader/adapters/adapter_adastrafanficcom.py

@@ -42,7 +42,7 @@ class AdAstraFanficComSiteAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -52,6 +52,7 @@ class AdAstraFanficComSiteAdapter(BaseSiteAdapter):
     def getSiteDomain():
         return 'www.adastrafanfic.com'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 
@@ -89,7 +90,7 @@ class AdAstraFanficComSiteAdapter(BaseSiteAdapter):
 
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php"))
@@ -195,6 +196,7 @@ class AdAstraFanficComSiteAdapter(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             

fanficdownloader/adapters/adapter_archiveofourownorg.py

@@ -1,4 +1,4 @@
-# -*- coding: utf-8 -*-
+#  -*- coding: utf-8 -*-
 
 # Copyright 2011 Fanficdownloader team
 #
@@ -51,13 +51,13 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # get storyId from url--url validation guarantees query correct
         m = re.match(self.getSiteURLPattern(),url)
         if m:
             self.story.setMetadata('storyId',m.group('id'))
-            logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+            
             # normalized story URL.
             self._setURL('http://' + self.getSiteDomain() + '/works/'+self.story.getMetadata('storyId'))
         else:
@@ -77,12 +77,14 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'archiveofourown.org'
 
+    @classmethod
     def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/works/123456 http://"+self.getSiteDomain()+"/collections/Some_Archive/works/123456"
+        return "http://"+self.getSiteDomain()+"/works/123456 http://"+self.getSiteDomain()+"/collections/Some_Archive/works/123456 http://"+self.getSiteDomain()+"/works/123456/chapters/78901"
 
     def getSiteURLPattern(self):
         # http://archiveofourown.org/collections/Smallville_Slash_Archive/works/159770
-        return re.escape("http://")+re.escape(self.getSiteDomain())+r"(/collections/[^/]+)?/works/(?P<id>\d+)"
+        # Discard leading zeros from story ID numbers--AO3 doesn't use them in it's own chapter URLs.
+        return re.escape("http://")+re.escape(self.getSiteDomain())+r"(/collections/[^/]+)?/works/0*(?P<id>\d+)"
         
     ## Login
     def needToLoginCheck(self, data):
@@ -161,7 +163,7 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('a', href=re.compile(r"^/works/\d+$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
 		
         # Find authorid and URL from... author url.
         alist = soup.findAll('a', href=re.compile(r"^/users/\w+/pseuds/\w+"))
@@ -208,10 +210,7 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
         if a != None:
             warnings = a.findAll('a',{'class':"tag"})
             for warning in warnings:
-                if warning.string == "Author Chose Not To Use Archive Warnings":
-                    warning.string = "No Archive Warnings Apply"
-                if warning.string != "No Archive Warnings Apply":
-                    self.story.addToList('warnings',warning.string)
+                self.story.addToList('warnings',warning.string)
 		
         a = metasoup.find('dd',{'class':"freeform tags"})
         if a != None:
@@ -286,18 +285,19 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
                 self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
 
 		
-        try:
-            # Find Series name from series URL.
-            a = metasoup.find('dd',{'class':"series"})
-            b = a.find('a', href=re.compile(r"/series/\d+"))
-            series_name = b.string
-            series_url = 'http://'+self.host+'/fanfic/'+b['href']
-            series_index = int(a.text.split(' ')[1])
-            self.setSeries(series_name, series_index)
-            
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
+        # Find Series name from series URL.
+        ddseries = metasoup.find('dd',{'class':"series"})
+
+        if ddseries:
+            for i, a in enumerate(ddseries.findAll('a', href=re.compile(r"/series/\d+"))):
+                series_name = stripHTML(a)
+                series_url = 'http://'+self.host+a['href']
+                series_index = int(stripHTML(a.previousSibling).replace(', ','').split(' ')[1]) # "Part # of" or ", Part #"
+                self.story.setMetadata('series%02d'%i,"%s [%s]"%(series_name,series_index))
+                self.story.setMetadata('series%02dUrl'%i,series_url)
+                if i == 0:
+                    self.setSeries(series_name, series_index)
+                    self.story.setMetadata('seriesUrl',series_url)
 
     # grab the text for an individual chapter.
     def getChapterText(self, url):

fanficdownloader/adapters/adapter_archiveskyehawkecom.py

@@ -49,7 +49,7 @@ class ArchiveSkyeHawkeComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/story.php?no='+self.story.getMetadata('storyId'))
@@ -66,14 +66,16 @@ class ArchiveSkyeHawkeComAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'archive.skyehawke.com'
 
+    @classmethod
+    def getAcceptDomains(cls):
+        return ['archive.skyehawke.com','www.skyehawke.com']
+
+    @classmethod
     def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/story.php?no=1234"
+        return "http://archive.skyehawke.com/story.php?no=1234 http://www.skyehawke.com/archive/story.php?no=1234  http://skyehawke.com/archive/story.php?no=1234"
 
     def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/story.php?no=")+r"\d+$"
-
-
-        
+        return re.escape("http://")+r"(archive|www)\.skyehawke\.com/(archive/)?story\.php\?no=\d+$"
 
     ## Getting the chapter list and the meta data, plus 'is adult' checking.
     def extractChapterUrlsAndMetadata(self):
@@ -99,7 +101,7 @@ class ArchiveSkyeHawkeComAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('div', {'class':"story border"}).find('span',{'class':'left'})
-        title=a.text.split('"')[1]
+        title=stripHTML(a).split('"')[1]
         self.story.setMetadata('title',title)
         
         # Find authorid and URL from... author url.

fanficdownloader/adapters/adapter_ashwindersycophanthexcom.py

@@ -48,7 +48,7 @@ class AshwinderSycophantHexComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
 		
         
         # normalized story URL.
@@ -66,6 +66,7 @@ class AshwinderSycophantHexComAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'ashwinder.sycophanthex.com'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 
@@ -156,7 +157,7 @@ class AshwinderSycophantHexComAdapter(BaseSiteAdapter):
         for info in asoup.findAll('table', {'width' : '100%', 'bordercolor' : re.compile(r'#')}):
             a = info.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
             if a != None:
-                self.story.setMetadata('title',a.string)
+                self.story.setMetadata('title',stripHTML(a))
                 break
 		
 

fanficdownloader/adapters/adapter_asr3slashzoneorg.py

@@ -0,0 +1,226 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return Asr3SlashzoneOrgAdapter
+
+class Asr3SlashzoneOrgAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/archive/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','asr3')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%d/%m/%y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'asr3.slashzone.org'
+
+    @classmethod
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/archive/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/archive/viewstory.php?sid=")+r"\d+$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=3"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid & error in url.
+                addurl = addurl.replace("&","&")
+                url = self.url+'&index=1'+addurl
+                logger.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        #print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',stripHTML(a))
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/archive/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Rating
+        rate = stripHTML(soup.find('div',{'id':'pagetitle'}))
+        rate = rate[rate.rindex('[')+1:rate.rindex(']')]
+        self.story.setMetadata('rating', rate)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/archive/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        metadiv = soup.find('div',{'class':'content'})
+        smalldiv = metadiv.find('div',{'class':'small'})
+
+        categorys = smalldiv.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+        for category in categorys:
+            self.story.addToList('category',category.string)
+        
+        chars = smalldiv.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+        for char in chars:
+            self.story.addToList('characters',char.string)
+
+        ships = smalldiv.parent.findAll('a',href=re.compile(r'browse\.php\?type=class&type_id=2&classid=1'))
+        for ship in ships:
+            self.story.addToList('ships',ship.string)
+
+        metatext = stripHTML(smalldiv)
+
+        if 'Completed: Yes' in metatext:
+            self.story.setMetadata('status', 'Completed')
+        else:
+            self.story.setMetadata('status', 'In-Progress')
+
+        wordstart=metatext.rindex('Word count:')+12
+        words = metatext[wordstart:metatext.index(' ',wordstart)]
+        self.story.setMetadata('numWords', words)
+
+        datesdiv = soup.find('div',{'class':'bottom'})
+        dates = stripHTML(datesdiv).split()
+        # Published: 04/26/2011 Updated: 03/06/2013
+        self.story.setMetadata('datePublished', makeDate(dates[1], self.dateformat))
+        self.story.setMetadata('dateUpdated', makeDate(dates[3], self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/archive/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            i=1
+            for a in storyas:
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
+                        break
+                    i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+        # remove 'small' leaving only summary.
+        smalldiv.extract()
+        self.setDescription(url,metadiv)
+        
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulSoup(self._fetchUrl(url))
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_bloodtiesfancom.py

@@ -70,7 +70,7 @@ class BloodTiesFansComAdapter(BaseSiteAdapter): # XXX
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
@@ -88,6 +88,7 @@ class BloodTiesFansComAdapter(BaseSiteAdapter): # XXX
         # The site domain.  Does have www here, if it uses it.
         return 'bloodties-fans.com' # XXX
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/fiction/viewstory.php?sid=1234"
 
@@ -206,12 +207,12 @@ class BloodTiesFansComAdapter(BaseSiteAdapter): # XXX
         
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
         self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/fiction/'+a['href'])
         self.story.setMetadata('author',a.string)
 
         # Find the chapters:
@@ -311,6 +312,7 @@ class BloodTiesFansComAdapter(BaseSiteAdapter): # XXX
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             

fanficdownloader/adapters/adapter_buffynfaithnet.py

@@ -0,0 +1,291 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+import cookielib as cl
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+
+# This function is called by the downloader in all adapter_*.py files
+# in this dir to register the adapter class.  So it needs to be
+# updated to reflect the class below it.  That, plus getSiteDomain()
+# take care of 'Registering'.
+def getClass():
+    return BuffyNFaithNetAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class BuffyNFaithNetAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+        
+        self.setHeader()
+        
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query correct
+        m = re.match(self.getSiteURLPattern(),url)
+        if m:
+            self.story.setMetadata('storyId',m.group('id'))
+            
+            # normalized story URL. gets rid of chapter if there, left with ch 1 URL on this site
+            nurl = "http://"+self.getSiteDomain()+"/fanfictions/index.php?act=vie&id="+self.story.getMetadata('storyId')
+            self._setURL(nurl)
+            #argh, this mangles the ampersands I need on metadata['storyUrl']
+            #will set it this way
+            self.story.setMetadata('storyUrl',nurl,condremoveentities=False)
+        else:
+            raise exceptions.InvalidStoryURL(url,
+                                             self.getSiteDomain(),
+                                             self.getSiteExampleURLs())
+        
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','bnfnet')
+
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'buffynfaith.net'
+
+    @classmethod
+    def stripURLParameters(cls,url):
+        "Only needs to be overriden if URL contains more than one parameter"
+        ## This adapter needs at least two parameters left on the URL, act and id
+        return re.sub(r"(\?act=(vie|ovr)&id=\d+)&.*$",r"\1",url)
+    
+    def setHeader(self):
+        "buffynfaith.net wants a Referer for images.  Used both above and below(after cookieproc added)"
+        self.opener.addheaders = [('Referer', 'http://'+self.getSiteDomain()+'/')]
+        
+    @classmethod
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/fanfictions/index.php?act=vie&id=1234 http://buffynfaith.net/fanfictions/index.php?act=ovr&id=1234 http://buffynfaith.net/fanfictions/index.php?act=vie&id=1234&ch=2"
+
+    def getSiteURLPattern(self):
+        #http://buffynfaith.net/fanfictions/index.php?act=vie&id=963
+        #http://buffynfaith.net/fanfictions/index.php?act=vie&id=949
+        #http://buffynfaith.net/fanfictions/index.php?act=vie&id=949&ch=2
+        p = re.escape("http://"+self.getSiteDomain()+"/fanfictions/index.php?act=")+\
+            r"(vie|ovr)&id=(?P<id>\d+)(&ch=(?P<ch>\d+))?$"
+        return p
+
+    def extractChapterUrlsAndMetadata(self):
+
+        dateformat = "%d %B %Y"
+        url = self.url
+        logger.debug("URL: "+url)
+        
+        #set a cookie to get past adult check
+        if self.is_adult or self.getConfig("is_adult"):
+            cookieproc = urllib2.HTTPCookieProcessor()
+            cookie = cl.Cookie(version=0, name='my_age', value='yes',
+                               port=None, port_specified=False,
+                               domain=self.getSiteDomain(), domain_specified=False, domain_initial_dot=False,
+                               path='/', path_specified=True,
+                               secure=False,
+                               expires=time.time()+10000,
+                               discard=False,
+                               comment=None,
+                               comment_url=None,
+                               rest={'HttpOnly': None},
+                               rfc2109=False)
+            cookieproc.cookiejar.set_cookie(cookie)
+            self.opener = urllib2.build_opener(cookieproc)        
+            self.setHeader()
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        #print data
+
+        if "ADULT CONTENT WARNING" in data: 
+            raise exceptions.AdultCheckRequired(self.url)
+
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        #stuff in <head>: description
+        svalue = soup.head.find('meta',attrs={'name':'description'})['content']
+        #self.story.setMetadata('description',svalue)
+        self.setDescription(url,svalue)
+        
+        #useful stuff in rest of doc, all contained in this:
+        doc = soup.body.find('div', id='my_wrapper')
+        
+        #first the site category (more of a genre to me, meh) and title, in this element:
+        mt = doc.find('div',attrs={'class':'maintitle'})
+        self.story.addToList('genre',mt.findAll('a')[1].string)
+        self.story.setMetadata('title',mt.findAll('a')[1].nextSibling[len('&nbsp;&raquo;&nbsp;'):])
+        del mt
+
+        #the actual category, for me, is 'Buffy: The Vampire Slayer'
+        #self.story.addToList('category','Buffy: The Vampire Slayer')
+        #No need to do it here, it is better to set it in in plugin-defaults.ini and defaults.ini
+        
+        #then a block that sits in a table cell like so:
+        #(contains a lot of metadata)
+        mblock = doc.find('td', align='left', width = '70%').contents
+        while len(mblock) > 0:
+            i = mblock.pop(0)
+            if 'Author:' in i.string:
+                #drop empty space
+                mblock.pop(0)
+                #get author link
+                a = mblock.pop(0)
+                authre = re.escape('./index.php?act=bio&id=')+'(?P<authid>\d+)'
+                m = re.match(authre,a['href'])
+                self.story.setMetadata('author',a.string)
+                self.story.setMetadata('authorId',m.group('authid'))
+                authurl = u'http://%s/fanfictions/index.php?act=bio&id=%s' % ( self.getSiteDomain(),
+                                            self.story.getMetadata('authorId'))
+                self.story.setMetadata('authorUrl',authurl,condremoveentities=False)
+                #drop empty space
+                mblock.pop(0)
+            if 'Rating:' in i.string:
+                self.story.setMetadata('rating',mblock.pop(0).strip())
+            if 'Published:' in i.string:
+                date = mblock.pop(0).strip()
+                #get rid of 'st', 'nd', 'rd', 'th' after day number
+                date = date[0:2]+date[4:]
+                self.story.setMetadata('datePublished',makeDate(date, dateformat))
+            if 'Last Updated:' in i.string:
+                date = mblock.pop(0).strip()
+                #get rid of 'st', 'nd', 'rd', 'th' after day number
+                date = date[0:2]+date[4:]
+                self.story.setMetadata('dateUpdated',makeDate(date, dateformat))
+            if 'Genre:' in i.string:
+                genres = mblock.pop(0).strip()
+                genres = genres.split('/')
+                for genre in genres: self.story.addToList('genre',genre)
+            #end ifs
+        #end while
+
+        # Find the chapter selector 
+        select = soup.find('select', { 'name' : 'ch' } )
+         
+        if select is None:
+           # no selector found, so it's a one-chapter story.
+           #self.chapterUrls.append((self.story.getMetadata('title'),url))
+           self.chapterUrls.append((self.story.getMetadata('title'),url))
+        else:
+            allOptions = select.findAll('option')
+            for o in allOptions:
+                url = u'http://%s/fanfictions/index.php?act=vie&id=%s&ch=%s' % ( self.getSiteDomain(),
+                                            self.story.getMetadata('storyId'),
+                                            o['value'])
+                title = u"%s" % o
+                title = stripHTML(title)
+                ts = title.split(' ',1)
+                title = ts[0]+'. '+ts[1]
+                self.chapterUrls.append((title,url))
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+        
+        ## Go scrape the rest of the metadata from the author's page.
+        data = self._fetchUrl(self.story.getMetadata('authorUrl'))
+        soup = bs.BeautifulSoup(data)
+        #find the story link and its parent div
+        storya = soup.find('a',{'href':self.story.getMetadata('storyUrl')})
+        storydiv = storya.parent
+        #warnings come under a <spawn> tag. Never seen that before...
+        #appears to just be a line of freeform text, not necessarily a list
+        #optional
+        spawn = storydiv.find('spawn',{'id':'warnings'})
+        if spawn is not None:
+            warns = spawn.nextSibling.strip()
+            self.story.addToList('warnings',warns)
+        #some meta in spans - this should get all, even the ones jammed in a table
+        spans = storydiv.findAll('span')
+        for s in spans:
+            if s.string == 'Ship:':
+                list = s.nextSibling.strip().split()
+                self.story.extendList('ships',list)
+            if s.string == 'Characters:':
+                list = s.nextSibling.strip().split(',')
+                self.story.extendList('characters',list)
+            if s.string == 'Status:':
+                st = s.nextSibling.strip()
+                self.story.setMetadata('status',st)
+            if s.string == 'Words:':
+                st = s.nextSibling.strip()
+                self.story.setMetadata('numWords',st)
+
+        #reviews - is this worth having?
+        #ffnet adapter gathers it, don't know if anything else does
+        #or if it's ever going to be used!
+        a = storydiv.find('a',{'id':'bold-blue'})
+        if a:
+            revs = a.nextSibling.strip()[1:-1]
+            self.story.setMetadata('reviews',st)
+        else:
+            revs = '0'
+            self.story.setMetadata('reviews',st)
+
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'fanfiction'})
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+        
+        #remove all the unnecessary bookmark tags
+        [s.extract() for s in div('div',{'class':"tiny_box2"})]
+        
+        #is there a review link?
+        r = div.find('a',href=re.compile(re.escape("./index.php?act=irv")+".*$"))
+        if r is not None:
+        #remove the review link and its parent div
+            r.parent.extract()
+        
+        #There might also be a link to the sequel on the last chapter
+        #I'm inclined to keep it in, but the URL needs to be changed from relative to absolute
+        #Shame there isn't proper series metadata available
+        #(I couldn't find it anyway)
+        s = div.find('a',href=re.compile(re.escape("./index.php?act=ovr")+".*$"))
+        if s is not None:
+            s['href'] = 'http://'+self.getSiteDomain()+'/fanfictions'+s['href'][1:]
+
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_castlefansorg.py

@@ -72,7 +72,7 @@ class CastleFansOrgAdapter(BaseSiteAdapter): # XXX
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
@@ -90,6 +90,7 @@ class CastleFansOrgAdapter(BaseSiteAdapter): # XXX
         # The site domain.  Does have www here, if it uses it.
         return 'castlefans.org' # XXX
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/fanfic/viewstory.php?sid=1234"
 
@@ -179,12 +180,12 @@ class CastleFansOrgAdapter(BaseSiteAdapter): # XXX
         pagetitle = soup.find('div',{'id':'pagetitle'})
         ## Title
         a = pagetitle.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = pagetitle.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
         self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/fanfic/'+a['href'])
         self.story.setMetadata('author',a.string)
 
         # Find the chapters:
@@ -284,6 +285,7 @@ class CastleFansOrgAdapter(BaseSiteAdapter): # XXX
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             

fanficdownloader/adapters/adapter_chaossycophanthexcom.py

@@ -48,7 +48,7 @@ class ChaosSycophantHexComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
 		
         
         # normalized story URL.
@@ -66,6 +66,7 @@ class ChaosSycophantHexComAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'chaos.sycophanthex.com'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 
@@ -115,7 +116,7 @@ class ChaosSycophantHexComAdapter(BaseSiteAdapter):
         ## Title
         pt = soup.find('div', {'id' : 'pagetitle'})
         a = pt.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.text)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = pt.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -212,6 +213,7 @@ class ChaosSycophantHexComAdapter(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             

fanficdownloader/adapters/adapter_checkmatedcom.py

@@ -48,7 +48,7 @@ class CheckmatedComAdapter(BaseSiteAdapter):
 
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
 
         self._setURL('http://' + self.getSiteDomain() + '/story.php?story='+self.story.getMetadata('storyId'))
 
@@ -64,6 +64,7 @@ class CheckmatedComAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'www.checkmated.com'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/story.php?story=1234"
 
@@ -142,11 +143,11 @@ class CheckmatedComAdapter(BaseSiteAdapter):
         # Now go hunting for all the meta data and the chapter list.
 
         ## Title
-        a = soup.findAll('span', {'class' : 'storytitle'})
-        self.story.setMetadata('title',a[0].string)
+        a = soup.find('span', {'class' : 'storytitle'})
+        self.story.setMetadata('title',stripHTML(a))
 
         # Find authorid and URL from... author url.
-        a = a[1].find('a', href=re.compile(r"authors.php\?name\=\w+"))
+        a = a.parent.find('a', href=re.compile(r"authors.php\?name\=\w+"))
         self.story.setMetadata('authorId',a['href'].split('=')[1])
         self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
         self.story.setMetadata('author',a.string)

fanficdownloader/adapters/adapter_darksolaceorg.py

@@ -48,8 +48,7 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-		
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/elysian/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -59,7 +58,7 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
 
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%d %B %Y"
+        self.dateformat = "%B %d, %Y"
             
     @staticmethod # must be @staticmethod, don't remove it.
     def getSiteDomain():
@@ -70,6 +69,7 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
     def getAcceptDomains(cls):
         return ['www.dark-solace.org','dark-solace.org']
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/elysian/viewstory.php?sid=1234"
 
@@ -79,7 +79,8 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
     ## Login seems to be reasonably standard across eFiction sites.
     def needToLoginCheck(self, data):
         if 'This story contains adult content not suitable for children' in data \
-                or "That password doesn't match the one in our database" in data:
+                or "That password doesn't match the one in our database" in data \
+                or "Registered Users Only" in data:
             return True
         else:
             return False
@@ -93,18 +94,16 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
         else:
             params['penname'] = self.getConfig("username")
             params['password'] = self.getConfig("password")
-        params['rememberme'] = '1'
-        params['sid'] = ''
-        params['intent'] = ''
+        params['action'] = 'login'
         params['submit'] = 'Submit'
     
-        loginUrl = 'http://' + self.getSiteDomain() + '/elysian/user.php'
+        loginUrl = 'http://www.' + self.getSiteDomain() + '/elysian/user.php'
         logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['penname']))
     
-        d = self._fetchUrl(loginUrl, params)
+        d = self._postUrl(loginUrl, params)
     
-        if "User Account Page" not in d : #Member Account
+        if "Member Account" not in d : #User Account Page
             logger.info("Failed to login to URL %s as %s, or have no authorization to access the story" % (loginUrl, params['penname']))
             raise exceptions.FailedToLogin(url,params['penname'])
             return False
@@ -113,9 +112,19 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
 
     ## Getting the chapter list and the meta data, plus 'is adult' checking.
     def extractChapterUrlsAndMetadata(self):
+        
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=5"
+        else:
+            addurl=""
+
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
-        url = self.url
+        url = self.url+'&index=1'+addurl
         logger.debug("URL: "+url)
 
         try:
@@ -128,9 +137,30 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
 
         if self.needToLoginCheck(data):
             # need to log in for this one.
-            addurl="&ageconsent=ok"
             self.performLogin(url)
-            data = self._fetchUrl(url+addurl)
+            data = self._fetchUrl(url)
+            
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid & error in url.
+                addurl = addurl.replace("&","&")
+                url = self.url+'&index=1'+addurl
+                logger.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
             
         if "Access denied. This story has not been validated by the adminstrators of this site." in data:
             raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
@@ -142,34 +172,42 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
         # Now go hunting for all the meta data and the chapter list.
         
         ## Title and author
-        a = soup.find('div', {'id' : 'pagetitle'})
+        div = soup.find('div', {'id' : 'pagetitle'})
         
-        aut = a.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        aut = div.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
         self.story.setMetadata('authorId',aut['href'].split('=')[1])
         self.story.setMetadata('authorUrl','http://'+self.host+'/elysian/'+aut['href'])
         self.story.setMetadata('author',aut.string)
         aut.extract()
+
+        # first a tag in pagetitle is title
+        self.story.setMetadata('title',stripHTML(div.find('a')))
+
+        for chapa in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+
+                                                       self.story.getMetadata('storyId')+'&chapter=\d+')):
+            self.chapterUrls.append((stripHTML(chapa),'http://'+self.host+'/elysian/'+chapa['href']))
         
-        self.story.setMetadata('title',a.string[:(len(a.string)-3)])
-
-        # Find the chapters:
-        chapters=soup.find('select', {'name' : 'chapter'})
-        if chapters != None:
-            for chapter in chapters.findAll('option'):
-                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/elysian/viewstory.php?sid='+self.story.getMetadata('storyId')+'&chapter='+chapter['value']))
-        else:
-            self.chapterUrls.append((self.story.getMetadata('title'),url))
-
         self.story.setMetadata('numChapters',len(self.chapterUrls))
-        
+
         asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
-        
-        for list in asoup.findAll('div', {'class' : re.compile('listbox\s+')}):
-            a = list.find('a', href=re.compile(r'viewstory.php\?sid='))
-            if a != None:
-                if 'viewstory.php?sid='+self.story.getMetadata('storyId') in a['href']:
+        storylink = asoup.find('a', href=re.compile(r'viewstory.php\?sid='+
+                                                    self.story.getMetadata('storyId')+'($|[^\d])'))
+        # author's story list is paginated if there's a pagelinks div.
+        # Only need to look in it if the story wasn't on the first page.
+        pagelinks = asoup.find('div',{'id':'pagelinks'})
+        if pagelinks and storylink==None:
+            authpageslist = pagelinks.findAll('a',href=re.compile(r'action=storiesby'))
+            for page in authpageslist[1:]: # skip first, already checked above.
+                asoup = bs.BeautifulSoup(self._fetchUrl('http://'+self.host+'/elysian/'+page['href']))
+                storylink = asoup.find('a', href=re.compile(r'viewstory.php\?sid='+
+                                                            self.story.getMetadata('storyId')+'($|[^\d])'))
+                if storylink:
                     break
 
+        if not storylink:
+            raise exceptions.FailedToDownload("Unable to find story metadata on author's page(s)")
+        
+        metalist = storylink.parent.parent
         # eFiction sites don't help us out a lot with their meta data
         # formating, so it's a little ugly.
 
@@ -182,7 +220,7 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
                 
 
         # <span class="label">Rated:</span> NC-17<br /> etc
-        labels = list.findAll('span', {'class' : 'classification'})
+        labels = metalist.findAll('span', {'class' : 'label'})
         for labelspan in labels:
             label = labelspan.text
             value = labelspan.nextSibling
@@ -190,7 +228,7 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
             if 'Summary' in label:
                 ## Everything until the next span class='label'
                 svalue = ""
-                while not (defaultGetattr(value,'class') == 'classification' or "Chapters: " in stripHTML(value)):
+                while value and not (defaultGetattr(value,'class') == 'label' or "Chapters: " in stripHTML(value)):
                     svalue += str(value)
                     value = value.nextSibling
                 self.setDescription(url,svalue)
@@ -238,34 +276,54 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
 
         try:
             # Find Series name from series URL.
-            a = list.find('a', href=re.compile(r"series.php\?seriesid=\d+"))
+            a = metalist.find('a', href=re.compile(r"series.php\?seriesid=\d+"))
             series_name = a.string
             series_url = 'http://'+self.host+'/elysian/'+a['href']
 
             # use BeautifulSoup HTML parser to make everything easier to find.
             seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
             # can't use ^viewstory...$ in case of higher rated stories with javascript href.
-            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
-            i=1
-            for a in storyas:
-                # skip 'report this' and 'TOC' links
-                if 'contact.php' not in a['href'] and 'index' not in a['href']:
-                    if ('viewstory.php?sid='+self.story.getMetadata('storyId')) in a['href']:
-                        self.setSeries(series_name, i)
+            storylink = seriessoup.find('a', href=re.compile(r'viewstory.php\?sid='+
+                                                             self.story.getMetadata('storyId')+'($|[^\d])'))
+            if storylink and storylink.parent and storylink.parent['class'] != 'title': # in case of links inside story summaries.
+                storylink = None
+
+            offset = 0
+            # series story list is paginated if there's a pagelinks div.
+            # Only need to look in it if the story wasn't on the first page.
+            pagelinks = seriessoup.find('div',{'id':'pagelinks'})
+            if pagelinks and storylink==None:
+                authpageslist = pagelinks.findAll('a',href=re.compile(r'offset='))
+                for page in authpageslist[1:]: # skip first, already checked above.
+                    seriessoup = bs.BeautifulSoup(self._fetchUrl('http://'+self.host+'/elysian/'+page['href']))
+                    storylink = seriessoup.find('a', href=re.compile(r'viewstory.php\?sid='+
+                                                                     self.story.getMetadata('storyId')+'($|[^\d])'))
+                    if storylink and storylink.parent and storylink.parent['class'] != 'title': # in case of links inside story summaries.
+                        storylink = None
+                    if storylink:
+                        offset = int(page['href'].split('=')[-1]) # offset is last.
                         break
-                    i+=1
+
+            # for reasons I don't understand, searching for story
+            # links by regex wasn't working reliably.  It was missing
+            # the javascript links sometimes.  This is cleaner anyway.
+            for i, div in enumerate(seriessoup.findAll('div', {'class':'title'})):
+                a = div.find('a') # first a is story link.
+                # skip 'report this' and 'TOC' links
+                if a == storylink:
+                    self.setSeries(series_name, 1+i+offset)
+                    self.story.setMetadata('seriesUrl',series_url)
+                    break
             
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
+        except Exception, e:
+            print("Series parsing failed: %s"%e)
             
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
         logger.debug('Getting chapter text from: %s' % url)
 
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        soup = bs.BeautifulSoup(self._fetchUrl(url))
         
         div = soup.find('div', {'id' : 'story'})
 

fanficdownloader/adapters/adapter_destinysgatewaycom.py

@@ -48,7 +48,7 @@ class DestinysGatewayComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -65,6 +65,7 @@ class DestinysGatewayComAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'www.destinysgateway.com'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 
@@ -130,7 +131,7 @@ class DestinysGatewayComAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -218,6 +219,7 @@ class DestinysGatewayComAdapter(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             

fanficdownloader/adapters/adapter_dokugacom.py

@@ -48,7 +48,7 @@ class DokugaComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[3])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
 
         # www.dokuga.com has two 'sections', shown in URL as
         # 'fanfiction' and 'spark' that change how things should be
@@ -75,12 +75,51 @@ class DokugaComAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'www.dokuga.com'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/fanfiction/story/1234/1 http://"+self.getSiteDomain()+"/spark/story/1234/1"
 
     def getSiteURLPattern(self):
         return r"http://"+self.getSiteDomain()+"/(fanfiction|spark)?/story/\d+/?\d+?$"
 
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'The author has disabled anonymous viewing for this story.' in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url,soup):
+        params = {}
+
+        if self.password:
+            params['username'] = self.username
+            params['passwd'] = self.password
+        else:
+            params['username'] = self.getConfig("username")
+            params['passwd'] = self.getConfig("password")
+        params['Submit'] = 'Submit'
+
+        # copy all hidden input tags to pick up appropriate tokens.
+        for tag in soup.findAll('input',{'type':'hidden'}):
+            params[tag['name']] = tag['value']
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/fanfiction'
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['username']))
+    
+        d = self._postUrl(loginUrl, params)
+
+        if "Your session has expired. Please log in again." in d:
+            d = self._postUrl(loginUrl, params)
+
+        if "Logout" not in d : #Member Account
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['username']))
+            raise exceptions.FailedToLogin(url,params['username'])
+            return False
+        else:
+            return True
 
     ## Getting the chapter list and the meta data, plus 'is adult' checking.
     def extractChapterUrlsAndMetadata(self):
@@ -97,12 +136,18 @@ class DokugaComAdapter(BaseSiteAdapter):
                 raise exceptions.StoryDoesNotExist(self.url)
             else:
                 raise e
+                
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url,soup)
+            data = self._fetchUrl(url)
+            soup = bs.BeautifulSoup(data)
             
         if "Access denied. This story has not been validated by the adminstrators of this site." in data:
             raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
         # print data
 
         # Now go hunting for all the meta data and the chapter list.
@@ -119,7 +164,7 @@ class DokugaComAdapter(BaseSiteAdapter):
         aut.extract()
         
         a = a.string[:(len(a.string)-4)]
-        self.story.setMetadata('title',a)
+        self.story.setMetadata('title',stripHTML(a))
 
         # Find the chapters:
         chapters = soup.find('select').findAll('option')
@@ -160,7 +205,6 @@ class DokugaComAdapter(BaseSiteAdapter):
             
             a=div.text.split('Category: ')
             if len(a) == 2: self.story.addToList('category', a[1].split('&')[0])
-            self.story.addToList('category', 'Fanfiction')
         
             a=div.text.split('Created: ')
             if len(a) == 2: self.story.setMetadata('datePublished', makeDate(stripHTML(a[1].split('&')[0]), self.dateformat))
@@ -218,10 +262,6 @@ class DokugaComAdapter(BaseSiteAdapter):
             a=div.text.split('Words ')
             if len(a)==2: self.story.setMetadata('numWords', a[1])
             
-        
-
-
-            
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 

fanficdownloader/adapters/adapter_dotmoonnet.py

@@ -0,0 +1,217 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return DotMoonNetAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class DotMoonNetAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        
+        
+        # normalized story URL. www.dotmoon.net/library_view.php?storyid=3
+        self._setURL('http://' + self.getSiteDomain() + '/library_view.php?storyid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','dotm')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%Y-%m-%d"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.dotmoon.net'
+
+    @classmethod
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/library_view.php?storyid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/library_view.php?storyid=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'You must be logged in to read adult-rated stories' in data \
+                or 'Password incorrect' in data \
+                or "That username does not exist" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['user'] = self.username
+            params['passwrd'] = self.password
+        else:
+            params['user'] = self.getConfig("username")
+            params['passwrd'] = self.getConfig("password")
+
+        loginUrl = 'http://' + self.getSiteDomain() + '/board/index.php'
+        
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['user']))
+    
+        d = self._fetchUrl(loginUrl+'?action=login2&user='+params['user']+'&passwrd='+params['passwrd'])
+        d = self._fetchUrl(loginUrl)
+    
+        if "Show unread posts since last visit" not in d : #Member Account
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['user']))
+            raise exceptions.FailedToLogin(url,params['user'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+            
+        if "Invalid story ID" in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Invalid story ID.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        body=soup.findAll('body')[1]
+        body.find('table').extract()
+        
+        ## Title
+        a = body.find('b')
+        self.story.setMetadata('title',stripHTML(a))
+        
+        # Find authorid and URL from... author url. http://www.dotmoon.net/board/index.php?action=profile;u=1'
+        a = body.find('a', href=re.compile(r"index.php\?action=profile;u=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[2])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters: 'library_storyview.php?chapterid=3
+        chapters=body.findAll('a', href=re.compile(r"library_storyview.php\?chapterid=\d+$"))
+        if len(chapters)==0:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: No php/html chapters found.")
+        if len(chapters)==1:
+            self.chapterUrls.append((self.story.getMetadata('title'),'http://'+self.host+'/'+chapters[0]['href']))
+        else:
+            for chapter in chapters:
+                # just in case there's tags, like <i> in chapter titles.
+                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # other tags
+
+        labels = body.find('table', {'width':'390'}).findAll('td')
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+            
+            if label != None:
+                if 'Fandom' in label:
+                    self.story.addToList('category',value.string)
+                
+                if 'Setting' in label:
+                    self.story.addToList('genre',value.string)
+                
+                if 'Genre' in label:
+                    self.story.addToList('genre',value.string)
+                
+                if 'Style' in label:
+                    self.story.addToList('genre',value.string)
+                
+                if 'Rating' in label:
+                    self.story.addToList('rating',value.string)
+                
+                if 'Created' in label:
+                    self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+                if 'Updated' in label:
+                    self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+        
+                if 'Status' in label:
+                    if 'Completed' in value.string:
+                        self.story.setMetadata('status', 'Completed')
+                    else:
+                        self.story.setMetadata('status', 'In-Progress')
+                        
+        table=body.findAll('table', {'width':'400'})[1].find('td')
+        self.setDescription(url,stripHTML(table).split('Summary: ')[1])
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('blockquote')
+        div.name='div'
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_dracoandginnycom.py

@@ -48,7 +48,7 @@ class DracoAndGinnyComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -65,6 +65,7 @@ class DracoAndGinnyComAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'www.dracoandginny.com'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 
@@ -169,7 +170,7 @@ class DracoAndGinnyComAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -273,6 +274,7 @@ class DracoAndGinnyComAdapter(BaseSiteAdapter):
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             

fanficdownloader/adapters/adapter_dramioneorg.py

@@ -48,7 +48,7 @@ class DramioneOrgAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -65,6 +65,7 @@ class DramioneOrgAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'dramione.org'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 
@@ -153,7 +154,7 @@ class DramioneOrgAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -237,6 +238,9 @@ class DramioneOrgAdapter(BaseSiteAdapter):
             if 'Word count' in label:
                 self.story.setMetadata('numWords', value)
 
+            if 'Read' in label:
+                self.story.setMetadata('read', value)
+
             if 'Categories' in label:
                 cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
                 for cat in cats:
@@ -274,6 +278,7 @@ class DramioneOrgAdapter(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             
@@ -281,6 +286,14 @@ class DramioneOrgAdapter(BaseSiteAdapter):
             # I find it hard to care if the series parsing fails
             pass
             
+        try:
+            self.story.setMetadata('reviews',
+                                   stripHTML(soup.find('h2',{'id':'pagetitle'}).
+                                             findAll('a', href=re.compile(r'^reviews.php'))[1]))
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 

fanficdownloader/adapters/adapter_efictionestelielde.py

@@ -0,0 +1,223 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return EfictionEstelielDeAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class EfictionEstelielDeAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','eesd')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%B %d, %Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'efiction.esteliel.de'
+
+    @classmethod
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+       
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title and author
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+
+        pagetitle = soup.find('div',{'id':'pagetitle'})
+        ## Title
+        a = pagetitle.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',stripHTML(a))
+        
+        # Find authorid and URL from... author url.
+        a = pagetitle.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        list = soup.find('div', {'class':'listbox'})
+        labelspan=list.find('span',{'class':'label'})
+        value = labelspan.nextSibling
+        label = labelspan.string
+        genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+        for genre in genres:
+            self.story.addToList('genre',genre.string)
+        
+        labels = list.findAll('b')
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while 'Rating' not in str(value):
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rating' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Words' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Category' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            if list.find('a', href=re.compile(r"series.php")) != None:
+                for series in asoup.findAll('a', href=re.compile(r"series.php\?seriesid=\d+")):
+                    # Find Series name from series URL.
+                    series_url = 'http://'+self.host+'/'+series['href']
+                    # use BeautifulSoup HTML parser to make everything easier to find.
+                    seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+                    storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+                    i=1
+                    for a in storyas:
+                        if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                            name=seriessoup.find('div', {'id' : 'pagetitle'})
+                            name.find('a').extract()
+                            self.setSeries(name.text.split(' by[')[0], i)
+                            self.story.setMetadata('seriesUrl',series_url)
+                            i=0
+                            break
+                        i+=1
+                    if i == 0:
+                        break
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+                                     
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_efpfanficnet.py

@@ -48,7 +48,7 @@ class EFPFanFicNet(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -65,6 +65,7 @@ class EFPFanFicNet(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'www.efpfanfic.net'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 
@@ -165,7 +166,7 @@ class EFPFanFicNet(BaseSiteAdapter):
         logger.debug("Normalizing to URL: "+url)
         self._setURL(url)
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
 
         # eFiction sites don't help us out a lot with their meta data
         # formating, so it's a little ugly.
@@ -280,6 +281,7 @@ class EFPFanFicNet(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId'))+'&i=1':
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             

fanficdownloader/adapters/adapter_erosnsapphosycophanthexcom.py

@@ -48,7 +48,7 @@ class ErosnSapphoSycophantHexComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
 		
         
         # normalized story URL.
@@ -66,6 +66,7 @@ class ErosnSapphoSycophantHexComAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'erosnsappho.sycophanthex.com'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 
@@ -131,7 +132,7 @@ class ErosnSapphoSycophantHexComAdapter(BaseSiteAdapter):
         ## Title
         pt = soup.find('div', {'id' : 'pagetitle'})
         a = pt.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.text)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = pt.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -230,6 +231,7 @@ class ErosnSapphoSycophantHexComAdapter(BaseSiteAdapter):
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             

fanficdownloader/adapters/adapter_fanfictionnet.py

@@ -64,8 +64,9 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
     def getAcceptDomains(cls):
         return ['www.fanfiction.net','m.fanfiction.net']
 
+    @classmethod
     def getSiteExampleURLs(self):
-        return "http://www.fanfiction.net/s/1234/1/ http://www.fanfiction.net/s/1234/12/ http://www.fanfiction.net/s/1234/1/Story_Title"
+        return "http://www.fanfiction.net/s/1234/1/ http://www.fanfiction.net/s/1234/12/ http://www.fanfiction.net/s/1234/1/Story_Title http://m.fanfiction.net/s/1234/1/"
 
     def getSiteURLPattern(self):
         return r"http://(www|m)?\.fanfiction\.net/s/\d+(/\d+)?(/|/[^/]+)?/?$"
@@ -132,14 +133,28 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         ## For 1, use the second link.
         ## For 2, fetch the crossover page and pull the two categories from there.
 
-        categories = soup.findAll('a',{'class':'xcontrast_txt'})
+        categories = soup.find('div',{'id':'pre_story_links'}).findAll('a',{'class':'xcontrast_txt'})
+        #print("xcontrast_txt a:%s"%categories)
         if len(categories) > 1:
-            self.story.addToList('category',stripHTML(categories[-1]))
+            # Strangely, the ones with *two* links are the
+            # non-crossover categories.  Each is in a category itself
+            # of Book, Movie, etc.
+            self.story.addToList('category',stripHTML(categories[1]))
         elif 'Crossover' in categories[0]['href']:
             caturl = "http://%s%s"%(self.getSiteDomain(),categories[0]['href'])
             catsoup = bs.BeautifulSoup(self._fetchUrl(caturl))
             for a in catsoup.findAll('a',href=re.compile(r"^/crossovers/")):
                 self.story.addToList('category',stripHTML(a))
+            else:
+                # Fall back.  I ran across a story with a Crossver
+                # category link to a broken page once.
+                # http://www.fanfiction.net/s/2622060/1/
+                # Naruto + Harry Potter Crossover
+                logger.info("Fall back category collection")
+                for c in stripHTML(categories[0]).replace(" Crossover","").split(' + '):
+                    self.story.addToList('category',c)
+                
+                
             
         a = soup.find('a', href='http://www.fictionratings.com/')
         rating = a.string
@@ -150,7 +165,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
 
         # after Rating, the same bit of text containing id:123456 contains
         # Complete--if completed.
-        gui_table1i = soup.find(id="gui_table1i")
+        gui_table1i = soup.find('table',{'cellpadding':'5'})
 
         self.story.setMetadata('title', stripHTML(gui_table1i.find('b'))) # title appears to be only(or at least first) bold tag in gui_table1i
 
@@ -158,12 +173,17 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         if summarydiv:
             self.setDescription(url,stripHTML(summarydiv))
             
-        
-        metatext = stripHTML(gui_table1i.find('div', {'style':'color:gray;'})).replace('Hurt/Comfort','Hurt-Comfort')
+
+        grayspan = gui_table1i.find('span', {'class':'xgray xcontrast_txt'})
+        # for b in grayspan.findAll('button'):
+        #     b.extract()
+        metatext = stripHTML(grayspan).replace('Hurt/Comfort','Hurt-Comfort')
+        #logger.debug("metatext:(%s)"%metatext)
         metalist = metatext.split(" - ")
-        #logger.debug("metatext:(%s)"%metalist)
+        #logger.debug("metalist:(%s)"%metalist)
 
         # Rated: Fiction K - English - Words: 158,078 - Published: 02-04-11
+        # Rated: Fiction T - English - Adventure/Sci-Fi - Naruto U. - Chapters: 22 - Words: 114,414 - Reviews: 395 - Favs: 779 - Follows: 835 - Updated: 03-21-13 - Published: 04-28-12 - id: 8067258 
 
         # rating is obtained above more robustly.
         if metalist[0].startswith('Rated:'):
@@ -179,7 +199,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         for g in genrelist:
             #logger.debug("g:(%s)"%g)
             if g.strip() not in ffnetgenres:
-                logger.info("g not in ffnetgenres")
+                #logger.info("g not in ffnetgenres")
                 goodgenres=False
         if goodgenres:
             self.story.extendList('genre',genrelist)
@@ -246,19 +266,26 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         return
 
     def getChapterText(self, url):
-        logger.debug('Getting chapter text from: %s' % url)
-        time.sleep(0.5) ## ffnet(and, I assume, fpcom) tends to fail
+        time.sleep(5.0) ## ffnet(and, I assume, fpcom) tends to fail
                         ## more if hit too fast.  This is in
                         ## additional to what ever the
                         ## slow_down_sleep_time setting is.
+        logger.debug('Getting chapter text from: %s' % url)
         data = self._fetchUrl(url)
 
+        if "Please email this error message in full to <a href='mailto:support@fanfiction.com'>support@fanfiction.com</a>" in data:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  FanFiction.net Site Error!" % url)
+        
         # some ancient stories have body tags inside them that cause
         # soup parsing to discard the content.  For story text we
         # don't care about anything before "<div class='storytextp"
         # (there's a space after storytextp, so no close quote(')) and
         # this kills any body tags.
-        data = data[data.index("<div class='storytextp"):]
+        divstr = "<div role='main' class='storytextp"
+        if divstr not in data:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+        else:
+            data = data[data.index(divstr):]
         data.replace("<body","<notbody").replace("<BODY","<NOTBODY")
         
         soup = bs.BeautifulSoup(data)

fanficdownloader/adapters/adapter_fanfiktionde.py

@@ -50,7 +50,7 @@ class FanFiktionDeAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/s/'+self.story.getMetadata('storyId') + '/1')
@@ -67,11 +67,12 @@ class FanFiktionDeAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'www.fanfiktion.de'
 
+    @classmethod
     def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/s/46ccbef30000616306614050"
+        return "http://"+self.getSiteDomain()+"/s/46ccbef30000616306614050 http://"+self.getSiteDomain()+"/s/46ccbef30000616306614050/1 http://"+self.getSiteDomain()+"/s/46ccbef30000616306614050/1/story-name"
 
     def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/s/")+r"\w+(/\d+)?$"
+        return re.escape("http://"+self.getSiteDomain()+"/s/")+r"\w+(/\d+)?"
         
         ## Login seems to be reasonably standard across eFiction sites.
     def needToLoginCheck(self, data):
@@ -137,15 +138,15 @@ class FanFiktionDeAdapter(BaseSiteAdapter):
         # Now go hunting for all the meta data and the chapter list.
         
         ## Title
-        a = soup.find('a', href=re.compile(r'/s/'+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        a = soup.find('a', href=re.compile(r'/s/'+self.story.getMetadata('storyId')+"/"))
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         head = soup.find('div', {'class' : 'story-metadata-left-top'})
         a = head.find('a')
         self.story.setMetadata('authorId',a['href'].split('/')[2])
         self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
+        self.story.setMetadata('author',stripHTML(a))
 
         # Find the chapters:
         for chapter in soup.find('select').findAll('option'):
@@ -172,14 +173,14 @@ class FanFiktionDeAdapter(BaseSiteAdapter):
         tr=asoup.findAll('tr')
         for i in range(1,len(tr)):
             a = tr[i].find('a')
-            if a['href'] == '/s/'+self.story.getMetadata('storyId'):
+            if '/s/'+self.story.getMetadata('storyId')+'/1/' in a['href']:
                 break
         self.setDescription(url,a['onmouseover'].split("', '")[1])
         
         td = tr[i].findAll('td')
         self.story.addToList('category',stripHTML(td[1]))
-        self.story.setMetadata('rating', td[4].string)
-        self.story.setMetadata('numWords', td[5].string)
+        self.story.setMetadata('rating', stripHTML(td[4]))
+        self.story.setMetadata('numWords', stripHTML(td[5]))
  
             
     # grab the text for an individual chapter.

fanficdownloader/adapters/adapter_ficbooknet.py

@@ -52,7 +52,7 @@ class FicBookNetAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/readfic/'+self.story.getMetadata('storyId'))
@@ -69,8 +69,9 @@ class FicBookNetAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'www.ficbook.net'
 
+    @classmethod
     def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/readfic/12345"
+        return "http://"+self.getSiteDomain()+"/readfic/12345 http://"+self.getSiteDomain()+"/readfic/93626/246417#part_content"
 
     def getSiteURLPattern(self):
         return re.escape("http://"+self.getSiteDomain()+"/readfic/")+r"\d+"

fanficdownloader/adapters/adapter_fictionalleyorg.py

@@ -45,7 +45,7 @@ class FictionAlleyOrgSiteAdapter(BaseSiteAdapter):
         if m:
             self.story.setMetadata('authorId',m.group('auth'))
             self.story.setMetadata('storyId',m.group('id'))
-            logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+            
             # normalized story URL.
             self._setURL(url)
         else:
@@ -57,6 +57,7 @@ class FictionAlleyOrgSiteAdapter(BaseSiteAdapter):
     def getSiteDomain():
         return 'www.fictionalley.org'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/authors/drt/DA.html http://"+self.getSiteDomain()+"/authors/drt/JOTP01a.html"
 
@@ -110,7 +111,7 @@ class FictionAlleyOrgSiteAdapter(BaseSiteAdapter):
             url=self.url
             logger.debug("Normalizing to URL: "+url)
             ## title's right there...
-            self.story.setMetadata('title',storya.string)
+            self.story.setMetadata('title',stripHTML(storya))
             data = self._fetchUrl(url)
             soup = bs.BeautifulSoup(data)
             chapterlinklist = soup.findAll('a',{'class':'chapterlink'})
@@ -120,7 +121,7 @@ class FictionAlleyOrgSiteAdapter(BaseSiteAdapter):
             ## same way.
             chapsoup = bs.BeautifulSoup(chapterdata)
             storya = chapsoup.find('div',{'class':'breadcrumbs'}).findAll('a')[1]
-            self.story.setMetadata('title',storya.string)
+            self.story.setMetadata('title',stripHTML(storya))
             del chapsoup
 
         del chapterdata

fanficdownloader/adapters/adapter_fictionpadcom.py

@@ -0,0 +1,188 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+import time
+import json
+
+from .. import BeautifulSoup as bs
+#from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+class FictionPadSiteAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+        self.story.setMetadata('siteabbrev','fpad')
+        self.dateformat = "%Y-%m-%dT%H:%M:%SZ"
+        self.is_adult=False
+        self.username = None
+        self.password = None
+        # get storyId from url--url validation guarantees query correct
+        m = re.match(self.getSiteURLPattern(),url)
+        if m:
+            self.story.setMetadata('storyId',m.group('id'))
+            
+            # normalized story URL.
+            self._setURL("https://"+self.getSiteDomain()
+                         +"/author/"+m.group('author')
+                         +"/stories/"+self.story.getMetadata('storyId'))
+        else:
+            raise exceptions.InvalidStoryURL(url,
+                                             self.getSiteDomain(),
+                                             self.getSiteExampleURLs())
+
+    @staticmethod
+    def getSiteDomain():
+        return 'fictionpad.com'
+
+    @classmethod
+    def getSiteExampleURLs(self):
+        return "https://fictionpad.com/author/Author/stories/1234/Some-Title"
+    
+    def getSiteURLPattern(self):
+        # http://fictionpad.com/author/Serdd/stories/4275
+        return r"http(s)?://(www\.)?fictionpad\.com/author/(?P<author>[^/]+)/stories/(?P<id>\d+)"
+
+# <form method="post" action="/signin">
+#     <input name="authenticity_token" type="hidden" value="u+cfdXh46dRnwVnSlmE2B2BFmHgu760paqgBG6KQeos=" />
+#     <input type="hidden" name="remember" value="1">
+#     <strong class="help-start text-center">or with FictionPad</strong>
+#     <label class="control-label hidden-placeholder">Pseudonym or Email Address</label>
+#     <input name="login" class="input-block-level" type="text" placeholder="Pseudonym or Email Address" maxlength="50" required autofocus>
+#     <label class="control-label hidden-placeholder">Password</label>
+#     <input name="password" class="input-block-level" type="password" placeholder="Password" minlength="6" required>
+#     <button type="submit" class="btn btn-primary btn-block">Sign In</button>
+#     <p class="help-end">
+#         <a href="/passwordreset">Forgot your password?</a>
+#     </p>
+# </form>
+    def performLogin(self):
+        params = {}
+
+        if self.password:
+            params['login'] = self.username
+            params['password'] = self.password
+        else:
+            params['login'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['remember'] = '1'
+
+        loginUrl = 'http://' + self.getSiteDomain() + '/signin'
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['login']))
+
+        ## need to pull empty login page first to get authenticity_token
+        soup = bs.BeautifulSoup(self._fetchUrl(loginUrl))
+        params['authenticity_token']=soup.find('input', {'name':'authenticity_token'})['value']
+        
+        data = self._postUrl(loginUrl, params)
+
+        if "Invalid email/pseudonym and password combination." in data:
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['login']))
+            raise exceptions.FailedToLogin(loginUrl,params['login'])
+            
+
+    def extractChapterUrlsAndMetadata(self):
+        # fetch the chapter.  From that we will get almost all the
+        # metadata and chapter list
+
+        url=self.url
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+            if "This is a mature story.  Please sign in to read it." in data:
+                self.performLogin()
+                data = self._fetchUrl(url)
+            
+            find = "wordyarn.config.page = "
+            data = data[data.index(find)+len(find):]
+            data = data[:data.index("</script>")]
+            data = data[:data.rindex(";")]
+            data = data.replace('tables:','"tables":')
+            tables = json.loads(data)['tables']
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(url)
+            else:
+                raise e
+
+        # looks like only one author per story allowed.
+        author = tables['users'][0]
+        story = tables['stories'][0]
+        story_ver = tables['story_versions'][0]
+        
+        self.story.setMetadata('authorId',author['id'])
+        self.story.setMetadata('author',author['display_name'])
+        self.story.setMetadata('authorUrl','https://'+self.host+'/author/'+author['display_name']+'/stories')
+
+        self.story.setMetadata('title',story_ver['title'])
+        self.setDescription(url,story_ver['description'])
+        
+        if not ('assets/story_versions/covers' in story_ver['profile_image_url@2x']):
+            self.setCoverImage(url,story_ver['profile_image_url@2x'])
+
+        self.story.setMetadata('datePublished',makeDate(story['published_at'], self.dateformat))
+        self.story.setMetadata('dateUpdated',makeDate(story['published_at'], self.dateformat))
+
+        self.story.setMetadata('followers',story['followers_count'])
+        self.story.setMetadata('comments',story['comments_count'])
+        self.story.setMetadata('views',story['views_count'])
+        self.story.setMetadata('likes',int(story['likes'])) # no idea why they floated these.
+        self.story.setMetadata('dislikes',int(story['dislikes']))
+
+        if story_ver['is_complete']:
+            self.story.setMetadata('status', 'Completed')
+        else:
+            self.story.setMetadata('status', 'In-Progress')
+
+        self.story.setMetadata('rating', story_ver['maturity_level'])
+        self.story.setMetadata('numWords', unicode(story_ver['word_count']))
+
+        for i in tables['fandoms']:
+            self.story.addToList('category',i['name'])
+        
+        for i in tables['genres']:
+            self.story.addToList('genre',i['name'])
+        
+        for i in tables['characters']:
+            self.story.addToList('characters',i['name'])
+
+        for c in tables['chapters']:
+            chtitle = "Chapter %d"%c['number']
+            if c['title']:
+                chtitle += " - %s"%c['title']
+            self.chapterUrls.append((chtitle,c['body_url']))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+            
+    def getChapterText(self, url):
+        logger.debug('Getting chapter text from: %s' % url)
+        soup = bs.BeautifulSoup(self._fetchUrl(url))
+        return self.utf8FromSoup(url,soup)
+    
+def getClass():
+    return FictionPadSiteAdapter
+

fanficdownloader/adapters/adapter_fictionpresscom.py

@@ -39,8 +39,9 @@ class FictionPressComSiteAdapter(FanFictionNetSiteAdapter):
     def getAcceptDomains(cls):
         return ['www.fictionpress.com','m.fictionpress.com']
 
+    @classmethod
     def getSiteExampleURLs(self):
-        return "http://www.fictionpress.com/s/1234/1/ http://www.fictionpress.com/s/1234/12/ http://www.fictionpress.com/s/1234/1/Story_Title"
+        return "http://www.fictionpress.com/s/1234/1/ http://www.fictionpress.com/s/1234/12/ http://www.fictionpress.com/s/1234/1/Story_Title http://m.fictionpress.com/s/1234/1/"
 
     def getSiteURLPattern(self):
         return r"http://(www|m)?\.fictionpress\.com/s/\d+(/\d+)?(/|/[a-zA-Z0-9_-]+)?/?$"

fanficdownloader/adapters/adapter_ficwadcom.py

@@ -43,10 +43,11 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
 
     @staticmethod
     def getSiteDomain():
-        return 'www.ficwad.com'
+        return 'ficwad.com'
 
+    @classmethod
     def getSiteExampleURLs(self):
-        return "http://www.ficwad.com/story/137169"
+        return "http://ficwad.com/story/1234"
 
     def getSiteURLPattern(self):
         return re.escape(r"http://"+self.getSiteDomain())+"/story/\d+?$"
@@ -115,7 +116,7 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
 
         # title - first h4 tag will be title.
         titleh4 = soup.find('h4')
-        self.story.setMetadata('title', titleh4.a.string)
+        self.story.setMetadata('title', stripHTML(titleh4.a))
 
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"^/author/\d+"))

fanficdownloader/adapters/adapter_fimfictionnet.py

@@ -21,15 +21,12 @@ logger = logging.getLogger(__name__)
 import re
 import urllib2
 import cookielib as cl
-from datetime import datetime
 import json
 
 from .. import BeautifulSoup as bs
 from ..htmlcleanup import stripHTML
 from .. import exceptions as exceptions
 
-from ..bbcodeutils.bbcodeparser import bbcodeparser
-
 from base_adapter import BaseSiteAdapter,  makeDate
 
 def getClass():
@@ -44,6 +41,10 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
         self._setURL("http://"+self.getSiteDomain()+"/story/"+self.story.getMetadata('storyId')+"/")
         self.is_adult = False
         
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%d %b %Y"
+            
     @staticmethod
     def getSiteDomain():
         return 'www.fimfiction.net'
@@ -53,6 +54,7 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
         # mobile.fimifction.com isn't actually a valid domain, but we can still get the story id from URLs anyway
         return ['www.fimfiction.net','mobile.fimfiction.net', 'www.fimfiction.com', 'mobile.fimfiction.com']
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://www.fimfiction.net/story/1234/story-title-here http://www.fimfiction.net/story/1234/ http://www.fimfiction.com/story/1234/1/ http://mobile.fimfiction.net/story/1234/1/story-title-here/chapter-title-here"
 
@@ -81,8 +83,10 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
             apiResponse = urllib2.urlopen("http://www.fimfiction.net/api/story.php?story=%s" % (self.story.getMetadata("storyId"))).read()
             apiData = json.loads(apiResponse)
             
-            # Unfortunately, we still need to load the story index page to parse the characters
+            # Unfortunately, we still need to load the story index
+            # page to parse the characters.  And chapters, now, too.
             data = self._fetchUrl(self.url)
+            soup = bs.BeautifulSoup(data)
         except urllib2.HTTPError, e:
             if e.code == 404:
                 raise exceptions.StoryDoesNotExist(self.url)
@@ -92,11 +96,10 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
         if "Warning: mysql_fetch_array(): supplied argument is not a valid MySQL result resource" in data:
             raise exceptions.StoryDoesNotExist(self.url)
 
-        if "/images/missing_story.png" in data:
-            raise exceptions.StoryDoesNotExist(self.url)
-        
-        if "Invalid story id" in apiData.values():
-            raise exceptions.StoryDoesNotExist(self.url)
+        # Can cause problems if a missing story is referenced in a comment.
+        # Shouldn't be needed anyway.
+        # if "/images/missing_story.png" in data:
+        #     raise exceptions.StoryDoesNotExist(self.url)
         
         if "This story has been marked as having adult content." in data:
             raise exceptions.AdultCheckRequired(self.url)
@@ -112,24 +115,39 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
             else:
                 raise exceptions.FailedToLogin(self.url,"Story requires individual password",passwdonly=True)
          
+        if "Invalid story id" in apiData.values():
+            raise exceptions.StoryDoesNotExist(self.url)
+        
         storyMetadata = apiData["story"]    
             
-        self.story.setMetadata("title", storyMetadata["title"])
+        ## Title
+        a = soup.find('a', href=re.compile(r'^/story/'+self.story.getMetadata('storyId')))
+        self.story.setMetadata('title',stripHTML(a))
+        
+        # self.story.setMetadata("title", storyMetadata["title"])
+        # if not storyMetadata["title"]:
+        #     raise exceptions.FailedToDownload("%s doesn't have a title in the API.  This is a known fimfiction.net bug with titles containing ."%self.url)
+        
         self.story.setMetadata("author", storyMetadata["author"]["name"])
         self.story.setMetadata("authorId", storyMetadata["author"]["id"])
         self.story.setMetadata("authorUrl", "http://%s/user/%s" % (self.getSiteDomain(), storyMetadata["author"]["name"]))
+
+        # chapters = [{"chapterTitle": chapter["title"], "chapterURL": chapter["link"]} for chapter in storyMetadata["chapters"]]
         
-        chapters = [{"chapterTitle": chapter["title"], "chapterURL": chapter["link"]} for chapter in storyMetadata["chapters"]]
-        
-        ## this is bit of a kludge based on the assumption all the
-        ## 'bad' chapters will be at the end.
-        ## limit down to the number of chapters reported by chapter_count.
-        chapters = chapters[:storyMetadata["chapter_count"]] 
-        
-        for chapter in chapters:
-            self.chapterUrls.append((chapter["chapterTitle"], chapter["chapterURL"]))
-        self.story.setMetadata("numChapters", len(self.chapterUrls))
+        # ## this is bit of a kludge based on the assumption all the
+        # ## 'bad' chapters will be at the end.
+        # ## limit down to the number of chapters reported by chapter_count.
+        # chapters = chapters[:storyMetadata["chapter_count"]] 
         
+        # for chapter in chapters:
+        #     self.chapterUrls.append((chapter["chapterTitle"], chapter["chapterURL"]))
+        # self.story.setMetadata("numChapters", len(self.chapterUrls))
+
+        for chapter in soup.findAll('a',{'class':'chapter_link'}):
+            self.chapterUrls.append((stripHTML(chapter), 'http://'+self.host+chapter['href']))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+            
         # In the case of fimfiction.net, possible statuses are 'Completed', 'Incomplete', 'On Hiatus' and 'Cancelled'
         # For the sake of bringing it in line with the other adapters, 'Incomplete' becomes 'In-Progress'
         # and 'Complete' beomes 'Completed'. 'Cancelled' seems an important enough (not to mention more strictly true) 
@@ -139,7 +157,17 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
         status = storyMetadata["status"].replace("Incomplete", "In-Progress").replace("Complete", "Completed")
         self.story.setMetadata("status", status)
         self.story.setMetadata("rating", storyMetadata["content_rating_text"])
-            
+
+        ## Warnings aren't included in the API.
+        bottomli = soup.find('li',{'class':'bottom'})
+        if bottomli:
+            bottomspans = bottomli.findAll('span')
+            # the first span in bottom is the rating, obtained above.
+            if bottomspans and len(bottomspans) > 1:
+                for warning in bottomspans[1:]:
+                    self.story.addToList('warnings',warning.string)
+                
+        
         for category in storyMetadata["categories"]:
             if storyMetadata["categories"][category]:
                 self.story.addToList("genre", category) 
@@ -157,23 +185,52 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
 
             self.setCoverImage(self.url,coverurl)
 
-        # the fimfic API gives bbcode for desc, not html.
-        # btw, bbcode honors newlines, html doesn't.  change newlines to br tags.
-        self.setDescription(self.url,
-                            bbcodeparser().parse(storyMetadata["description"]).html(doDeepCopy=False).replace('\r','').replace('\n','<br />'))
-        
+        # fimf has started including extra stuff inside the description div.
+        descdivstr = "%s"%soup.find("div", {"class":"description"})
+        hrstr="<hr />"
+        descdivstr = '<div class="description">'+descdivstr[descdivstr.index(hrstr)+len(hrstr):]
+        self.setDescription(self.url,descdivstr)
+
+        # Can't trust dates from API anymore I'm told.
         # Dates are in Unix time
         # Take the publish date from the first chapter posted
-        rawDatePublished = storyMetadata["chapters"][0]["date_modified"]
-        self.story.setMetadata("datePublished", datetime.fromtimestamp(rawDatePublished))
-        rawDateUpdated = storyMetadata["date_modified"]
-        self.story.setMetadata("dateUpdated", datetime.fromtimestamp(rawDateUpdated))
+        # rawDatePublished = storyMetadata["chapters"][0]["date_modified"]
+        # self.story.setMetadata("datePublished", datetime.fromtimestamp(rawDatePublished))
+        # rawDateUpdated = storyMetadata["date_modified"]
+        # self.story.setMetadata("dateUpdated", datetime.fromtimestamp(rawDateUpdated))
+        
+        oldestChapter = None
+        newestChapter = None
+        self.newestChapterNum = None # save for comparing during update.
+        # Scan all chapters to find the oldest and newest, on
+        # FiMFiction it's possible for authors to insert new chapters
+        # out-of-order or change the dates of earlier ones by editing
+        # them--That WILL break epub update.
+        for index, chapterDate in enumerate(soup.findAll('span', {'class':'date'})):
+            date=re.sub(r"(\d+)(st|nd|rd|th)",r"\1",chapterDate.contents[1].strip())
+            chapterDate = makeDate(date,self.dateformat)
+            if oldestChapter == None or chapterDate < oldestChapter:
+                oldestChapter = chapterDate
+            if newestChapter == None or chapterDate > newestChapter:
+                newestChapter = chapterDate
+                self.newestChapterNum = index
 
-        soup = bs.BeautifulSoup(data).find("div", {"class":"story"})
+        self.story.setMetadata("dateUpdated", newestChapter)
+        
+        pubdatetag = soup.find('span', {'class':'date_approved'})
+        if pubdatetag is None:
+            self.story.setMetadata("datePublished", oldestChapter)            
+        else:
+            pubdateraw = pubdatetag('span')[1].text
+            datestripped=re.sub(r"(\d+)(st|nd|rd|th)",r"\1",pubdateraw.strip())
+            pubDate = makeDate(datestripped,self.dateformat)
+            self.story.setMetadata("datePublished", pubDate)
+            
+        chars = soup.find("div", {"class":"inner_data"})
         # fimfic stopped putting the char name on or around the char
         # icon now for some reason.  Pull it from the image name with
         # some heuristics.
-        for character in [character_icon["src"] for character_icon in soup.findAll("img", {"class":"character_icon"})]:
+        for character in [character_icon["src"] for character_icon in chars.findAll("img", {"class":"character_icon"})]:
             # //static.fimfiction.net/images/characters/twilight_sparkle.png
             # 5th split /, remove last four, replace _, capitolize every word(title())
             char = character.split('/')[5][:-4].replace('_',' ').title()
@@ -191,11 +248,23 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
                 if not isinstance(value,basestring):
                     value = unicode(value)
                 self.story.setMetadata(metakey, value)
-        
+
+        rawGroupList = soup.find('ul', {'id':'story_group_list'})
+        if rawGroupList is not None:
+            for groupName in rawGroupList.findAll('a', {'href':re.compile('^/group/')}):
+                self.story.addToList("groups",stripHTML(groupName))
             
+    def hookForUpdates(self,chaptercount):
+        if self.oldchapters and len(self.oldchapters) > self.newestChapterNum:
+            print("Existing epub has %s chapters\nNewest chapter is %s.  Discarding old chapters from there on."%(len(self.oldchapters), self.newestChapterNum+1))
+            self.oldchapters = self.oldchapters[:self.newestChapterNum]
+        return len(self.oldchapters)
+            
+        
     def getChapterText(self, url):
         logger.debug('Getting chapter text from: %s' % url)
-        soup = bs.BeautifulSoup(self._fetchUrl(url),selfClosingTags=('br','hr')).find('div', {'id' : 'chapter_container'})
+        
+        soup = bs.BeautifulSoup(self._fetchUrl(url),selfClosingTags=('br','hr')).find('div', {'class' : 'chapter_content'})
         if soup == None:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
         return self.utf8FromSoup(url,soup)

fanficdownloader/adapters/adapter_finestoriescom.py

@@ -50,7 +50,7 @@ class FineStoriesComAdapter(BaseSiteAdapter):
         self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2].split(':')[0])
         if 'storyInfo' in self.story.getMetadata('storyId'):
             self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/s/storyInfo.php?id='+self.story.getMetadata('storyId'))
@@ -67,8 +67,9 @@ class FineStoriesComAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'finestories.com'
 
+    @classmethod
     def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/s/10537 http://"+self.getSiteDomain()+"/s/10537:4010 http://"+self.getSiteDomain()+"/library/storyInfo.php?id=10537"
+        return "http://"+self.getSiteDomain()+"/s/1234 http://"+self.getSiteDomain()+"/s/1234:4010 http://"+self.getSiteDomain()+"/library/storyInfo.php?id=1234"
 
     def getSiteURLPattern(self):
         return re.escape("http://"+self.getSiteDomain())+r"/(s|library)?/(storyInfo.php\?id=)?\d+(:\d+)?(;\d+)?$"
@@ -141,7 +142,7 @@ class FineStoriesComAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('a', href=re.compile(r'/s/'+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.text)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"/a/\w+"))
@@ -187,6 +188,7 @@ class FineStoriesComAdapter(BaseSiteAdapter):
             a = lc4.find('a', href=re.compile(r"/library/show_series.php\?id=\d+"))
             i = a.parent.text.split('(')[1].split(')')[0]
             self.setSeries(a.text, i)
+            self.story.setMetadata('seriesUrl','http://'+self.host+a['href'])
         except:
             pass
         try:

fanficdownloader/adapters/adapter_grangerenchantedcom.py

@@ -48,7 +48,7 @@ class GrangerEnchantedCom(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         self.section=self.parsedUrl.path.split('/',)[1]
              
         # normalized story URL.
@@ -74,6 +74,7 @@ class GrangerEnchantedCom(BaseSiteAdapter):
     def getAcceptDomains(cls):
         return ['grangerenchanted.com','malfoymanor.grangerenchanted.com']
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://grangerenchanted.com/enchant/viewstory.php?sid=1234 http://malfoymanor.grangerenchanted.com/themanor/viewstory.php?sid=1234"
 
@@ -181,7 +182,7 @@ class GrangerEnchantedCom(BaseSiteAdapter):
         
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -227,6 +228,9 @@ class GrangerEnchantedCom(BaseSiteAdapter):
             if 'Word count' in label:
                 self.story.setMetadata('numWords', value)
 
+            if 'Read' in label:
+                self.story.setMetadata('read', value)
+
             if 'Categories' in label:
                 cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
                 for cat in cats:
@@ -275,9 +279,17 @@ class GrangerEnchantedCom(BaseSiteAdapter):
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
-            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+
+        try:
+            self.story.setMetadata('reviews',
+                                   stripHTML(soup.find('div',{'id':'sort'}).
+                                             findAll('a', href=re.compile(r'^reviews.php'))[1]))
         except:
             # I find it hard to care if the series parsing fails
             pass

fanficdownloader/adapters/adapter_harrypotterfanfictioncom.py

@@ -42,7 +42,7 @@ class HarryPotterFanFictionComSiteAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only psid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?psid='+self.story.getMetadata('storyId'))
@@ -56,8 +56,9 @@ class HarryPotterFanFictionComSiteAdapter(BaseSiteAdapter):
     def getAcceptDomains(cls):
         return ['www.harrypotterfanfiction.com','harrypotterfanfiction.com']
 
+    @classmethod
     def getSiteExampleURLs(self):
-        return "http://www.harrypotterfanfiction.com/viewstory.php?psid=1234 http://harrypotterfanfiction.com/viewstory.php?psid=5678"
+        return "http://www.harrypotterfanfiction.com/viewstory.php?psid=1234"
 
     def getSiteURLPattern(self):
         return re.escape("http://")+r"(www\.)?"+re.escape("harrypotterfanfiction.com/viewstory.php?psid=")+r"\d+$"
@@ -91,7 +92,7 @@ class HarryPotterFanFictionComSiteAdapter(BaseSiteAdapter):
 
         ## Title
         a = soup.find('a', href=re.compile(r'\?psid='+self.story.getMetadata('storyId')))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         ## javascript:if (confirm('Please note. This story may contain adult themes. By clicking here you are stating that you are over 17. Click cancel if you do not meet this requirement.')) location = '?psid=290995'
         if "This story may contain adult themes." in a['href'] and not (self.is_adult or self.getConfig("is_adult")):
             raise exceptions.AdultCheckRequired(self.url)
@@ -130,55 +131,55 @@ class HarryPotterFanFictionComSiteAdapter(BaseSiteAdapter):
 
         ## Finding the metadata is a bit of a pain.  Most of the meta
         ## data is in a center.table without a bgcolor.
-        for center in soup.findAll('center'):
-            table = center.find('table',{'bgcolor':None})
-            if table:
-                metastr = stripHTML(str(table)).replace('\n',' ').replace('\t',' ')
-                # Rating: 12+ Story Reviews: 3
-                # Chapters: 3
-                # Characters: Andromeda, Ted, Bellatrix, R. Lestrange, Lucius, Narcissa, OC
-                # Genre(s): Fluff, Romance, Young Adult Era: OtherPairings: Other Pairing, Lucius/Narcissa
-                # Status: Completed
-                # First Published: 2010.09.02
-                # Last Published Chapter: 2010.09.28
-                # Last Updated: 2010.09.28
-                # Favorite Story Of: 1 users
-                # Warnings: Scenes of a Mild Sexual Nature
+        #for center in soup.findAll('center'):
+        table = soup.find('table',{'class':'storymaininfo'})
+        if table:
+            metastr = stripHTML(str(table)).replace('\n',' ').replace('\t',' ')
+            # Rating: 12+ Story Reviews: 3
+            # Chapters: 3
+            # Characters: Andromeda, Ted, Bellatrix, R. Lestrange, Lucius, Narcissa, OC
+            # Genre(s): Fluff, Romance, Young Adult Era: OtherPairings: Other Pairing, Lucius/Narcissa
+            # Status: Completed
+            # First Published: 2010.09.02
+            # Last Published Chapter: 2010.09.28
+            # Last Updated: 2010.09.28
+            # Favorite Story Of: 1 users
+            # Warnings: Scenes of a Mild Sexual Nature
 
-                m = re.match(r".*?Status: Completed.*?",metastr)
-                if m:
-                    self.story.setMetadata('status','Completed')
-                else:
-                    self.story.setMetadata('status','In-Progress')
+            m = re.match(r".*?Status: Completed.*?",metastr)
+            if m:
+                self.story.setMetadata('status','Completed')
+            else:
+                self.story.setMetadata('status','In-Progress')
 
-                m = re.match(r".*?Rating: (.+?) Story Reviews.*?",metastr)
-                if m:
-                    self.story.setMetadata('rating', m.group(1))
+            m = re.match(r".*?Rating: (.+?) Story Reviews.*?",metastr)
+            if m:
+                self.story.setMetadata('rating', m.group(1))
 
-                m = re.match(r".*?Genre\(s\): (.+?) Era.*?",metastr)
-                if m:
-                    for g in m.group(1).split(','):
-                        self.story.addToList('genre',g)
-        
-                m = re.match(r".*?Characters: (.+?) Genre.*?",metastr)
-                if m:
-                    for g in m.group(1).split(','):
-                        self.story.addToList('characters',g)
-        
-                m = re.match(r".*?Warnings: (.+).*?",metastr)
-                if m:
-                    for w in m.group(1).split(','):
-                        if w != 'Now Warnings':
-                            self.story.addToList('warnings',w)
-        
-                m = re.match(r".*?First Published: ([0-9\.]+).*?",metastr)
-                if m:
-                    self.story.setMetadata('datePublished',makeDate(m.group(1), "%Y.%m.%d"))
+            m = re.match(r".*?Genre\(s\): (.+?) Era.*?",metastr)
+            if m:
+                for g in m.group(1).split(','):
+                    self.story.addToList('genre',g)
+    
+            m = re.match(r".*?Characters: (.+?) Genre.*?",metastr)
+            if m:
+                for g in m.group(1).split(','):
+                    self.story.addToList('characters',g)
+    
+            m = re.match(r".*?Warnings: (.+).*?",metastr)
+            if m:
+                for w in m.group(1).split(','):
+                    if w != 'Now Warnings':
+                        self.story.addToList('warnings',w)
+    
+            m = re.match(r".*?First Published: ([0-9\.]+).*?",metastr)
+            if m:
+                self.story.setMetadata('datePublished',makeDate(m.group(1), "%Y.%m.%d"))
 
-                # Updated can have more than one space after it. <shrug>
-                m = re.match(r".*?Last Updated: ([0-9\.]+).*?",metastr)
-                if m:
-                    self.story.setMetadata('dateUpdated',makeDate(m.group(1), "%Y.%m.%d"))
+            # Updated can have more than one space after it. <shrug>
+            m = re.match(r".*?Last Updated: ([0-9\.]+).*?",metastr)
+            if m:
+                self.story.setMetadata('dateUpdated',makeDate(m.group(1), "%Y.%m.%d"))
 
     def getChapterText(self, url):
 

fanficdownloader/adapters/adapter_hennethannunnet.py

@@ -0,0 +1,172 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return HennethAnnunNetAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class HennethAnnunNetAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/stories/chapter.cfm?stid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','htan')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%m/%d/%y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.henneth-annun.net'
+
+    @classmethod
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/stories/chapter.cfm?stid=1234"
+
+    def getSiteURLPattern(self):
+        return "http://"+self.getSiteDomain()+"/stories/chapter(_view)?.cfm\?stid="+r"\d+$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+            
+        if "We're sorry. This story is not available." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: This story is not available.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('h2', {'id':'page_heading'})
+        self.story.setMetadata('title',stripHTML(a))
+
+        # Find the chapters: chapter_view.cfm?stid=6663&spordinal=1" 
+        for chapter in soup.findAll('a', href=re.compile(r'chapter_view.cfm\?stid='+self.story.getMetadata('storyId')+"&spordinal=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/stories/'+chapter['href']))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+        
+        self.story.setMetadata('numWords', soup.find('tr', {'class':'foot'}).findAll('td')[1].text)
+
+        self.setDescription(url,soup.find('div', {'id':'summary'}))
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        info = soup.find('div', {'id':'storyinformation'})
+        labels=info.findAll('b')
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+                
+            if 'Completion' in label:
+                if 'Complete' in value.string:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Rating' in label:
+                self.story.setMetadata('rating', value.string)
+
+            if 'Era:' in label:
+                self.story.addToList('category',value.string)
+
+            if 'Genre' in label:
+                self.story.addToList('genre',value.string)
+                
+        labels=info.findAll('strong')
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+            
+            if 'Author' in label:
+                value=value.nextSibling
+                self.story.setMetadata('authorId',value['href'].split('=')[1])
+                self.story.setMetadata('authorUrl','http://'+self.host+'/'+value['href'])
+                self.story.setMetadata('author',value.string)
+                
+            if 'Post' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated:' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+                
+        for char in soup.findAll('a', href=re.compile(r"/resources/bios_view.cfm\?scid=\d+")):
+            self.story.addToList('characters',stripHTML(char))
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'class' : 'block chapter'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_hlfictionnet.py

@@ -48,7 +48,7 @@ class HLFictionNetAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -65,6 +65,7 @@ class HLFictionNetAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'hlfiction.net'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 
@@ -105,7 +106,7 @@ class HLFictionNetAdapter(BaseSiteAdapter):
         self.story.setMetadata('author',aut.string)
         aut.extract()
         
-        self.story.setMetadata('title',a.string[:(len(a.string)-3)])
+        self.story.setMetadata('title',stripHTML(a)[:(len(a.string)-3)])
 
         # Find the chapters:
         chapters=soup.find('select')
@@ -206,6 +207,7 @@ class HLFictionNetAdapter(BaseSiteAdapter):
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if ('viewstory.php?sid='+self.story.getMetadata('storyId')) in a['href']:
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             

fanficdownloader/adapters/adapter_hpfandomnet.py

@@ -52,7 +52,7 @@ class HPFandomNetAdapterAdapter(BaseSiteAdapter): # XXX
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         # XXX Most sites don't have the /eff part.  Replace all to remove it usually.
@@ -70,6 +70,7 @@ class HPFandomNetAdapterAdapter(BaseSiteAdapter): # XXX
         # The site domain.  Does have www here, if it uses it.
         return 'www.hpfandom.net' # XXX
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/eff/viewstory.php?sid=1234"
 

fanficdownloader/adapters/adapter_hpfanficarchivecom.py

@@ -48,7 +48,7 @@ class HPFanficArchiveComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/stories/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -65,6 +65,7 @@ class HPFanficArchiveComAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'www.hpfanficarchive.com'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/stories/viewstory.php?sid=1234"
 
@@ -99,7 +100,7 @@ class HPFanficArchiveComAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -193,6 +194,7 @@ class HPFanficArchiveComAdapter(BaseSiteAdapter):
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             

fanficdownloader/adapters/adapter_iketernalnet.py

@@ -48,7 +48,7 @@ class IkEternalNetAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -65,6 +65,7 @@ class IkEternalNetAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'www.ik-eternal.net'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 
@@ -182,7 +183,7 @@ class IkEternalNetAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))

fanficdownloader/adapters/adapter_imagineeficcom.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-# Copyright 2012 Fanficdownloader team
+# Copyright 2013 Fanficdownloader team
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -28,11 +28,11 @@ from .. import exceptions as exceptions
 from base_adapter import BaseSiteAdapter,  makeDate
 
 def getClass():
-    return SVUFictionComAdapter
+    return ImagineEFicComAdapter
 
 # Class name has to be unique.  Our convention is camel case the
 # sitename with Adapter at the end.  www is skipped.
-class SVUFictionComAdapter(BaseSiteAdapter):
+class ImagineEFicComAdapter(BaseSiteAdapter):
 
     def __init__(self, config, url):
         BaseSiteAdapter.__init__(self, config, url)
@@ -48,23 +48,24 @@ class SVUFictionComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
         
         # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','svuf')
+        self.story.setMetadata('siteabbrev','ime')
 
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%b %d, %Y"
+        self.dateformat = "%Y.%m.%d"
             
     @staticmethod # must be @staticmethod, don't remove it.
     def getSiteDomain():
         # The site domain.  Does have www here, if it uses it.
-        return 'svufiction.com'
+        return 'imagine.e-fic.com'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 
@@ -108,13 +109,13 @@ class SVUFictionComAdapter(BaseSiteAdapter):
 
     ## Getting the chapter list and the meta data, plus 'is adult' checking.
     def extractChapterUrlsAndMetadata(self):
-    
+
         if self.is_adult or self.getConfig("is_adult"):
             # Weirdly, different sites use different warning numbers.
             # If the title search below fails, there's a good chance
             # you need a different number.  print data at that point
             # and see what the 'click here to continue' url says.
-            addurl = "&ageconsent=ok&warning=6"
+            addurl = "&ageconsent=ok&warning=4"
         else:
             addurl=""
 
@@ -130,12 +131,12 @@ class SVUFictionComAdapter(BaseSiteAdapter):
                 raise exceptions.StoryDoesNotExist(self.url)
             else:
                 raise e
-                
+
         if self.needToLoginCheck(data):
             # need to log in for this one.
             self.performLogin(url)
             data = self._fetchUrl(url)
-                
+
         m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
         if m != None:
             if self.is_adult or self.getConfig("is_adult"):
@@ -168,22 +169,15 @@ class SVUFictionComAdapter(BaseSiteAdapter):
         # Now go hunting for all the meta data and the chapter list.
         
         ## Title
-        pt = soup.find('div', {'class' : 'title'})
-        a = pt.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        if a.text != "":
-            self.story.setMetadata('title',a.text)
-        else:
-            self.story.setMetadata('title','Banner')
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
-        a = pt.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
         self.story.setMetadata('authorId',a['href'].split('=')[1])
         self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
         self.story.setMetadata('author',a.string)
 
-        rating=pt.text.split('[')[1].split(']')[0]
-        self.story.setMetadata('rating', rating)
-
         # Find the chapters:
         for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
             # just in case there's tags, like <i> in chapter titles.
@@ -200,39 +194,61 @@ class SVUFictionComAdapter(BaseSiteAdapter):
                 return d[k]
             except:
                 return ""
+                
 
-        bottom=soup.find('div', {'class' : 'bottom'}).text
-        self.story.setMetadata('numWords', bottom.split('Word count: ')[1].split(' Read:')[0])
-        self.story.setMetadata('datePublished', makeDate(bottom.split('Published: ')[1].split(' Updated:')[0], self.dateformat))
-        self.story.setMetadata('dateUpdated', makeDate(bottom.split('Updated: ')[1], self.dateformat))
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
 
-        
-        content=soup.find('div', {'class' : 'content'})
-        value=content.find('h1').nextSibling
-        svalue = ""
-        while not defaultGetattr(value,'class') == 'smaller':
-            svalue += str(value)
-            value = value.nextSibling
-        self.setDescription(url,svalue)
-        
-        status = content.text.split('Completed: ')[1]
-        if 'Yes' in status:
-            self.story.setMetadata('status', 'Completed')
-        else:
-            self.story.setMetadata('status', 'In-Progress')
-        
-        cats = content.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-        for cat in cats:
-            self.story.addToList('category',cat.string)
-            
-        chars = content.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-        for char in chars:
-            self.story.addToList('characters',char.string)
-            
-        genres = content.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
-        for genre in genres:
-            self.story.addToList('genre',genre.string)
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
             
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
         try:
             # Find Series name from series URL.
             a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
@@ -249,6 +265,7 @@ class SVUFictionComAdapter(BaseSiteAdapter):
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             

fanficdownloader/adapters/adapter_indeathnet.py

@@ -46,12 +46,12 @@ class InDeathNetAdapter(BaseSiteAdapter):
         self.password = ""
         self.is_adult=False
         
-		
-		  # get storyId from url--url validation guarantees query correct
+        
+        # get storyId from url--url validation guarantees query correct
         m = re.match(self.getSiteURLPattern(),url)
         if m:
             self.story.setMetadata('storyId',m.group('id'))
-            logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+            
             # normalized story URL.
             self._setURL('http://www.' + self.getSiteDomain() + '/blog/archive/'+self.story.getMetadata('storyId')+'-'+m.group('name')+'/')
         else:
@@ -72,6 +72,7 @@ class InDeathNetAdapter(BaseSiteAdapter):
         return 'indeath.net'
 
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/blog/archive/123-story-in-death/"
 
@@ -86,6 +87,34 @@ class InDeathNetAdapter(BaseSiteAdapter):
         postdate = makeDate(d.group('day')+' '+ym.group('mon')+' '+ym.group('year'),self.dateformat)
         return postdate
 
+    def getAuthorData(self):
+        
+        mainUrl = self.url.replace("/archive","")
+        
+        try:
+            maindata = self._fetchUrl(mainUrl)
+            
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.meta)
+            else:
+                raise e
+
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        mainsoup = bs.BeautifulSoup(maindata)
+
+        # find first entry
+        e = mainsoup.find('div',{'class':"entry"})
+        
+        # get post author as author
+        d = e.find('div',{'class':"desc"})
+        a = d.find('strong')
+        self.story.setMetadata('author',a.contents[0].string.strip())
+        
+        # Don't seem to be able to get author pages anymore
+        self.story.setMetadata('authorUrl','http://www.indeath.net/')
+        self.story.setMetadata('authorId','0')
+        
     ## Getting the chapter list and the meta data, plus 'is adult' checking.
     def extractChapterUrlsAndMetadata(self):
 
@@ -108,21 +137,14 @@ class InDeathNetAdapter(BaseSiteAdapter):
         ## Title
         h = soup.find('a', id="blog_title")
         t = h.find('span')
-        self.story.setMetadata('title',t.contents[0].string.strip())
-		
+        self.story.setMetadata('title',stripHTML(t.contents[0]).strip())
+        
         s = t.find('div')        
         if s != None:
             self.setDescription(url,s)
-		
-        # Find authorid and URL from first link in Recent Entries (don't yet reference 'recent entries' - let's see if that is required)
-        a = soup.find('a', href=re.compile(r"http://www.indeath.net/user/\d+\-[a-z0-9]+/$"))		#http://www.indeath.net/user/9083-cyrex/
-        m = re.search('http://www.indeath.net/user/(?P<id>\d+)\-(?P<name>[a-z0-9]*)/$',a['href'])
-        self.story.setMetadata('authorId',m.group('id'))
-        self.story.setMetadata('authorUrl',a['href'])
-        self.story.setMetadata('author',m.group('name'))
-
-
 
+        # Get Author from main blog page since it's not reliably on the archive page
+        self.getAuthorData()
 
         # Find the chapters:
         chapters=soup.findAll('a', title="View entry", href=re.compile(r'http://www.indeath.net/blog/'+self.story.getMetadata('storyId')+"/entry\-(\d+)\-([^/]*)/$"))
@@ -151,21 +173,20 @@ class InDeathNetAdapter(BaseSiteAdapter):
             if len(chapters)==1:
                 self.chapterUrls.append((self.story.getMetadata('title'),chapter['href']))
             else:
-            	ct = stripHTML(chapter)
-            	tnew = re.match("(?i)"+self.story.getMetadata('title')+r" - (?P<newtitle>.*)$",ct)
-            	if tnew:
-            		chaptertitle = tnew.group('newtitle')
-            	else:
-            		chaptertitle = ct
+                ct = stripHTML(chapter)
+                tnew = re.match("(?i)"+self.story.getMetadata('title')+r" - (?P<newtitle>.*)$",ct)
+                if tnew:
+                    chaptertitle = tnew.group('newtitle')
+                else:
+                    chaptertitle = ct
                 self.chapterUrls.append((chaptertitle,chapter['href']))
-	
-		
+
         
         
     # grab the text for an individual chapter.
     def getChapterText(self, url):
         logger.debug('Getting chapter text from: %s' % url)
-		
+        
         #chapter=bs.BeautifulSoup('<div class="story"></div>')
         data = self._fetchUrl(url)
         soup = bs.BeautifulSoup(data,selfClosingTags=('br','hr','span','center'))

fanficdownloader/adapters/adapter_ksarchivecom.py

@@ -52,7 +52,7 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
@@ -74,6 +74,7 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
         # The site domain.  Does have www here, if it uses it.
         return 'ksarchive.com' # XXX
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 
@@ -285,6 +286,7 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             

fanficdownloader/adapters/adapter_libraryofmoriacom.py

@@ -49,7 +49,7 @@ class LibraryOfMoriaComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/a/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -66,6 +66,7 @@ class LibraryOfMoriaComAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'www.libraryofmoria.com'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/a/viewstory.php?sid=1234"
 
@@ -128,12 +129,12 @@ class LibraryOfMoriaComAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
         self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/a/'+a['href'])
         self.story.setMetadata('author',a.string)
 
         # Find the chapters:
@@ -225,6 +226,7 @@ class LibraryOfMoriaComAdapter(BaseSiteAdapter):
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             

fanficdownloader/adapters/adapter_lumossycophanthexcom.py

@@ -48,7 +48,7 @@ class LumosSycophantHexComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
 		
         
         # normalized story URL.
@@ -66,6 +66,7 @@ class LumosSycophantHexComAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'lumos.sycophanthex.com'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 
@@ -115,7 +116,7 @@ class LumosSycophantHexComAdapter(BaseSiteAdapter):
         ## Title
         pt = soup.find('div', {'id' : 'pagetitle'})
         a = pt.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.text)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = pt.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -212,6 +213,7 @@ class LumosSycophantHexComAdapter(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             

fanficdownloader/adapters/adapter_mediaminerorg.py

@@ -43,7 +43,7 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
         m = re.match(self.getSiteURLPattern(),url)
         if m:
             self.story.setMetadata('storyId',m.group('id'))
-            logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+            
             # normalized story URL.
             self._setURL('http://' + self.getSiteDomain() + '/fanfic/view_st.php/'+self.story.getMetadata('storyId'))
         else:
@@ -55,6 +55,7 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
     def getSiteDomain():
         return 'www.mediaminer.org'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/fanfic/view_st.php/123456 http://"+self.getSiteDomain()+"/fanfic/view_ch.php/1234123/123444#fic_c"
 
@@ -106,10 +107,10 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
         for font in title.findAll('font'):
             font.extract() # removes 'font' tags from inside the td.        
         if title.has_key('colspan'):
-            titlet = title.text
+            titlet = stripHTML(title)
         else:
             ## No colspan, it's part chapter title--even if it's a one-shot.
-            titlet = ':'.join(title.text.split(':')[:-1]) # strip trailing 'Chapter X' or chapter title
+            titlet = ':'.join(stripHTML(title).split(':')[:-1]) # strip trailing 'Chapter X' or chapter title
         self.story.setMetadata('title',titlet)
         ## The story title is difficult to reliably parse from the
         ## story pages.  Getting it from the author page is, but costs

fanficdownloader/adapters/adapter_merlinficdtwinscouk.py

@@ -48,7 +48,7 @@ class MerlinFicDtwinsCoUk(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -65,6 +65,7 @@ class MerlinFicDtwinsCoUk(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'merlinfic.dtwins.co.uk'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 
@@ -169,7 +170,7 @@ class MerlinFicDtwinsCoUk(BaseSiteAdapter):
         
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -268,6 +269,7 @@ class MerlinFicDtwinsCoUk(BaseSiteAdapter):
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             

fanficdownloader/adapters/adapter_midnightwhispersca.py

@@ -52,7 +52,7 @@ class MidnightwhispersCaAdapter(BaseSiteAdapter): # XXX
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
@@ -70,6 +70,7 @@ class MidnightwhispersCaAdapter(BaseSiteAdapter): # XXX
         # The site domain.  Does have www here, if it uses it.
         return 'www.midnightwhispers.ca' # XXX
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 
@@ -260,6 +261,7 @@ class MidnightwhispersCaAdapter(BaseSiteAdapter): # XXX
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             

fanficdownloader/adapters/adapter_mugglenetcom.py

@@ -52,7 +52,7 @@ class MuggleNetComAdapter(BaseSiteAdapter): # XXX
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -66,14 +66,19 @@ class MuggleNetComAdapter(BaseSiteAdapter): # XXX
             
     @staticmethod # must be @staticmethod, don't remove it.
     def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'fanfiction.mugglenet.com' # XXX
+        # The site domain.
+        return 'fanfiction.mugglenet.com'
 
+    @classmethod
+    def getAcceptDomains(cls):
+        return ['fanfiction.mugglenet.com','fanfic.mugglenet.com']
+
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+        return re.escape("http://")+r"fanfic(tion)?\.mugglenet\.com"+re.escape("/viewstory.php?sid=")+r"\d+$"
 
     ## Login seems to be reasonably standard across eFiction sites.
     def needToLoginCheck(self, data):
@@ -188,7 +193,7 @@ class MuggleNetComAdapter(BaseSiteAdapter): # XXX
         
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -306,6 +311,7 @@ class MuggleNetComAdapter(BaseSiteAdapter): # XXX
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             

fanficdownloader/adapters/adapter_nationallibrarynet.py

@@ -48,7 +48,7 @@ class NationalLibraryNetAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only storyid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?storyid='+self.story.getMetadata('storyId'))
@@ -68,8 +68,10 @@ class NationalLibraryNetAdapter(BaseSiteAdapter):
     def getAcceptDomains(cls):
         return ['www.national-library.net','national-library.net']
 
+    @classmethod
     def getSiteExampleURLs(self):
-        return "ONLY the stories archived on or after June 17, 2006 and that are hosted on the website: http://"+self.getSiteDomain()+"/viewstory.php?storyid=1234"
+        # ONLY the stories archived on or after June 17, 2006 and that are hosted on the website: 
+        return "http://"+self.getSiteDomain()+"/viewstory.php?storyid=1234"
 
     def getSiteURLPattern(self):
         return re.escape("http://")+"(www\.)?"+re.escape(self.getSiteDomain()+"/viewstory.php?storyid=")+r"\d+$"
@@ -102,7 +104,7 @@ class NationalLibraryNetAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('h1')
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"authorresults.php\?author=\d+"))
@@ -163,6 +165,7 @@ class NationalLibraryNetAdapter(BaseSiteAdapter):
             
             if 'Series' in label:
                 self.setSeries(stripHTML(value.nextSibling), value.nextSibling.nextSibling.string[2:])
+                self.story.setMetadata('seriesUrl','http://'+self.host+'/'+value.nextSibling['href'])
                 
         asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
         story=asoup.find('a', href=re.compile(r'viewstory.php\?storyid='+self.story.getMetadata('storyId')))

fanficdownloader/adapters/adapter_ncisficcom.py

@@ -48,7 +48,7 @@ class NCISFicComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only storyid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?storyid='+self.story.getMetadata('storyId'))
@@ -68,6 +68,7 @@ class NCISFicComAdapter(BaseSiteAdapter):
     def getAcceptDomains(cls):
         return ['www.ncisfic.com','ncisfic.com']
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?storyid=1234"
 
@@ -102,7 +103,7 @@ class NCISFicComAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('h1')
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"authorresults.php\?author=\d+"))
@@ -164,6 +165,7 @@ class NCISFicComAdapter(BaseSiteAdapter):
             if 'Series' in label:
                 if "No Series" not in value.nextSibling.string:
                     self.setSeries(stripHTML(value.nextSibling), value.nextSibling.nextSibling.string[2:])
+                    self.story.setMetadata('seriesUrl','http://'+self.host+'/'+value.nextSibling['href'])
                 
         asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
         story=asoup.find('a', href=re.compile(r'viewstory.php\?storyid='+self.story.getMetadata('storyId')))

fanficdownloader/adapters/adapter_ncisfictionnet.py

@@ -37,11 +37,6 @@ class NCISFictionNetAdapter(BaseSiteAdapter):
     def __init__(self, config, url):
         BaseSiteAdapter.__init__(self, config, url)
 
-        # ncisfiction.net blocks IPs the default user-agent.  However,
-        # when asked, they said it was just general anti-spam, not
-        # targeted at us.  That lets me do this in good conscience:
-        self.opener.addheaders = [('User-agent', 'FFDL/1.6')]
-        
         self.decode = ["iso-8859-1",
                        "Windows-1252"] # 1252 is a superset of iso-8859-1.
                                # Most sites that claim to be
@@ -53,7 +48,7 @@ class NCISFictionNetAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL("http://"+self.getSiteDomain()\
@@ -79,6 +74,7 @@ class NCISFictionNetAdapter(BaseSiteAdapter):
     def getAcceptDomains(cls):
         return ['www.ncisfiction.net','www.ncisfiction.com']
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/story.php?stid=01234 http://"+self.getSiteDomain()+"/chapters.php?stid=1234"
 
@@ -193,7 +189,7 @@ class NCISFictionNetAdapter(BaseSiteAdapter):
             series_name = a.find('a').string
             i = a.text.split('#')[1]
             self.setSeries(series_name, i)
-            
+            self.story.setMetadata('seriesUrl','http://'+self.host+'/'+a.find('a')['href'])
         except:
             # I find it hard to care if the series parsing fails
             pass

fanficdownloader/adapters/adapter_netraptororg.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-# Copyright 2012 Fanficdownloader team
+# Copyright 2013 Fanficdownloader team
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -28,11 +28,9 @@ from .. import exceptions as exceptions
 from base_adapter import BaseSiteAdapter,  makeDate
 
 def getClass():
-    return PrisonBreakFicNetAdapter
+    return NetRaptorOrgAdapter
 
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class PrisonBreakFicNetAdapter(BaseSiteAdapter):
+class NetRaptorOrgAdapter(BaseSiteAdapter):
 
     def __init__(self, config, url):
         BaseSiteAdapter.__init__(self, config, url)
@@ -48,34 +46,33 @@ class PrisonBreakFicNetAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        self._setURL('http://' + self.getSiteDomain() + '/fanfiction/viewstory.php?sid='+self.story.getMetadata('storyId'))
         
         # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','pbf')
+        self.story.setMetadata('siteabbrev','netrap')
 
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%B %d, %Y"
+        self.dateformat = "%d/%m/%y"
             
     @staticmethod # must be @staticmethod, don't remove it.
     def getSiteDomain():
         # The site domain.  Does have www here, if it uses it.
-        return 'www.prisonbreakfic.net'
+        return 'netraptor.org'
 
+    @classmethod
     def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+        return "http://"+self.getSiteDomain()+"/fanfiction/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+        return re.escape("http://"+self.getSiteDomain()+"/fanfiction/viewstory.php?sid=")+r"\d+$"
 
     ## Getting the chapter list and the meta data, plus 'is adult' checking.
     def extractChapterUrlsAndMetadata(self):
 
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
         url = self.url+'&index=1'
         logger.debug("URL: "+url)
 
@@ -86,6 +83,10 @@ class PrisonBreakFicNetAdapter(BaseSiteAdapter):
                 raise exceptions.StoryDoesNotExist(self.url)
             else:
                 raise e
+
+
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
             
         # use BeautifulSoup HTML parser to make everything easier to find.
         soup = bs.BeautifulSoup(data)
@@ -94,19 +95,20 @@ class PrisonBreakFicNetAdapter(BaseSiteAdapter):
         # Now go hunting for all the meta data and the chapter list.
         
         ## Title
-        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        pagetitle = soup.find('div',{'id':'pagetitle'})
+        a = pagetitle.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        a = pagetitle.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
         self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/fanfiction/'+a['href'])
         self.story.setMetadata('author',a.string)
 
         # Find the chapters:
         for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
             # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']))
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/fanfiction/'+chapter['href']))
 
         self.story.setMetadata('numChapters',len(self.chapterUrls))
 
@@ -119,10 +121,7 @@ class PrisonBreakFicNetAdapter(BaseSiteAdapter):
                 return d[k]
             except:
                 return ""
-                
-# summary, rated, word count, categories, characters, genre, warnings, completed, published, updated, seires
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
+				
         labels = soup.findAll('span',{'class':'label'})
         for labelspan in labels:
             value = labelspan.nextSibling
@@ -145,21 +144,18 @@ class PrisonBreakFicNetAdapter(BaseSiteAdapter):
 
             if 'Categories' in label:
                 cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-                for cat in cats:
+                catstext = [cat.string for cat in cats]
+                for cat in catstext:
                     self.story.addToList('category',cat.string)
 
             if 'Characters' in label:
                 chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-                for char in chars:
+                charstext = [char.string for char in chars]
+                for char in charstext:
                     self.story.addToList('characters',char.string)
 
-            if 'Pairing' in label:
-                ships = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=3')) # XXX
-                for ship in ships:
-                    self.story.addToList('ships',ship.string.replace(" and ","/"))
-
             if 'Genre' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) # XXX
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
                 for genre in genres:
                     self.story.addToList('genre',genre.string)
 
@@ -184,7 +180,7 @@ class PrisonBreakFicNetAdapter(BaseSiteAdapter):
             # Find Series name from series URL.
             a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
             series_name = a.string
-            series_url = 'http://'+self.host+'/'+a['href']
+            series_url = 'http://'+self.host+'/fanfiction/'+a['href']
 
             # use BeautifulSoup HTML parser to make everything easier to find.
             seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
@@ -193,6 +189,7 @@ class PrisonBreakFicNetAdapter(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             
@@ -205,8 +202,7 @@ class PrisonBreakFicNetAdapter(BaseSiteAdapter):
 
         logger.debug('Getting chapter text from: %s' % url)
 
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        soup = bs.BeautifulSoup(self._fetchUrl(url))
         
         div = soup.find('div', {'id' : 'story'})
 

fanficdownloader/adapters/adapter_nfacommunitycom.py

@@ -52,7 +52,7 @@ class NfaCommunityComAdapter(BaseSiteAdapter): # XXX
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
@@ -74,6 +74,7 @@ class NfaCommunityComAdapter(BaseSiteAdapter): # XXX
         # The site domain.  Does have www here, if it uses it.
         return 'nfacommunity.com' # XXX
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 
@@ -155,7 +156,7 @@ class NfaCommunityComAdapter(BaseSiteAdapter): # XXX
         
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -264,6 +265,7 @@ class NfaCommunityComAdapter(BaseSiteAdapter): # XXX
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             

fanficdownloader/adapters/adapter_nhamagicalworldsus.py

@@ -48,7 +48,7 @@ class NHAMagicalWorldsUsAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
 		
         
         # normalized story URL.
@@ -66,6 +66,7 @@ class NHAMagicalWorldsUsAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'nha.magical-worlds.us'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 
@@ -112,8 +113,9 @@ class NHAMagicalWorldsUsAdapter(BaseSiteAdapter):
 		
         for info in asoup.findAll('table', {'width' : '100%', 'bordercolor' : re.compile(r'#')}):
             a = info.find('a')
-            if ('viewstory.php?sid='+self.story.getMetadata('storyId')) in a['href']:
-                self.story.setMetadata('title',a.string)
+            if 'viewstory.php?sid='+self.story.getMetadata('storyId') == a['href'] or \
+                    ('viewstory.php?sid='+self.story.getMetadata('storyId')+'&') in a['href']:
+                self.story.setMetadata('title',stripHTML(a))
                 break
 		
 
@@ -142,14 +144,14 @@ class NHAMagicalWorldsUsAdapter(BaseSiteAdapter):
         for cat in cats:
             self.story.addToList('category',cat.string)
 					
-        a = info.find('a', href=re.compile(r'reviews.php\?sid='+self.story.getMetadata('storyId')))
+        a = info.find('a', href=re.compile(r'viewuser.php'))
         val = a.nextSibling
         svalue = ""
         while not defaultGetattr(val) == 'br':
             val = val.nextSibling
         val = val.nextSibling
         while not defaultGetattr(val) == 'br':
-            svalue += str(val)
+            svalue += unicode(val)
             val = val.nextSibling
         self.setDescription(url,svalue)
         

fanficdownloader/adapters/adapter_nickandgregnet.py

@@ -0,0 +1,175 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return NickAndGregNetAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class NickAndGregNetAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        
+        
+        # normalized story URL.
+        # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
+        self._setURL('http://' + self.getSiteDomain() + '/desert_archive/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','nag')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%Y/%m/%d"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.nickandgreg.net'
+
+    @classmethod
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/desert_archive/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/desert_archive/viewstory.php?sid=")+r"\d+$"
+    
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&i=1'
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',stripHTML(a))
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/desert_archive/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        chapters = soup.find('select')
+        for chapter in chapters.findAll('option'):
+            if chapter.text != 'Story Index' and chapter.text != 'Chapters':
+                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/desert_archive/'+chapter['value']))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+        
+        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
+        
+        for div in asoup.findAll('td', {'class' : 'tblborder6'}):
+            a = div.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+            if a != None:
+                break
+        
+        self.setDescription(url,div.find('br').nextSibling)
+        
+        a=div.text.split('Rating:')
+        if len(a) == 2: self.story.setMetadata('rating', a[1].split(' -')[0])
+        
+        a=div.text.split('Characters:')
+        if len(a) == 2: 
+        	for char in a[1].split(' -')[0].split(', '):
+        	    self.story.addToList('characters',char)
+        	    
+        a=div.text.split('Genres:')
+        if len(a) == 2: 
+        	for genre in a[1].split(' -')[0].split(', '):
+        	    self.story.addToList('genre',genre)
+        	    
+        a=div.text.split('Warnings:')
+        if len(a) == 2: 
+        	for warn in a[1].split(' -')[0].split(', '):
+        	    if 'none' not in warn:
+        	        self.story.addToList('warnings',warn)
+        	    
+        a=div.text.split('Completed:')
+        if len(a) ==2:
+            if 'Yes' in a[1]:
+                self.story.setMetadata('status', 'Completed')
+            else:
+                self.story.setMetadata('status', 'In-Progress')
+                
+        a=div.text.split('Published:')
+        if len(a) == 2: self.story.setMetadata('datePublished', makeDate(stripHTML(a[1].split(' -')[0]), self.dateformat))
+        
+        a=div.text.split('Updated:')
+        if len(a) == 2: self.story.setMetadata('dateUpdated', makeDate(stripHTML(a[1].split(' -')[0]), self.dateformat))
+
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('table', {'class' : 'tblborder6'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_occlumencysycophanthexcom.py

@@ -48,7 +48,7 @@ class OcclumencySycophantHexComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
 		
         
         # normalized story URL.
@@ -66,6 +66,7 @@ class OcclumencySycophantHexComAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'occlumency.sycophanthex.com'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 
@@ -157,7 +158,7 @@ class OcclumencySycophantHexComAdapter(BaseSiteAdapter):
         for info in asoup.findAll('table', {'class' : 'border'}):
             a = info.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
             if a != None:
-                self.story.setMetadata('title',a.text)
+                self.story.setMetadata('title',stripHTML(a))
                 break
 		
 

fanficdownloader/adapters/adapter_onedirectionfanfictioncom.py

@@ -48,7 +48,7 @@ class OneDirectionFanfictionComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -70,6 +70,7 @@ class OneDirectionFanfictionComAdapter(BaseSiteAdapter):
         return ['www.onedirectionfanfiction.com','onedirectionfanfiction.com']
 
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 
@@ -157,7 +158,7 @@ class OneDirectionFanfictionComAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -244,6 +245,7 @@ class OneDirectionFanfictionComAdapter(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             

fanficdownloader/adapters/adapter_phoenixsongnet.py

@@ -48,7 +48,7 @@ class PhoenixSongNetAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[3])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/fanfiction/story/' +self.story.getMetadata('storyId')+'/')
@@ -65,6 +65,7 @@ class PhoenixSongNetAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'www.phoenixsong.net'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/fanfiction/story/1234/"
 
@@ -112,6 +113,8 @@ class PhoenixSongNetAdapter(BaseSiteAdapter):
         logger.debug("URL: "+url)
 
         try:
+            if self.getConfig('force_login'):
+                self.performLogin(url)
             data = self._fetchUrl(url)
         except urllib2.HTTPError, e:
             if e.code == 404:
@@ -123,7 +126,6 @@ class PhoenixSongNetAdapter(BaseSiteAdapter):
             # need to log in for this one.
             self.performLogin(url)
             data = self._fetchUrl(url)
-			
             
         # use BeautifulSoup HTML parser to make everything easier to find.
         soup = bs.BeautifulSoup(data)
@@ -134,7 +136,7 @@ class PhoenixSongNetAdapter(BaseSiteAdapter):
         ## Title
         b = soup.find('div', {'id' : 'nav25'})
         a = b.find('a', href=re.compile(r'fanfiction/story/'+self.story.getMetadata('storyId')+"/$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.  /fanfiction/stories.php?psid=125
         a = b.find('a', href=re.compile(r"/fanfiction/stories.php\?psid=\d+"))

fanficdownloader/adapters/adapter_pommedesangcom.py

@@ -0,0 +1,300 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return PommeDeSangComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class PommeDeSangComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        
+        
+        # pommedesang.com has two 'sections', shown in URL as
+        # 'efiction' and 'sds' that change how things should be
+        # handled.
+        # http://pommedesang.com/efiction/viewstory.php?sid=1234
+        # http://pommedesang.com/sds/viewstory.php?sid=1234
+        self.section=self.parsedUrl.path.split('/',)[1]
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/'+self.section+'/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','pmds')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        if 'efiction' in self.section:
+            self.dateformat = "%b %d, %Y"
+        else:
+            self.dateformat = "%m/%d/%y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'pommedesang.com'
+
+    @classmethod
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/efiction/viewstory.php?sid=1234 http://"+self.getSiteDomain()+"/sds/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return r"http://"+self.getSiteDomain()+"/(efiction|sds)?/viewstory.php\?sid=\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Registered Users Only' in data \
+                or 'There is no such account on our website' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['cookiecheck'] = '1'
+        params['submit'] = 'Submit'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/'+self.section+'/user.php?action=login'
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "Member Account" not in d : #Member Account
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=5"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid & error in url.
+                addurl = addurl.replace("&","&")
+                url = self.url+'&index=1'+addurl
+                logger.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile('viewstory.php\?sid=\d+'))
+        self.story.setMetadata('title',stripHTML(a))
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+self.section+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+                
+# summary, rated, word count, categories, characters, genre, warnings, completed, published, updated, seires
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) # XXX
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+self.section+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
+            storyas = seriessoup.findAll('a', href=re.compile('viewstory.php\?sid=\d+'))
+            i=1
+            for a in storyas:
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if ('viewstory.php?sid='+self.story.getMetadata('storyId')) in a['href']:
+                        self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
+                        break
+                    i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_ponyfictionarchivenet.py

@@ -48,8 +48,7 @@ class PonyFictionArchiveNetAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-        logger.info(self.parsedUrl.netloc)
+        
         # normalized story URL.
         if "explicit" in self.parsedUrl.netloc:
             self._setURL('http://explicit.' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -70,8 +69,9 @@ class PonyFictionArchiveNetAdapter(BaseSiteAdapter):
     def getAcceptDomains(cls):
         return ['www.ponyfictionarchive.net','ponyfictionarchive.net','explicit.ponyfictionarchive.net']
 
+    @classmethod
     def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234 http://explicit."+self.getSiteDomain()+"/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
         return re.escape("http://")+"(www\.|explicit\.)?"+re.escape(self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
@@ -136,7 +136,7 @@ class PonyFictionArchiveNetAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -225,6 +225,7 @@ class PonyFictionArchiveNetAdapter(BaseSiteAdapter):
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             

fanficdownloader/adapters/adapter_portkeyorg.py

@@ -55,7 +55,7 @@ class PortkeyOrgAdapter(BaseSiteAdapter): # XXX
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/story/'+self.story.getMetadata('storyId'))
@@ -72,6 +72,7 @@ class PortkeyOrgAdapter(BaseSiteAdapter): # XXX
         # The site domain.  Does have www here, if it uses it.
         return 'fanfiction.portkey.org' # XXX
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/story/1234"
 
@@ -250,8 +251,10 @@ class PortkeyOrgAdapter(BaseSiteAdapter): # XXX
 
         logger.debug('Getting chapter text from: %s' % url)
         data = self._fetchUrl(url)
-        soup = bs.BeautifulStoneSoup(data,
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+
+        data = data.replace("HTML>","div>")
+        
+        soup = bs.BeautifulSoup(data)
 
         #print("soup:%s"%soup)
         tag = soup.find('td', {'class' : 'story'})

fanficdownloader/adapters/adapter_potionsandsnitchesnet.py

@@ -41,7 +41,7 @@ class PotionsAndSnitchesNetSiteAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/fanfiction/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -55,8 +55,9 @@ class PotionsAndSnitchesNetSiteAdapter(BaseSiteAdapter):
     def getAcceptDomains(cls):
         return ['www.potionsandsnitches.net','potionsandsnitches.net']
 
+    @classmethod
     def getSiteExampleURLs(self):
-        return "http://www.potionsandsnitches.net/fanfiction/viewstory.php?sid=1234 http://potionsandsnitches.net/fanfiction/viewstory.php?sid=5678"
+        return "http://www.potionsandsnitches.net/fanfiction/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
         return re.escape("http://")+r"(www\.)?"+re.escape("potionsandsnitches.net/fanfiction/viewstory.php?sid=")+r"\d+$"
@@ -82,7 +83,7 @@ class PotionsAndSnitchesNetSiteAdapter(BaseSiteAdapter):
 
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -182,6 +183,7 @@ class PotionsAndSnitchesNetSiteAdapter(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             

fanficdownloader/adapters/adapter_potterficscom.py

@@ -54,7 +54,7 @@ class PotterFicsComAdapter(BaseSiteAdapter):
         m = re.match(self.getSiteURLPattern(),url)
         if m:
             self.story.setMetadata('storyId',m.group('id'))
-            logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+            
             # normalized story URL. gets rid of chapter if there, left with chapter index URL
             nurl = "http://"+self.getSiteDomain()+"/historias/"+self.story.getMetadata('storyId')
             self._setURL(nurl)
@@ -62,7 +62,7 @@ class PotterFicsComAdapter(BaseSiteAdapter):
             raise exceptions.InvalidStoryURL(url,
                                              self.getSiteDomain(),
                                              self.getSiteExampleURLs())
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','potficscom')
@@ -72,13 +72,9 @@ class PotterFicsComAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'www.potterfics.com'
 
+    @classmethod
     def getSiteExampleURLs(self):
-        return \
-                "http://www.potterfics.com/historias/127583 "\
-                "http://www.potterfics.com/historias/127583/capitulo-1 "\
-                "http://www.potterfics.com/historias/127583/capitulo-4 "\
-                "http://www.potterfics.com/historias/92810 "\
-                "http://www.potterfics.com/historias/111194"
+        return "http://www.potterfics.com/historias/12345 http://www.potterfics.com/historias/12345/capitulo-1 "
 
     def getSiteURLPattern(self):
         #http://www.potterfics.com/historias/127583
@@ -151,7 +147,7 @@ class PotterFicsComAdapter(BaseSiteAdapter):
         #find first metadata block
         mb = cell.div.findNextSibling('div')
         #Get meta...
-        self.story.setMetadata('title', mb.b.string)
+        self.story.setMetadata('title', stripHTML(mb.b))
         #strip out brackets on rating
         self.story.setMetadata('rating', mb.span.string[1:-1])
         #Completion status is denoted by the presence of this image:

fanficdownloader/adapters/adapter_potterheadsanonymouscom.py

@@ -0,0 +1,298 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return PotterHeadsAnonymousComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class PotterHeadsAnonymousComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','pha')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%d %b %Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'fanfic.potterheadsanonymous.com'
+
+    @classmethod
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Registered Users Only' in data \
+                or 'There is no such account on our website' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['cookiecheck'] = '1'
+        params['submit'] = 'Submit'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "Member Account" not in d : #Member Account
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=4"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+            
+        # Since the warning text can change by warning level, let's
+        # look for the warning pass url.  ksarchive uses
+        # &warning= -- actually, so do other sites.  Must be an
+        # eFiction book.
+            
+        # viewstory.php?sid=1882&warning=4
+        # viewstory.php?sid=1654&ageconsent=ok&warning=5
+        #print data
+        #m = re.search(r"'viewstory.php\?sid=1882(&warning=4)'",data)
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid & error in url.
+                addurl = addurl.replace("&","&")
+                url = self.url+'&index=1'+addurl
+                logger.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+
+        pagetitle = soup.find('div',{'id':'pagetitle'})
+        
+        ## Title
+        a = pagetitle.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',stripHTML(a))
+        
+        # Find authorid and URL from... author url.
+        a = pagetitle.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=3'))
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_pretendercentrecom.py

@@ -48,7 +48,7 @@ class PretenderCenterComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/missingpieces/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -69,6 +69,7 @@ class PretenderCenterComAdapter(BaseSiteAdapter):
     def getAcceptDomains(cls):
         return ['www.pretendercentre.com','pretendercentre.com']
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/missingpieces/viewstory.php?sid=1234"
 
@@ -133,7 +134,7 @@ class PretenderCenterComAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.text)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -228,6 +229,7 @@ class PretenderCenterComAdapter(BaseSiteAdapter):
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             

fanficdownloader/adapters/adapter_psychficcom.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-# Copyright 2011 Fanficdownloader team
+# Copyright 2013 Fanficdownloader team
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -27,12 +27,12 @@ from .. import exceptions as exceptions
 
 from base_adapter import BaseSiteAdapter,  makeDate
 
-
 def getClass():
-    return JLAUnlimitedComAdapter
+    return PsychFicComAdapter
 
-
-class JLAUnlimitedComAdapter(BaseSiteAdapter):
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class PsychFicComAdapter(BaseSiteAdapter): 
 
     def __init__(self, config, url):
         BaseSiteAdapter.__init__(self, config, url)
@@ -42,36 +42,35 @@ class JLAUnlimitedComAdapter(BaseSiteAdapter):
                                # Most sites that claim to be
                                # iso-8859-1 (and some that claim to be
                                # utf8) are really windows-1252.
-        self.username = "" # if left empty, site doesn't return any message at all.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
         self.password = ""
         self.is_adult=False
-
+        
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-
-        self._setURL('http://' + self.getSiteDomain() + '/eFiction1.1/viewstory.php?sid='+self.story.getMetadata('storyId'))
-
+        
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
         # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','jla')
+        self.story.setMetadata('siteabbrev','psyf')
 
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%m/%d/%y"
-
+        self.dateformat = "%B %d, %Y"
+            
     @staticmethod # must be @staticmethod, don't remove it.
     def getSiteDomain():
         # The site domain.  Does have www here, if it uses it.
-        return 'www.jlaunlimited.com'
+        return 'www.psychfic.com'
 
+    @classmethod
     def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/eFiction1.1/viewstory.php?sid=1234"
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/eFiction1.1/viewstory.php?sid=")+r"\d+$"
-
-
-
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
 
     ## Getting the chapter list and the meta data, plus 'is adult' checking.
     def extractChapterUrlsAndMetadata(self):
@@ -81,7 +80,7 @@ class JLAUnlimitedComAdapter(BaseSiteAdapter):
             # If the title search below fails, there's a good chance
             # you need a different number.  print data at that point
             # and see what the 'click here to continue' url says.
-            addurl = "&ageconsent=ok&warning=4" # XXX
+            addurl = "&ageconsent=ok&warning=4"
         else:
             addurl=""
 
@@ -98,8 +97,7 @@ class JLAUnlimitedComAdapter(BaseSiteAdapter):
             else:
                 raise e
 
-        # Assume that if there is a url with 'warning=#' in the page then it is a 'check' page
-        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)",data)
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
         if m != None:
             if self.is_adult or self.getConfig("is_adult"):
                 # We tried the default and still got a warning, so
@@ -120,16 +118,19 @@ class JLAUnlimitedComAdapter(BaseSiteAdapter):
                         raise e    
             else:
                 raise exceptions.AdultCheckRequired(self.url)
-                
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
         # use BeautifulSoup HTML parser to make everything easier to find.
         soup = bs.BeautifulSoup(data)
         # print data
 
         # Now go hunting for all the meta data and the chapter list.
-
+        
         ## Title
-        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')))
-        self.story.setMetadata('title',a.string)
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -138,9 +139,9 @@ class JLAUnlimitedComAdapter(BaseSiteAdapter):
         self.story.setMetadata('author',a.string)
 
         # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+")):
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
             # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/eFiction1.1/'+chapter['href']+addurl))
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
 
         self.story.setMetadata('numChapters',len(self.chapterUrls))
 
@@ -158,7 +159,7 @@ class JLAUnlimitedComAdapter(BaseSiteAdapter):
         labels = soup.findAll('span',{'class':'label'})
         for labelspan in labels:
             value = labelspan.nextSibling
-            label = labelspan.string
+            label = labelspan.text
 
             if 'Summary' in label:
                 ## Everything until the next span class='label'
@@ -177,34 +178,22 @@ class JLAUnlimitedComAdapter(BaseSiteAdapter):
 
             if 'Categories' in label:
                 cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-                catstext = [cat.string for cat in cats]
-                for cat in catstext:
+                for cat in cats:
                     self.story.addToList('category',cat.string)
 
             if 'Characters' in label:
                 chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-                charstext = [char.string for char in chars]
-                for char in charstext:
+                for char in chars:
                     self.story.addToList('characters',char.string)
 
-            ## Not all sites use Genre, but there's no harm to
-            ## leaving it in.  Check to make sure the type_id number
-            ## is correct, though--it's site specific.
             if 'Genre' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) # XXX
-                genrestext = [genre.string for genre in genres]
-                self.genre = ', '.join(genrestext)
-                for genre in genrestext:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+                for genre in genres:
                     self.story.addToList('genre',genre.string)
 
-            ## Not all sites use Warnings, but there's no harm to
-            ## leaving it in.  Check to make sure the type_id number
-            ## is correct, though--it's site specific.
             if 'Warnings' in label:
-                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
-                warningstext = [warning.string for warning in warnings]
-                self.warning = ', '.join(warningstext)
-                for warning in warningstext:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
+                for warning in warnings:
                     self.story.addToList('warnings',warning.string)
 
             if 'Completed' in label:
@@ -217,43 +206,43 @@ class JLAUnlimitedComAdapter(BaseSiteAdapter):
                 self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
             
             if 'Updated' in label:
-                # there's a stray [ at the end.
-                #value = value[0:-1]
                 self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
 
         try:
             # Find Series name from series URL.
             a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
             series_name = a.string
-            series_url = 'http://'+self.host+'/fanfic/'+a['href']
+            series_url = 'http://'+self.host+'/'+a['href']
 
             # use BeautifulSoup HTML parser to make everything easier to find.
             seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
             i=1
             for a in storyas:
-                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                    self.setSeries(series_name, i)
-                    break
-                i+=1
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
+                        break
+                    i+=1
             
         except:
             # I find it hard to care if the series parsing fails
             pass
-
-
+            
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
         logger.debug('Getting chapter text from: %s' % url)
 
-        soup = bs.BeautifulSoup(self._fetchUrl(url),
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-
-        div = soup.find('div', {'id' : 'story'})
         
+        div = soup.find('div', {'id' : 'story'})
+
         if None == div:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-
+    
         return self.utf8FromSoup(url,div)
-

fanficdownloader/adapters/adapter_qafficcom.py

@@ -48,7 +48,7 @@ class QafFicComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/atp/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -65,6 +65,7 @@ class QafFicComAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'www.qaf-fic.com'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/atp/viewstory.php?sid=1234"
 
@@ -137,7 +138,7 @@ class QafFicComAdapter(BaseSiteAdapter):
         self.story.setMetadata('author',aut.string)
         aut.extract()
         
-        self.story.setMetadata('title',a.string[:(len(a.string)-3)])
+        self.story.setMetadata('title',stripHTML(a)[:(len(a.string)-3)])
 
         # Find the chapters:
         chapters=soup.find('select')
@@ -236,6 +237,7 @@ class QafFicComAdapter(BaseSiteAdapter):
                             name=seriessoup.find('div', {'id' : 'pagetitle'})
                             name.find('a').extract()
                             self.setSeries(name.text.split(' by[')[0], i)
+                            self.story.setMetadata('seriesUrl',series_url)
                             i=0
                             break
                         i+=1

fanficdownloader/adapters/adapter_restrictedsectionorg.py

@@ -0,0 +1,265 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+import cookielib as cl
+from datetime import datetime
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return RestrictedSectionOrgSiteAdapter
+
+class RestrictedSectionOrgSiteAdapter(BaseSiteAdapter):
+    
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+            
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        
+        # normalized story URL.
+        # get story/file and storyId from url--url validation guarantees query correct
+        m = re.match(self.getSiteURLPattern(),url)
+        if m:
+            self.story.setMetadata('storyId',m.group('id'))
+            
+            # normalized story URL.
+            self._setURL('http://' + self.getSiteDomain() + '/' + m.group('filestory') + '.php?' + m.group('filestory') + '=' + self.story.getMetadata('storyId'))
+        else:
+            raise exceptions.InvalidStoryURL(url,
+                                             self.getSiteDomain(),
+                                             self.getSiteExampleURLs())
+        
+        self.story.setMetadata('siteabbrev','ressec')
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%d %b %Y" # 20 Nov 2005
+        
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        return 'www.restrictedsection.org'
+
+    @classmethod
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/story.php?story=1234 http://"+self.getSiteDomain()+"/file.php?file=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain())+r"/(?P<filestory>file|story).php\?(file|story)=(?P<id>\d+)$"
+        
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        url = self.url
+        logger.debug("URL: "+url)
+
+        # one-shot stories use file url instead of story.  'Luckily',
+        # we don't have to worry about one-shots becoming
+        # multi-chapter because ressec is frozen.  Still need 'story'
+        # url for metadata, however.
+        try:
+            if 'file' in url:
+                data = self._postUrlUP(url)
+                soup = bs.BeautifulSoup(data)
+                storya = soup.find('a',href=re.compile(r"^story.php\?story=\d+"))
+                url = 'http://'+self.host+'/'+storya['href'].split('&')[0]  # strip rs_session
+
+                fileas = soup.find('a',href=re.compile(r"^file.php\?file=\d+"))
+                if fileas:
+                    for filea in fileas:
+                        if 'Previous Chapter' in filea.string or 'Next Chapter' in filea.string:
+                            raise exceptions.FailedToDownload(self.getSiteDomain() +" Cannot use chapter url with multi-chapter stories on this site.")
+                    
+            logger.debug("metadata URL: "+url)
+            data = self._fetchUrl(url)
+            # print data
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+        if "Story not found" in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Story not found.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+
+        # check user/pass on a chapter for multi-chapter
+        if 'file' not in self.url:
+            self._postUrlUP('http://'+self.host+'/'+soup.find('a', href=re.compile(r"^file.php\?file=\d+"))['href'])
+
+        ## Title
+        h2 = soup.find('h2')
+
+        # Find authorid and URL from... author url.
+        a = h2.find('a')
+        ahref = a['href'].split('&')[0]  # strip rs_session
+        
+        self.story.setMetadata('authorId',ahref.split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+ahref)
+        self.story.setMetadata('author',stripHTML(a))
+
+        # title, remove byauthorname.
+        auth=stripHTML(a)
+        title=stripHTML(h2)
+        self.story.setMetadata('title',title[:title.index(" by "+auth)])
+
+        dates = soup.findAll('span', {'class':'date'})
+        if dates: # only for multi-chapter
+            self.story.setMetadata('datePublished', makeDate(stripHTML(dates[0]), self.dateformat))
+            self.story.setMetadata('dateUpdated', makeDate(stripHTML(dates[-1]), self.dateformat))
+
+        words = soup.findAll('span', {'class':'size'})
+        wordcount=0
+        for w in words:
+            wordcount = wordcount + int(w.string[:-6].replace(',',''))
+
+        self.story.setMetadata('numWords',"%s"%wordcount)
+
+        self.story.setMetadata('rating', soup.find('a',href=re.compile(r"^rating.php\?rating=\d+")).string)
+        
+        # other tags
+
+        labels = soup.find('table', {'class':'info'}).findAll('th')
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+            
+            if label != None:
+                
+                if 'Categories' in label:
+                    for g in stripHTML(value).split('\n'):
+                        self.story.addToList('genre',g)
+                
+                if 'Pairings' in label:
+                    for g in stripHTML(value).split('\n'):
+                        self.story.addToList('ships',g)
+                
+                if 'Summary' in label:
+                    self.setDescription(url,stripHTML(value).replace("\n"," ").replace("\r",""))
+                    value.extract() # remove summary incase it contains file URLs.
+
+                if 'Updated' in label: # one-shots only.
+                    print "value:%s"%value
+                    value.find('sup').extract() # remove 'st', 'nd', 'th' ordinals
+                    print "value:%s"%value
+                    date = makeDate(stripHTML(value), '%d %B %Y') # full month name
+                    self.story.setMetadata('datePublished', date)
+
+                if 'Length' in label: # one-shots only.
+                    self.story.setMetadata('numWords',value.string[:-6])
+                    
+        # one-shot.
+        if 'file' in self.url:
+            self.chapterUrls.append((self.story.getMetadata('title'),self.url))
+        else: # multi-chapter
+            # Find the chapters: 'library_storyview.php?chapterid=3
+            chapters=soup.findAll('a', href=re.compile(r"^file.php\?file=\d+"))
+            if len(chapters)==0:
+                raise exceptions.FailedToDownload(self.getSiteDomain() +" says: No chapters found.")
+            else:
+                for chapter in chapters:
+                    chhref = chapter['href'].split('&')[0] # strip rs_session
+                    # just in case there's tags, like <i> in chapter titles.
+                    self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chhref))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+
+
+    def _postUrlUP(self, url):
+        params = {}
+        if self.password:
+            params['username'] = self.username
+            params['password'] = self.password
+        else:
+            params['username'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['accept.x'] = 1
+        params['accept.y'] = 1
+
+        excpt=None
+        for sleeptime in [0.5, 1.5, 4, 9]:
+            time.sleep(sleeptime)	
+            try:
+                data = self._postUrl(url, params)
+                if data == "Unable to connect to the database":
+                    raise exceptions.FailedToDownload("Site reported 'Unable to connect to the database'")
+                if "I certify that I am over the age of 18 and that accessing the following story will not violate the laws of my country or local ordinances." in data:
+                    raise exceptions.FailedToLogin(url,params['username'])
+                return data
+            except exceptions.FailedToLogin, ftl:
+                # no need to retry these.
+                raise(ftl)
+            except Exception, e:
+                excpt=e
+                logger.warn("Caught an exception reading URL: %s  Exception %s."%(unicode(url),unicode(e)))
+
+        logger.error("Giving up on %s" %url)
+        logger.exception(excpt)
+        raise(excpt)
+        
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        data = self._postUrlUP(url)
+        #print("data:%s"%data)
+        
+        # some stories have html that confuses the parser.  For story
+        # text we don't care about anything before '<table id="page"'
+        # and seems to clear the issue.
+        data = data[data.index('<table id="page"'):]
+        
+        soup = bs.BeautifulSoup(data)
+        
+        div = soup.find('td',{'id':'page_content'})
+        div.name='div'
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+
+        ## Remove stuff from page_content
+
+        # Remove all tags before the first <hr> after class=info table (including hr)
+        hr = div.find('table',{'class':'info'}).findNext('hr')
+        for tag in hr.findAllPrevious():
+            tag.extract()
+        hr.extract()
+        
+        # Remove all tags after the last <hr> (including hr)
+        hr = div.findAll('hr')[-1]
+        for tag in hr.findAllNext():
+            tag.extract()
+        hr.extract()
+        
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_samdeanarchivenu.py

@@ -48,7 +48,7 @@ class SamDeanArchiveNuAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -69,6 +69,7 @@ class SamDeanArchiveNuAdapter(BaseSiteAdapter):
     def getAcceptDomains(cls):
         return ['www.samdean.archive.nu','samdean.archive.nu']
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 
@@ -109,7 +110,7 @@ class SamDeanArchiveNuAdapter(BaseSiteAdapter):
         self.story.setMetadata('author',aut.string)
         aut.extract()
         
-        self.story.setMetadata('title',a.string[:(len(a.string)-3)])
+        self.story.setMetadata('title',stripHTML(a)[:(len(stripHTML(a))-3)])
 
         # Find the chapters:
         chapters=soup.find('select')
@@ -207,6 +208,7 @@ class SamDeanArchiveNuAdapter(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             

fanficdownloader/adapters/adapter_scarheadnet.py

@@ -0,0 +1,299 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return ScarHeadNetAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class ScarHeadNetAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','shn')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%d/%m/%y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'scarhead.net'
+
+    @classmethod
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Registered Users Only' in data \
+                or 'There is no such account on our website' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['cookiecheck'] = '1'
+        params['submit'] = 'Submit'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "Member Account" not in d : #Member Account
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=5"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+            
+        # Since the warning text can change by warning level, let's
+        # look for the warning pass url.  ksarchive uses
+        # &warning= -- actually, so do other sites.  Must be an
+        # eFiction book.
+            
+        # viewstory.php?sid=1882&warning=4
+        # viewstory.php?sid=1654&ageconsent=ok&warning=5
+        #print data
+        #m = re.search(r"'viewstory.php\?sid=1882(&warning=4)'",data)
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid & error in url.
+                addurl = addurl.replace("&","&")
+                url = self.url+'&index=1'+addurl
+                logger.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+
+        pagetitle = soup.find('tr',{'valign':'top'})
+        
+        ## Title
+        a = pagetitle.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',stripHTML(a))
+        
+        # Find authorid and URL from... author url.
+        a = pagetitle.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+        
+        cats = soup.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+        for cat in cats:
+            if '/' == cat.string[0]:
+                self.story.addToList('ships','Harry Potter'+cat.string.split('(')[0])
+            elif 'Harry' in cat.string:
+                self.story.addToList('ships',cat.string.split('(')[0])
+            else:
+                self.story.addToList('category',cat.string)
+            if '(' in cat.string:
+                self.story.addToList('category',cat.string.split('(')[1].split(')')[0])
+            
+                
+            
+
+        chars = soup.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+        for char in chars:
+            self.story.addToList('characters',char.string)
+
+        genres = soup.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
+        for genre in genres:
+            self.story.addToList('genre',genre.string)
+
+        warnings = soup.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+        for warning in warnings:
+            self.story.addToList('warnings',warning.string)
+            
+        textsoup = stripHTML(soup)
+
+        a = textsoup.split('Published: ')[1].split(' ')[0]
+        self.story.setMetadata('datePublished', makeDate(stripHTML(a), self.dateformat))
+        a = textsoup.split('Updated: ')[1].split(' ')[0]
+        self.story.setMetadata('dateUpdated', makeDate(stripHTML(a), self.dateformat))
+        a = textsoup.split('Rating: ')[1].split(' ')[0]
+        self.story.setMetadata('rating', a)
+        a = textsoup.split('Length: ')[1].split('(')[1].split(' ')[0]
+        self.story.setMetadata('numWords', a)
+        a = textsoup.split('Completed: ')[1].split(' ')[0]
+        if 'Yes' in a:
+            self.story.setMetadata('status', 'Completed')
+        else:
+            self.story.setMetadata('status', 'In-Progress')
+        #a = textsoup.split('Summary: ')[1].split('Add Story to Favorites')[0]
+        #self.setDescription(url,a)
+        
+
+        
+        a=soup.find(text=re.compile("Summary: "))
+        i=0
+        svalue = ""
+        while i == 0:
+            try:
+                b = str(a)
+                svalue += b.split('Summary: ')[1]
+            except:
+                svalue += str(a)
+            if a.nextSibling != None:
+                a = a.nextSibling
+            else:
+                a = a.parent.nextSibling
+            if 'Disclaimer: ' in stripHTML(a):
+                i=1
+        self.setDescription(url,svalue)
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_scarvesandcoffeenet.py

@@ -48,7 +48,7 @@ class ScarvesAndCoffeeNetAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -65,6 +65,7 @@ class ScarvesAndCoffeeNetAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'www.scarvesandcoffee.net'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 
@@ -138,10 +139,10 @@ class ScarvesAndCoffeeNetAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        a = soup.find('div',{"id":"pagetitle"}).find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
         self.story.setMetadata('authorId',a['href'].split('=')[1])
         self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
         self.story.setMetadata('author',a.string)
@@ -222,6 +223,7 @@ class ScarvesAndCoffeeNetAdapter(BaseSiteAdapter):
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             

fanficdownloader/adapters/adapter_sg1heliopoliscom.py

@@ -48,7 +48,7 @@ class SG1HeliopolisComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         self.section=self.parsedUrl.path.split('/',)[1]
         
         # normalized story URL.
@@ -74,6 +74,7 @@ class SG1HeliopolisComAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'sg1-heliopolis.com'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/archive/viewstory.php?sid=1234 http://"+self.getSiteDomain()+"/adult/viewstory.php?sid=1234 http://"+self.getSiteDomain()+"/atlantis/viewstory.php?sid=1234"
 
@@ -139,7 +140,7 @@ class SG1HeliopolisComAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -233,6 +234,7 @@ class SG1HeliopolisComAdapter(BaseSiteAdapter):
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             

fanficdownloader/adapters/adapter_simplyundeniablecom.py

@@ -0,0 +1,218 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return SimplyUndeniableComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class SimplyUndeniableComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','sud')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%m/%d/%Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.simplyundeniable.com'
+
+    @classmethod
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Please log in now' in data \
+                or 'There is no such account on our website' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['cookiecheck'] = '1'
+        params['submit'] = 'Submit'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "My Account" not in d : #Member Account
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        if "Please log in now" in data:
+                raise exceptions.FailedToDownload(self.getSiteDomain() +" says: You need to have access to the restricted section.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('h1')
+        self.story.setMetadata('title',stripHTML(a))
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
+        
+        for info in asoup.findAll('table', {'cellpadding' : '5'}):
+            a = info.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+            if a != None:
+                self.story.setMetadata('title',stripHTML(a))
+                break
+                
+        # Find the chapters:
+        if "Disclaimer" in data:
+            self.chapterUrls.append((self.story.getMetadata('title'),url))
+        else:
+            for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+&i=1$')):
+            # just in case there's tags, like <i> in chapter titles.
+                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+        
+        a = info.find('td', {'valign' : 'top'}).find('p')
+        self.setDescription(url,a)
+        
+        a = info.find('td', {'class' : 'greysm'}).findAll('b')
+        self.story.setMetadata('datePublished', makeDate(stripHTML(a[0].nextSibling), self.dateformat))
+        self.story.setMetadata('dateUpdated', makeDate(stripHTML(a[1].nextSibling), self.dateformat))
+        if 'Yes' in a[2].nextSibling:
+            self.story.setMetadata('status', 'Completed')
+        else:
+            self.story.setMetadata('status', 'In-Progress')
+        self.story.setMetadata('rating', a[3].nextSibling)
+        self.story.setMetadata('numWords', a[4].nextSibling)
+        
+        
+        warnings = info.find('td', {'width' : '45'}).nextSibling.nextSibling.text.split(', ')
+        for warning in warnings:
+            if 'none' not in warning:
+                self.story.addToList('warnings',warning)
+        chars = info.find('td', {'width' : '51'}).nextSibling.nextSibling.text.split(', ')
+        for char in chars:
+            if '&' in char:
+                self.story.addToList('ships',char)
+            else:
+                self.story.addToList('characters',char)
+        genres = info.find('td', {'width' : '36'}).nextSibling.nextSibling.text.split(', ')
+        for genre in genres:
+            self.story.addToList('genre',genre)
+            
+        cat = info.find('a', href=re.compile(r'categories.php'))
+        self.story.addToList('category',cat.string)
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('span', {'class' : 'style'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_sinfuldesireorg.py

@@ -48,7 +48,7 @@ class SinfulDesireOrgAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/archive/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -65,6 +65,7 @@ class SinfulDesireOrgAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'www.sinful-desire.org'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/archive/viewstory.php?sid=1234"
 
@@ -129,7 +130,7 @@ class SinfulDesireOrgAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -226,6 +227,7 @@ class SinfulDesireOrgAdapter(BaseSiteAdapter):
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             

fanficdownloader/adapters/adapter_siyecouk.py

@@ -49,7 +49,7 @@ class SiyeCoUkAdapter(BaseSiteAdapter): # XXX
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/siye/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -70,6 +70,7 @@ class SiyeCoUkAdapter(BaseSiteAdapter): # XXX
     def getAcceptDomains(cls):
         return ['www.siye.co.uk','siye.co.uk']
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/siye/viewstory.php?sid=1234"
 
@@ -110,7 +111,7 @@ class SiyeCoUkAdapter(BaseSiteAdapter): # XXX
 
         ## Title
         titlea = authsoup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',titlea.string)
+        self.story.setMetadata('title',stripHTML(titlea))
         
         # Find the chapters (from soup, not authsoup):
         for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
@@ -215,6 +216,7 @@ class SiyeCoUkAdapter(BaseSiteAdapter): # XXX
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             

fanficdownloader/adapters/adapter_squidgeorgpeja.py

@@ -59,7 +59,7 @@ class SquidgeOrgPejaAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/peja/cgi-bin/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -81,6 +81,7 @@ class SquidgeOrgPejaAdapter(BaseSiteAdapter):
         # The config section name.  Only override if != site domain.
         return cls.getSiteDomain()+'/peja'
     
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/peja/cgi-bin/viewstory.php?sid=1234"
 
@@ -121,7 +122,7 @@ class SquidgeOrgPejaAdapter(BaseSiteAdapter):
         # formating, so it's a little ugly.
 
         title = authorSoup.find('a',{'href':'viewstory.php?sid='+self.story.getMetadata('storyId')})
-        self.story.setMetadata('title',title.string)
+        self.story.setMetadata('title',stripHTML(title))
         titleblock=title.parent.parent
         
         chapterselect=soup.find('select',{'name':'chapter'})
@@ -212,6 +213,7 @@ class SquidgeOrgPejaAdapter(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
                 

fanficdownloader/adapters/adapter_stargateatlantisorg.py

@@ -48,7 +48,7 @@ class StargateAtlantisOrgAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/fanfics/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -65,6 +65,7 @@ class StargateAtlantisOrgAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'stargate-atlantis.org'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/fanfics/viewstory.php?sid=1234"
 
@@ -105,7 +106,7 @@ class StargateAtlantisOrgAdapter(BaseSiteAdapter):
         self.story.setMetadata('author',aut.string)
         aut.extract()
         
-        self.story.setMetadata('title',a.string[:(len(a.string)-3)])
+        self.story.setMetadata('title',stripHTML(a)[:(len(stripHTML(a))-3)])
 
         # Find the chapters:
         chapters=soup.findAll('div', {'class' : 'content'})
@@ -204,6 +205,7 @@ class StargateAtlantisOrgAdapter(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             

fanficdownloader/adapters/adapter_storiesofardacom.py

@@ -48,7 +48,7 @@ class StoriesOfArdaComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/chapterlistview.asp?SID='+self.story.getMetadata('storyId'))
@@ -65,6 +65,7 @@ class StoriesOfArdaComAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'www.storiesofarda.com'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/chapterlistview.asp?SID=1234"
 
@@ -103,7 +104,7 @@ class StoriesOfArdaComAdapter(BaseSiteAdapter):
         asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
         
         a.find('em').extract()
-        self.story.setMetadata('title',a.text)
+        self.story.setMetadata('title',stripHTML(a))
 
         # Find the chapters: chapterview.asp?sid=7000&cid=30919
         chapters=soup.findAll('a', href=re.compile(r'chapterview.asp\?sid='+self.story.getMetadata('storyId')+"&cid=\d+$"))
@@ -136,8 +137,20 @@ class StoriesOfArdaComAdapter(BaseSiteAdapter):
 
         logger.debug('Getting chapter text from: %s' % url)
 
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+        if self.getConfig('is_adult'):
+            params = {'confirmAge':'1'}
+            data = self._postUrl(url,params)
+        else:
+            data = self._fetchUrl(url)
+
+        data = data[data.index('<table width="90%" align="center">'):]
+        data.replace("<body","<notbody").replace("<BODY","<NOTBODY")
+            
+        soup = bs.BeautifulStoneSoup(data,
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+
+        if "Please indicate that you are an adult by selecting the appropriate choice below" in data:
+            raise exceptions.FailedToDownload("Chapter requires you be an adult.  Set is_adult in personal.ini (chapter url:%s)" % url)
         
         div = soup.find('table', {'width' : '90%'}).find('td')
         div.name='div'

fanficdownloader/adapters/adapter_tenhawkpresentscom.py

@@ -44,7 +44,7 @@ class TenhawkPresentsComSiteAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -55,6 +55,7 @@ class TenhawkPresentsComSiteAdapter(BaseSiteAdapter):
     def getSiteDomain():
         return 'fanfiction.tenhawkpresents.com'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 
@@ -132,7 +133,7 @@ class TenhawkPresentsComSiteAdapter(BaseSiteAdapter):
 
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -220,6 +221,7 @@ class TenhawkPresentsComSiteAdapter(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             

fanficdownloader/adapters/adapter_test1.py

@@ -40,6 +40,7 @@ class TestSiteAdapter(BaseSiteAdapter):
     def getSiteDomain():
         return 'test1.com'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"?sid=1234"
 
@@ -47,31 +48,63 @@ class TestSiteAdapter(BaseSiteAdapter):
         return BaseSiteAdapter.getSiteURLPattern(self)+r'/?\?sid=\d+$'
 
     def extractChapterUrlsAndMetadata(self):
+        idstr = self.story.getMetadata('storyId')
+        idnum = int(idstr)
 
-        if self.story.getMetadata('storyId') == '665' and not (self.is_adult or self.getConfig("is_adult")):
+        if idnum >= 1000:
+            logger.warn("storyId:%s - Custom INI data will be used."%idstr)
+
+            sections = ['teststory:%s'%idstr,'teststory:defaults']
+            #print("self.get_config_list(sections,'valid_entries'):%s"%self.get_config_list(sections,'valid_entries'))
+            for key in self.get_config_list(sections,'valid_entries'):
+                if key.endswith("_list"):
+                    nkey = key[:-len("_list")]
+                    #print("addList:%s"%(nkey))
+                    for val in self.get_config_list(sections,key):
+                        #print("addList:%s->%s"%(nkey,val))
+                        self.story.addToList(nkey,val.decode('utf-8').replace('{{storyId}}',idstr))
+                else:
+                    # Special cases:
+                    if key in ['datePublished','dateUpdated']:
+                        self.story.setMetadata(key,makeDate(self.get_config(sections,key),"%Y-%m-%d"))
+                    else:
+                        self.story.setMetadata(key,self.get_config(sections,key).decode('utf-8').replace('{{storyId}}',idstr))
+                    #print("set:%s->%s"%(key,self.story.getMetadata(key)))
+
+            self.chapterUrls = []
+            for (j,chap) in enumerate(self.get_config_list(sections,'chaptertitles'),start=1):
+                self.chapterUrls.append( (chap,self.url+"&chapter=%d"%j) )
+            # self.chapterUrls = [(u'Prologue '+self.crazystring,self.url+"&chapter=1"),
+            #                 ('Chapter 1, Xenos on Cinnabar',self.url+"&chapter=2"),
+            #                 ]
+            self.story.setMetadata('numChapters',len(self.chapterUrls))
+            
+            return
+
+        if idstr == '665' and not (self.is_adult or self.getConfig("is_adult")):
             logger.warn("self.is_adult:%s"%self.is_adult)
             raise exceptions.AdultCheckRequired(self.url)
 
-        if self.story.getMetadata('storyId') == '666':
+        if idstr == '666':
             raise exceptions.StoryDoesNotExist(self.url)
 
-        if self.story.getMetadata('storyId').startswith('670'):
+        if idstr.startswith('670'):
             time.sleep(1.0)
             
-        if self.story.getMetadata('storyId').startswith('671'):
+        if idstr.startswith('671'):
             time.sleep(1.0)
             
         if self.getConfig("username"):
             self.username = self.getConfig("username")
         
-        if self.story.getMetadata('storyId') == '668' and self.username != "Me" :
+        if idstr == '668' and self.username != "Me" :
             raise exceptions.FailedToLogin(self.url,self.username)
 
-        if self.story.getMetadata('storyId') == '664':
-            self.story.setMetadata(u'title',"Test Story Title "+self.story.getMetadata('storyId')+self.crazystring)
+        if idstr == '664':
+            self.story.setMetadata(u'title',"Test Story Title "+idstr+self.crazystring)
             self.story.setMetadata('author','Test Author aa bare amp(&) quote(') amp(&)')
         else:
-            self.story.setMetadata(u'title',"Test Story Title "+self.story.getMetadata('storyId'))
+            self.story.setMetadata(u'title',"Test Story Title "+idstr)
             self.story.setMetadata('author','Test Author aa')
         self.story.setMetadata('storyUrl',self.url)
         self.setDescription(self.url,u'Description '+self.crazystring+u''' Done
@@ -79,13 +112,12 @@ class TestSiteAdapter(BaseSiteAdapter):
 Some more longer description.  "I suck at summaries!"  "Better than it sounds!"  "My first fic"
 ''')
         self.story.setMetadata('datePublished',makeDate("1975-03-15","%Y-%m-%d"))
-        if self.story.getMetadata('storyId') == '669':
+        if idstr == '669':
             self.story.setMetadata('dateUpdated',datetime.datetime.now())
         else:
             self.story.setMetadata('dateUpdated',makeDate("1975-04-15","%Y-%m-%d"))
         self.story.setMetadata('numWords','123456')
 
-        idnum = int(self.story.getMetadata('storyId'))
         if idnum % 2 == 1:
             self.story.setMetadata('status','In-Progress')
         else:
@@ -101,19 +133,78 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
                 }
             self.story.setMetadata('language',langs[idnum%len(langs)])
             self.setSeries('The Great Test',idnum)
-        
+            self.story.setMetadata('seriesUrl','http://test1.com?seriesid=1')
+        if idnum == 0:
+            self.setSeries("A Nook Hyphen Test "+self.story.getMetadata('dateCreated'),idnum)
+            self.story.setMetadata('seriesUrl','http://test1.com?seriesid=0')
+            
         self.story.setMetadata('rating','Tweenie')
-        
 
-        if self.story.getMetadata('storyId') == '673':
-            self.story.addToList('author','Author From List')
+        if idstr == '673':
+            self.story.addToList('author','Author From List 1')
             self.story.addToList('author','Author From List 2')
+            self.story.addToList('author','Author From List 3')
+            self.story.addToList('author','Author From List 4')
+            self.story.addToList('author','Author From List 5')
+            self.story.addToList('author','Author From List 6')
+            self.story.addToList('author','Author From List 7')
+            self.story.addToList('author','Author From List 8')
+            self.story.addToList('author','Author From List 9')
+            self.story.addToList('author','Author From List 0')
+            self.story.addToList('author','Author From List q')
+            self.story.addToList('author','Author From List w')
+            self.story.addToList('author','Author From List e')
+            self.story.addToList('author','Author From List r')
+            self.story.addToList('author','Author From List t')
+            self.story.addToList('author','Author From List y')
+            self.story.addToList('author','Author From List u')
+            self.story.addToList('author','Author From List i')
+            self.story.addToList('author','Author From List o')
         
-            self.story.addToList('authorId','98765')
+            self.story.addToList('authorId','98765-1')
             self.story.addToList('authorId','98765-2')
+            self.story.addToList('authorId','98765-3')
+            self.story.addToList('authorId','98765-4')
+            self.story.addToList('authorId','98765-5')
+            self.story.addToList('authorId','98765-6')
+            self.story.addToList('authorId','98765-7')
+            self.story.addToList('authorId','98765-8')
+            self.story.addToList('authorId','98765-9')
+            self.story.addToList('authorId','98765-0')
+            self.story.addToList('authorId','98765-q')
+            self.story.addToList('authorId','98765-w')
+            self.story.addToList('authorId','98765-e')
+            self.story.addToList('authorId','98765-r')
+            self.story.addToList('authorId','98765-t')
+            self.story.addToList('authorId','98765-y')
+            self.story.addToList('authorId','98765-u')
+            self.story.addToList('authorId','98765-i')
+            self.story.addToList('authorId','98765-o')
         
-            self.story.addToList('authorUrl','http://author/url')
+            self.story.addToList('authorUrl','http://author/url-1')
             self.story.addToList('authorUrl','http://author/url-2')
+            self.story.addToList('authorUrl','http://author/url-3')
+            self.story.addToList('authorUrl','http://author/url-4')
+            self.story.addToList('authorUrl','http://author/url-5')
+            self.story.addToList('authorUrl','http://author/url-6')
+            self.story.addToList('authorUrl','http://author/url-7')
+            self.story.addToList('authorUrl','http://author/url-8')
+            self.story.addToList('authorUrl','http://author/url-9')
+            self.story.addToList('authorUrl','http://author/url-0')
+            self.story.addToList('authorUrl','http://author/url-q')
+            self.story.addToList('authorUrl','http://author/url-w')
+            self.story.addToList('authorUrl','http://author/url-e')
+            self.story.addToList('authorUrl','http://author/url-r')
+            self.story.addToList('authorUrl','http://author/url-t')
+            self.story.addToList('authorUrl','http://author/url-y')
+            self.story.addToList('authorUrl','http://author/url-u')
+            self.story.addToList('authorUrl','http://author/url-i')
+            self.story.addToList('authorUrl','http://author/url-o')
+
+            self.story.addToList('category','Power Rangers')
+            self.story.addToList('category','SG-1')
+            self.story.addToList('genre','Porn')
+            self.story.addToList('genre','Drama')
         else:
             self.story.setMetadata('authorId','98765')
             self.story.setMetadata('authorUrl','http://author/url')
@@ -121,21 +212,40 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
         self.story.addToList('warnings','Swearing')
         self.story.addToList('warnings','Violence')
 
-        self.story.addToList('category','Harry Potter')
-        self.story.addToList('category','Furbie')
-        self.story.addToList('category','Crossover')
-        self.story.addToList('category',u'Puella Magi Madoka Magica/魔法少女まどか★マギカ')
-        self.story.addToList('category',u'Magical Girl Lyrical Nanoha')
-
+        if idstr == '80':
+            self.story.addToList('category',u'Rizzoli & Isles')
+            self.story.addToList('characters','J. Rizzoli')
+        elif idstr == '81':
+            self.story.addToList('category',u'Pitch Perfect')
+            self.story.addToList('characters','Chloe B.')
+        elif idstr == '83':
+            self.story.addToList('category',u'Rizzoli & Isles')
+            self.story.addToList('characters','J. Rizzoli')
+            self.story.addToList('category',u'Pitch Perfect')
+            self.story.addToList('characters','Chloe B.')
+            self.story.addToList('ships','Chloe B. & J. Rizzoli')
+        elif idstr == '82':
+            self.story.addToList('characters','Henry (Once Upon a Time)')        
+            self.story.addToList('category',u'Once Upon a Time (TV)')
+        else:
+            self.story.addToList('category','Harry Potter')
+            self.story.addToList('category','Furbie')
+            self.story.addToList('category','Crossover')
+            self.story.addToList('category',u'Puella Magi Madoka Magica/魔法少女まどか★マギカ')
+            self.story.addToList('category',u'Magical Girl Lyrical Nanoha')
+            self.story.addToList('category',u'Once Upon a Time (TV)')
+            self.story.addToList('characters','Bob Smith')
+            self.story.addToList('characters','George Johnson')
+            self.story.addToList('characters','Fred Smythe')
+            self.story.addToList('ships','Harry Potter/Ginny Weasley')
+            self.story.addToList('ships','Harry Potter/Ginny Weasley/Albus Dumbledore')
+            self.story.addToList('ships','Harry Potter & Hermione Granger')
+            
         self.story.addToList('genre','Fantasy')
         self.story.addToList('genre','Comedy')
         self.story.addToList('genre','Sci-Fi')
         self.story.addToList('genre','Noir')
-        
-        self.story.addToList('characters','Bob Smith')
-        self.story.addToList('characters','George Johnson')
-        self.story.addToList('characters','Fred Smythe')
-        
+                
         self.story.addToList('listX','xVal1')
         self.story.addToList('listX','xVal2')
         self.story.addToList('listX','xVal3')
@@ -161,9 +271,9 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
                             ('Chapter 3, Over Cinnabar',self.url+"&chapter=4"),
                             ('Chapter 4',self.url+"&chapter=5"),
                             ('Chapter 5',self.url+"&chapter=6"),
-                            #('Chapter 6',self.url+"&chapter=7"),
-                            #('Chapter 7',self.url+"&chapter=8"),
-                            #('Chapter 8',self.url+"&chapter=9"),
+                            ('Chapter 6',self.url+"&chapter=7"),
+                            ('Chapter 7',self.url+"&chapter=8"),
+                            ('Chapter 8',self.url+"&chapter=9"),
                             #('Chapter 9',self.url+"&chapter=0"),
                             #('Chapter 0',self.url+"&chapter=a"),
                             #('Chapter a',self.url+"&chapter=b"),
@@ -186,9 +296,6 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
 
     def getChapterText(self, url):
         logger.debug('Getting chapter text from: %s' % url)
-        if self.story.getMetadata('storyId') == '667':
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!" % url)
-
         if self.story.getMetadata('storyId').startswith('670') or \
                 self.story.getMetadata('storyId').startswith('672'):
             time.sleep(1.0)
@@ -198,23 +305,42 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
 <div>
 <h3>Prologue</h3>
 <p>This is a fake adapter for testing purposes.  Different sid's will give different errors:</p>
+<h4>Config(personal.ini)</h4>
+<p>sid&gt;=1000 will use custom test story data from your configuration(personal.ini)</p>
+<p>Hard coded ids:</p>
 <p>http://test1.com?sid=664 - Crazy string title</p>
 <p>http://test1.com?sid=665 - raises AdultCheckRequired</p>
 <p>http://test1.com?sid=666 - raises StoryDoesNotExist</p>
-<p>http://test1.com?sid=667 - raises FailedToDownload on chapter 1</p>
+<p>http://test1.com?sid=667 - raises FailedToDownload on chapters 2+</p>
 <p>http://test1.com?sid=668 - raises FailedToLogin unless username='Me'</p>
 <p>http://test1.com?sid=669 - Succeeds with Updated Date=now</p>
 <p>http://test1.com?sid=670 - Succeeds, but sleeps 2sec on each chapter</p>
 <p>http://test1.com?sid=671 - Succeeds, but sleeps 2sec metadata only</p>
 <p>http://test1.com?sid=672 - Succeeds, quick meta, sleeps 2sec chapters only</p>
-<p>http://test1.com?sid=673 - Succeeds, multiple authors</p>
+<p>http://test1.com?sid=673 - Succeeds, multiple authors, extra categories, genres</p>
+<p>http://test1.com?sid=0 - Succeeds, generates some text specifically for testing hyphenation problems with Nook STR/STRwG</p>
 <p>Odd sid's will be In-Progress, evens complete.  sid&lt;10 will be assigned one of four languages and included in a series.</p>
 </div>
+'''
+        elif self.story.getMetadata('storyId') == '0':
+            text=u'''
+<h3>45. Pronglet Returns to Hogwarts: Chapter 7</h3>
+<br />
+    eyes… but I’m not convinced we should automatically<br />
+<br /><br />
+<b>Thanks to the latest to recommend me: Alastor</b><br />
+<br /><br />
+    “Sure, invite her along. Does she have children?”<br />
+<br />
 '''
         else:
+            if self.story.getMetadata('storyId') == '667':
+                raise exceptions.FailedToDownload("Error downloading Chapter: %s!" % url)
+
             text=u'''
 <div>
 <h3>Chapter title from site</h3>
+<p>Timestamp:'''+datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")+'''</p>
 <p>Lorem '''+self.crazystring+u''' <i>italics</i>, <b>bold</b>, <u>underline</u> consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p>
 br breaks<br><br>
 Puella Magi Madoka Magica/魔法少女まどか★マギカ
@@ -224,7 +350,7 @@ Don't&#8212e;ver&#8212d;o&#8212;that&#8212a;gain, &#27861; &#xE9;
 <hr>
 horizontal rules
 <hr size=1 noshade>
-<p>Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p>
+<p>"Lorem ipsum dolor sit amet", consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore--et dolore magna aliqua. 'Ut enim ad minim veniam', quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p>
 <p>Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p>
 </div>
 '''

fanficdownloader/adapters/adapter_thealphagatecom.py

@@ -48,7 +48,7 @@ class TheAlphaGateComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -65,6 +65,7 @@ class TheAlphaGateComAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'www.thealphagate.com'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 
@@ -98,7 +99,7 @@ class TheAlphaGateComAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -189,6 +190,7 @@ class TheAlphaGateComAdapter(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             

fanficdownloader/adapters/adapter_thehexfilesnet.py

@@ -48,7 +48,7 @@ class TheHexFilesNetAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -69,6 +69,7 @@ class TheHexFilesNetAdapter(BaseSiteAdapter):
     def getAcceptDomains(cls):
         return ['www.thehexfiles.net','thehexfiles.net']
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 
@@ -102,7 +103,7 @@ class TheHexFilesNetAdapter(BaseSiteAdapter):
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
         self.story.setMetadata('authorId',a['href'].split('=')[1])
         self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
+        self.story.setMetadata('author',stripHTML(a))
         asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
 		
         try:
@@ -117,7 +118,7 @@ class TheHexFilesNetAdapter(BaseSiteAdapter):
         for info in asoup.findAll('table', {'cellspacing' : '4'}):
             a = info.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
             if a != None:
-                self.story.setMetadata('title',a.string)
+                self.story.setMetadata('title',stripHTML(a))
                 break
 		
 

fanficdownloader/adapters/adapter_thehookupzonenet.py

@@ -72,7 +72,7 @@ class TheHookupZoneNetAdapter(BaseSiteAdapter): # XXX
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
@@ -90,6 +90,7 @@ class TheHookupZoneNetAdapter(BaseSiteAdapter): # XXX
         # The site domain.  Does have www here, if it uses it.
         return 'thehookupzone.net' # XXX
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/CriminalMinds/viewstory.php?sid=1234"
 
@@ -178,12 +179,12 @@ class TheHookupZoneNetAdapter(BaseSiteAdapter): # XXX
         
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
         self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/CriminalMinds/'+a['href'])
         self.story.setMetadata('author',a.string)
 
         # Find the chapters:
@@ -283,6 +284,7 @@ class TheHookupZoneNetAdapter(BaseSiteAdapter): # XXX
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             

fanficdownloader/adapters/adapter_themasquenet.py

@@ -48,7 +48,7 @@ class TheMasqueNetAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         if self.parsedUrl.path.split('/',)[1] == 'wiktt':
             self.story.addToList("category","Harry Potter")
@@ -72,6 +72,7 @@ class TheMasqueNetAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'themasque.net'
         
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://themasque.net/wiktt/efiction/viewstory.php?sid=1234 http://themasque.net/efiction/viewstory.php?sid=1234"
 
@@ -176,7 +177,7 @@ class TheMasqueNetAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))

fanficdownloader/adapters/adapter_thepetulantpoetesscom.py

@@ -48,7 +48,7 @@ class ThePetulantPoetessComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId') +'&i=1')
@@ -65,6 +65,7 @@ class ThePetulantPoetessComAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'www.thepetulantpoetess.com'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 
@@ -164,7 +165,7 @@ class ThePetulantPoetessComAdapter(BaseSiteAdapter):
             for info in asoup.findAll('td', {'class' : 'highlightcolor1'}):
                 a = info.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
                 if a != None:
-                    self.story.setMetadata('title',a.text)                
+                    self.story.setMetadata('title',stripHTML(a))
                     found = 1
                     break
             index=index+1

fanficdownloader/adapters/adapter_thequidditchpitchorg.py

@@ -52,7 +52,7 @@ class TheQuidditchPitchOrgAdapter(BaseSiteAdapter): # XXX
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         # XXX Most sites don't have the  part.  Replace all to remove it usually.
@@ -74,6 +74,7 @@ class TheQuidditchPitchOrgAdapter(BaseSiteAdapter): # XXX
     def getAcceptDomains(cls):
         return ['www.thequidditchpitch.org','thequidditchpitch.org']
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 
@@ -164,7 +165,7 @@ class TheQuidditchPitchOrgAdapter(BaseSiteAdapter): # XXX
         
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -263,6 +264,7 @@ class TheQuidditchPitchOrgAdapter(BaseSiteAdapter): # XXX
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             

fanficdownloader/adapters/adapter_thewriterscoffeeshopcom.py

@@ -44,17 +44,18 @@ class TheWritersCoffeeShopComSiteAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/library/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        self.dateformat = "%B %d, %Y"
+        self.dateformat = "%d %b %Y"
 
             
     @staticmethod
     def getSiteDomain():
         return 'www.thewriterscoffeeshop.com'
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/library/viewstory.php?sid=1234"
 
@@ -227,6 +228,7 @@ class TheWritersCoffeeShopComSiteAdapter(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             
@@ -243,16 +245,23 @@ class TheWritersCoffeeShopComSiteAdapter(BaseSiteAdapter):
         # issues with different SGML parsers in python.  This is a
         # nasty hack, but it works.
         data = data[data.index("<body"):]
-        
-        soup = bs.BeautifulStoneSoup(data,
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        span = soup.find('div', {'id' : 'story'})
 
-        if None == span:
+        chapter=bs.BeautifulSoup('<div class="story"></div>')
+        
+        soup = bs.BeautifulSoup(data)
+
+        found=False
+        for div in soup.findAll('div'):
+            if div.has_key('class') and div['class'] == 'notes':
+                chapter.append(div)
+            if div.has_key('id') and div['id'] == 'story':
+                chapter.append(div)
+                found=True
+
+        if not found:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,span)
+
+        return self.utf8FromSoup(url,chapter)
 
 def getClass():
     return TheWritersCoffeeShopComSiteAdapter

fanficdownloader/adapters/adapter_tokrafandomnetcom.py

@@ -0,0 +1,237 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return TokraFandomnetComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class TokraFandomnetComAdapter(BaseSiteAdapter): 
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','tokra')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%m/%d/%Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.  But it
+        # doesn't matter too much anymore.
+        return 'tokra.fandomnet.com'
+
+    @classmethod
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=3"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid & error in url.
+                addurl = addurl.replace("&","&")
+                url = self.url+'&index=1'+addurl
+                logger.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        #print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',stripHTML(a))
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Rating
+        rate = stripHTML(soup.find('div',{'id':'pagetitle'}))
+        rate = rate[rate.rindex('[')+1:rate.rindex(']')]
+        self.story.setMetadata('rating', rate)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        metadiv = soup.find('div',{'class':'content'})
+        smalldiv = metadiv.find('div',{'class':'small'})
+
+        # tokra categories -> genre
+        # categories will be filled from ini.
+        genres = smalldiv.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+        for genre in genres:
+            self.story.addToList('genre',genre.string)
+        
+        chars = smalldiv.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+        for char in chars:
+            self.story.addToList('characters',char.string)
+
+        metatext = stripHTML(smalldiv)
+
+        if 'Completed: Yes' in metatext:
+            self.story.setMetadata('status', 'Completed')
+        else:
+            self.story.setMetadata('status', 'In-Progress')
+
+        wordstart=metatext.rindex('Word count:')+12
+        words = metatext[wordstart:metatext.index(' ',wordstart)]
+        self.story.setMetadata('numWords', words)
+
+        datesdiv = soup.find('div',{'class':'bottom'})
+        dates = stripHTML(datesdiv).split()
+        # Published: 04/26/2011 Updated: 03/06/2013
+        self.story.setMetadata('datePublished', makeDate(dates[1], self.dateformat))
+        self.story.setMetadata('dateUpdated', makeDate(dates[3], self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            i=1
+            for a in storyas:
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
+                        break
+                    i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+        # remove 'small' leaving only summary.
+        smalldiv.extract()
+        self.setDescription(url,metadiv)
+        
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulSoup(self._fetchUrl(url))
+        
+        div = soup.find('div', {'class' : 'content'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+
+        # remove some decorations while keeping notes.
+        remove = div.find('div', {'id' : 'pagetitle'})
+        remove.extract()
+    
+        for remove in div.findAll('div', {'class' : 'right'}):
+            remove.extract()
+    
+        for remove in div.findAll('div', {'class' : 'left'}):
+            remove.extract()
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_tthfanficorg.py

@@ -41,7 +41,7 @@ class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter):
         m = re.match(self.getSiteURLPattern(),url)
         if m:
             self.story.setMetadata('storyId',m.group('id'))
-            logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+            
             # normalized story URL.
             self._setURL("http://"+self.getSiteDomain()\
                          +"/Story-"+self.story.getMetadata('storyId'))
@@ -54,13 +54,14 @@ class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter):
     def getSiteDomain():
         return 'www.tthfanfic.org'
 
+    @classmethod
     def getSiteExampleURLs(self):
-        return "http://www.tthfanfic.org/Story-5583 http://www.tthfanfic.org/Story-5583/Greywizard+Marked+By+Kane.htm http://www.tthfanfic.org/T-526321777890480578489880055880/Story-26448-15/batzulger+Willow+Rosenberg+and+the+Mind+Riders.htm"
+        return "http://www.tthfanfic.org/Story-1234 http://www.tthfanfic.org/Story-1234/Author+Story+Title.htm http://www.tthfanfic.org/T-99999999/Story-1234-1/Author+Story+Title.htm http://www.tthfanfic.org/story.php?no=12345"
 
-    # http://www.tthfanfic.org/T-526321777848988007890480555880/Story-26448-15/batzulger+Willow+Rosenberg+and+the+Mind+Riders.htm
-    # http://www.tthfanfic.org/Story-5583
-    # http://www.tthfanfic.org/Story-5583/Greywizard+Marked+By+Kane.htm
-    # http://www.tthfanfic.org/story.php?no=26093
+    # http://www.tthfanfic.org/T-999999999999/Story-12345-1/Author+Story+Title.htm
+    # http://www.tthfanfic.org/Story-12345
+    # http://www.tthfanfic.org/Story-12345/Author+Story+Title.htm
+    # http://www.tthfanfic.org/story.php?no=12345
     def getSiteURLPattern(self):
         return r"http://www.tthfanfic.org(/(T-\d+/)?Story-|/story.php\?no=)(?P<id>\d+)(-\d+)?(/.*)?$"
 
@@ -100,8 +101,8 @@ class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter):
     
         if "Stories Published" not in d : #Member Account
             logger.info("Failed to login to URL %s as %s" % (loginUrl,
-                                                              params['penname']))
-            raise exceptions.FailedToLogin(url,params['penname'])
+                                                             params['urealname']))
+            raise exceptions.FailedToLogin(self.url,params['urealname'])
             return False
         else:
             return True
@@ -154,26 +155,6 @@ class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter):
         self.story.setMetadata('author',stripHTML(a))
         authorurl = 'http://'+self.host+a['href']
 
-        ainfo = soup.find('a', href='/StoryInfo-%s-1'%self.story.getMetadata('storyId'))
-        if ainfo != None: # indicates multiple authors/contributors.
-            try:
-                # going to pull part of the meta data from author list page.
-                infourl = 'http://'+self.host+ainfo['href']
-                logger.debug("**StoryInfo** URL: "+infourl)
-                infodata = self._fetchUrl(infourl)
-                infosoup = bs.BeautifulSoup(infodata)
-
-                for a in infosoup.findAll('a',href=re.compile(r"^/Author-\d+")):
-                    self.story.addToList('authorId',a['href'].split('/')[1].split('-')[1])
-                    self.story.addToList('authorUrl','http://'+self.host+a['href'].replace("/Author-","/AuthorStories-"))
-                    self.story.addToList('author',stripHTML(a))                
-                                            
-            except urllib2.HTTPError, e:
-                if e.code == 404:
-                    raise exceptions.StoryDoesNotExist(url)
-                else:
-                    raise e
-    
         try:
             # going to pull part of the meta data from *primary* author list page.
             logger.debug("**AUTHOR** URL: "+authorurl)
@@ -198,15 +179,62 @@ class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter):
         #self.story.setMetadata('description',stripHTML(storydiv.find('div',{'class':'storydesc'})))
         self.story.setMetadata('title',stripHTML(storydiv.find('a',{'class':'storylink'})))
 
+        ainfo = soup.find('a', href='/StoryInfo-%s-1'%self.story.getMetadata('storyId'))
+        if ainfo != None: # indicates multiple authors/contributors.
+            try:
+                # going to pull part of the meta data from author list page.
+                infourl = 'http://'+self.host+ainfo['href']
+                logger.debug("**StoryInfo** URL: "+infourl)
+                infodata = self._fetchUrl(infourl)
+                infosoup = bs.BeautifulSoup(infodata)
+
+                # for a in infosoup.findAll('a',href=re.compile(r"^/Author-\d+")):
+                #     self.story.addToList('authorId',a['href'].split('/')[1].split('-')[1])
+                #     self.story.addToList('authorUrl','http://'+self.host+a['href'].replace("/Author-","/AuthorStories-"))
+                #     self.story.addToList('author',stripHTML(a))
+
+                # second verticaltable is the chapter list.
+                table = infosoup.findAll('table',{'class':'verticaltable'})[1]
+                for a in table.findAll('a',href=re.compile(r"^/Story-"+self.story.getMetadata('storyId'))):
+                    autha = a.findNext('a',href=re.compile(r"^/Author-\d+"))
+                    self.story.addToList('authorId',autha['href'].split('/')[1].split('-')[1])
+                    self.story.addToList('authorUrl','http://'+self.host+autha['href'].replace("/Author-","/AuthorStories-"))
+                    self.story.addToList('author',stripHTML(autha))
+                    # include leading number to match 1. ... 2. ...
+                    self.chapterUrls.append(("%d. %s by %s"%(len(self.chapterUrls)+1,
+                                                             stripHTML(a),
+                                                             stripHTML(autha)),'http://'+self.host+a['href']))
+                                            
+            except urllib2.HTTPError, e:
+                if e.code == 404:
+                    raise exceptions.StoryDoesNotExist(url)
+                else:
+                    raise e
+        else: # single author:
+            # Find the chapter selector 
+            select = soup.find('select', { 'name' : 'chapnav' } )
+        	 
+            if select is None:
+        	   # no selector found, so it's a one-chapter story.
+        	   self.chapterUrls.append((self.story.getMetadata('title'),url))
+            else:
+                allOptions = select.findAll('option')
+                for o in allOptions:
+                    url = "http://"+self.host+o['value']
+                    # just in case there's tags, like <i> in chapter titles.
+                    self.chapterUrls.append((stripHTML(o),url))
+    
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
         verticaltable = soup.find('table', {'class':'verticaltable'})
 
         BtVS = True
         BtVSNonX = False
         for cat in verticaltable.findAll('a', href=re.compile(r"^/Category-")):
-            if cat.string not in ['General', 'Non-BtVS/AtS Stories', 'BtVS/AtS Non-Crossover', 'Non-BtVS Crossovers']:
+            if cat.string not in ['General', 'Non-BtVS/AtS Stories', 'Non-BTVS/AtS Stories', 'BtVS/AtS Non-Crossover', 'Non-BtVS Crossovers']:
                 self.story.addToList('category',cat.string)
             else:
-                if 'Non-BtVS' in cat.string:
+                if 'Non-BtVS' in cat.string or 'Non-BTVS' in cat.string:
                     BtVS = False
                 if 'BtVS/AtS Non-Crossover' == cat.string:
                     BtVSNonX = True
@@ -235,29 +263,12 @@ class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter):
         if BtVS:
             self.story.addToList('category','Buffy: The Vampire Slayer')
             
-        # Find the chapter selector 
-        select = soup.find('select', { 'name' : 'chapnav' } )
-    	 
-        if select is None:
-    	   # no selector found, so it's a one-chapter story.
-    	   self.chapterUrls.append((self.story.getMetadata('title'),url))
-        else:
-            allOptions = select.findAll('option')
-            for o in allOptions:
-                url = "http://"+self.host+o['value']
-                # just in case there's tags, like <i> in chapter titles.
-                self.chapterUrls.append((stripHTML(o),url))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
         pseries = soup.find('p', {'style':'margin-top:0px'})
         m = re.match('This story is No\. (?P<num>\d+) in the series &quot;(?P<series>.+)&quot;\.',
                      pseries.text)
         if m:
             self.setSeries(m.group('series'),m.group('num'))
-        
-        return
-
+            self.story.setMetadata('seriesUrl',"http://"+self.host+pseries.find('a')['href'])
 
     def getChapterText(self, url):
         logger.debug('Getting chapter text from: %s' % url)

fanficdownloader/adapters/adapter_twilightarchivescom.py

@@ -48,7 +48,7 @@ class TwilightArchivesComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL. http://www.twilightarchives.com/read/9353
         self._setURL('http://' + self.getSiteDomain() + '/read/'+self.story.getMetadata('storyId'))
@@ -66,6 +66,7 @@ class TwilightArchivesComAdapter(BaseSiteAdapter):
         return 'www.twilightarchives.com'
        
 
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/read/1234"
 
@@ -99,7 +100,7 @@ class TwilightArchivesComAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('h1')
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find the chapters:
         chapters=soup.find('ol', {'class' : 'chapters'})
@@ -163,6 +164,7 @@ class TwilightArchivesComAdapter(BaseSiteAdapter):
                 for a in storyas:
                     if a['href'] == ('/read/'+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1