Bump versions.

meta section, fix highbyte chars in description.

app.yaml

@@ -1,6 +1,6 @@
 # ffd-retief-hrd fanfictiondownloader
 application: fanfictiondownloader
-version: 4-4-45
+version: 4-4-63
 runtime: python27
 api_version: 1
 threadsafe: true

calibre-plugin/__init__.py

@@ -1,11 +1,10 @@
 #!/usr/bin/env python
-# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
 # -*- coding: utf-8 -*-
 from __future__ import (unicode_literals, division, absolute_import,
                         print_function)
 
 __license__   = 'GPL v3'
-__copyright__ = '2011, Jim Miller'
+__copyright__ = '2013, Jim Miller'
 __docformat__ = 'restructuredtext en'
 
 # The class that all Interface Action plugin wrappers must inherit from
@@ -27,7 +26,7 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
     description         = 'UI plugin to download FanFiction stories from various sites.'
     supported_platforms = ['windows', 'osx', 'linux']
     author              = 'Jim Miller'
-    version             = (1, 7, 11)
+    version             = (1, 7, 30)
     minimum_calibre_version = (0, 8, 57)
 
     #: This field defines the GUI plugin class that contains all the code
@@ -80,11 +79,29 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
         if ac is not None:
             ac.apply_settings()
 
-# For testing, run from command line with this:
-# calibre-debug -e __init__.py
-# 
-if __name__ == '__main__':
-    from PyQt4.Qt import QApplication
-    from calibre.gui2.preferences import test_widget
-    app = QApplication([])
-    test_widget('Advanced', 'Plugins')
+    def cli_main(self,argv):
+        # I believe there's no performance hit loading these here when
+        # CLI--it would load everytime anyway.
+        from StringIO import StringIO
+        from calibre.library import db
+        from calibre_plugins.fanfictiondownloader_plugin.downloader import main as ffdl_main
+        from calibre_plugins.fanfictiondownloader_plugin.prefs import PrefsFacade
+        from calibre.utils.config import prefs as calibre_prefs
+        from optparse import OptionParser      
+
+        parser = OptionParser('%prog --run-plugin '+self.name+' -- [options] <storyurl>')
+        parser.add_option('--library-path', '--with-library', default=None, help=_('Path to the calibre library. Default is to use the path stored in the settings.'))
+        # parser.add_option('--dont-notify-gui', default=False, action='store_true',
+        #               help=_('Do not notify the running calibre GUI (if any) that the database has'
+        #                      ' changed. Use with care, as it can lead to database corruption!'))
+
+        pargs = [x for x in argv if x.startswith('--with-library') or x.startswith('--library-path')
+                 or not x.startswith('-')]
+        opts, args = parser.parse_args(pargs)
+
+        ffdl_prefs = PrefsFacade(db(path=opts.library_path,
+                                    read_only=True))
+        ffdl_main(argv[1:],
+                  parser=parser,
+                  passed_defaultsini=StringIO(get_resources("defaults.ini")),
+                  passed_personalini=StringIO(ffdl_prefs["personal.ini"]))

calibre-plugin/about.txt

@@ -4,7 +4,7 @@
 '<a href="http://www.mobileread.com/forums/showthread.php?t=134856">Reading List</a>',
 '<a href="http://www.mobileread.com/forums/showthread.php?t=126727">Extract ISBN</a>' and
 '<a href="http://www.mobileread.com/forums/showthread.php?t=134000">Count Pages</a>'
-plugins.  bbcodeutils code contributed by Pau Sanchez.</p>
+plugins.</p>
 
 <p>
 Calibre officially distributes plugins from the mobileread.com forum site.

calibre-plugin/common_utils.py

@@ -239,8 +239,8 @@ class SizePersistedDialog(QDialog):
             self.restoreGeometry(self.geom)
 
     def dialog_closing(self, result):
-        geom = bytearray(self.saveGeometry())
-        gprefs[self.unique_pref_name] = geom
+        self.geom = bytearray(self.saveGeometry())
+        gprefs[self.unique_pref_name] = self.geom
 
 
 class ReadOnlyTableWidgetItem(QTableWidgetItem):

calibre-plugin/config.py

@@ -13,162 +13,40 @@ from collections import OrderedDict
 from PyQt4.Qt import (QDialog, QWidget, QVBoxLayout, QHBoxLayout, QLabel,
                       QLineEdit, QFont, QWidget, QTextEdit, QComboBox,
                       QCheckBox, QPushButton, QTabWidget, QVariant, QScrollArea,
-                      QDialogButtonBox )
+                      QDialogButtonBox, QGroupBox )
 
-from calibre.gui2 import dynamic, info_dialog
-from calibre.utils.config import JSONConfig
 from calibre.gui2.ui import get_gui
+from calibre.gui2 import dynamic, info_dialog
 
+from calibre_plugins.fanfictiondownloader_plugin.prefs import prefs, PREFS_NAMESPACE
 from calibre_plugins.fanfictiondownloader_plugin.dialogs \
     import (UPDATE, UPDATEALWAYS, OVERWRITE, collision_order, RejectListDialog,
-            EditTextDialog)
+            EditTextDialog, RejectUrlEntry)
     
 from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.adapters \
     import (getConfigSections, getNormalStoryURL)
 
 from calibre_plugins.fanfictiondownloader_plugin.common_utils \
-    import ( get_library_uuid, KeyboardConfigDialog, PrefsViewerDialog )
+    import ( KeyboardConfigDialog, PrefsViewerDialog )
 
 from calibre.gui2.complete import MultiCompleteLineEdit
 
-PREFS_NAMESPACE = 'FanFictionDownLoaderPlugin'
-PREFS_KEY_SETTINGS = 'settings'
-
-# Set defaults used by all.  Library specific settings continue to
-# take from here.
-default_prefs = {}
-default_prefs['personal.ini'] = get_resources('plugin-example.ini')
-default_prefs['rejecturls'] = ''
-default_prefs['rejectreasons'] = '''Sucked
-Boring
-Dup from another site'''
-
-default_prefs['updatemeta'] = True
-default_prefs['updatecover'] = False
-default_prefs['updateepubcover'] = False
-default_prefs['keeptags'] = False
-default_prefs['urlsfromclip'] = True
-default_prefs['updatedefault'] = True
-default_prefs['fileform'] = 'epub'
-default_prefs['collision'] = OVERWRITE
-default_prefs['deleteotherforms'] = False
-default_prefs['adddialogstaysontop'] = False
-default_prefs['includeimages'] = False
-default_prefs['lookforurlinhtml'] = False
-default_prefs['injectseries'] = False
-
-default_prefs['send_lists'] = ''
-default_prefs['read_lists'] = ''
-default_prefs['addtolists'] = False
-default_prefs['addtoreadlists'] = False
-default_prefs['addtolistsonread'] = False
-
-default_prefs['gcnewonly'] = False
-default_prefs['gc_site_settings'] = {}
-default_prefs['allow_gc_from_ini'] = True
-
-default_prefs['countpagesstats'] = []
-
-default_prefs['errorcol'] = ''
-default_prefs['custom_cols'] = {}
-default_prefs['custom_cols_newonly'] = {}
-default_prefs['allow_custcol_from_ini'] = True
-
-default_prefs['std_cols_newonly'] = {}
-
-def set_library_config(library_config):
-    get_gui().current_db.prefs.set_namespaced(PREFS_NAMESPACE,
-                                              PREFS_KEY_SETTINGS,
-                                              library_config)
-    
-def get_library_config():
-    db = get_gui().current_db
-    library_id = get_library_uuid(db)
-    library_config = None
-    # Check whether this is a configuration needing to be migrated
-    # from json into database.  If so: get it, set it, rename it in json.
-    if library_id in old_prefs:
-        #print("get prefs from old_prefs")
-        library_config = old_prefs[library_id]
-        set_library_config(library_config)
-        old_prefs["migrated to library db %s"%library_id] = old_prefs[library_id]
-        del old_prefs[library_id]
-
-    if library_config is None:
-        #print("get prefs from db")
-        library_config = db.prefs.get_namespaced(PREFS_NAMESPACE, PREFS_KEY_SETTINGS,
-                                                 copy.deepcopy(default_prefs))
-    return library_config
-
-# This is where all preferences for this plugin *were* stored
-# Remember that this name (i.e. plugins/fanfictiondownloader_plugin) is also
-# in a global namespace, so make it as unique as possible.
-# You should always prefix your config file name with plugins/,
-# so as to ensure you dont accidentally clobber a calibre config file
-old_prefs = JSONConfig('plugins/fanfictiondownloader_plugin')
-
-# fake out so I don't have to change the prefs calls anywhere.  The
-# Java programmer in me is offended by op-overloading, but it's very
-# tidy.
-class PrefsFacade():
-    def __init__(self,default_prefs):
-        self.default_prefs = default_prefs
-        self.libraryid = None
-        self.current_prefs = None
-        
-    def _get_prefs(self):
-        libraryid = get_library_uuid(get_gui().current_db)
-        if self.current_prefs == None or self.libraryid != libraryid:
-            #print("self.current_prefs == None(%s) or self.libraryid != libraryid(%s)"%(self.current_prefs == None,self.libraryid != libraryid))
-            self.libraryid = libraryid
-            self.current_prefs = get_library_config()
-        return self.current_prefs
-        
-    def __getitem__(self,k):            
-        prefs = self._get_prefs()
-        if k not in prefs:
-            # pulls from default_prefs.defaults automatically if not set
-            # in default_prefs
-            return self.default_prefs[k]
-        return prefs[k]
-
-    def __setitem__(self,k,v):
-        prefs = self._get_prefs()
-        prefs[k]=v
-        # self._save_prefs(prefs)
-
-    def __delitem__(self,k):
-        prefs = self._get_prefs()
-        if k in prefs:
-            del prefs[k]
-
-    def save_to_db(self):
-        set_library_config(self._get_prefs())
-        
-prefs = PrefsFacade(default_prefs)
-
 class RejectURLList:
     def __init__(self,prefs):
         self.prefs = prefs
         self.sync_lock = threading.RLock()
         self.listcache = None
 
-    def _read_list_from_text(self,text,addreasontext=None):
-        cache = {}
+    def _read_list_from_text(self,text,addreasontext=''):
+        cache = OrderedDict()
+
+        #print("_read_list_from_text")
         for line in text.splitlines():
-            if ',' in line:
-                (rejurl,note) = line.split(',',1)
-            else:
-                (rejurl,note) = (line,'')
-            rejurl = getNormalStoryURL(rejurl)
-            if rejurl:
-                if addreasontext and note:
-                    note = note +" - "+addreasontext
-                elif addreasontext:
-                    note = addreasontext
-                cache[rejurl] = note
-        return cache
-        
+            rue = RejectUrlEntry(line,addreasontext=addreasontext,fromline=True)
+            #print("rue.url:%s"%rue.url)
+            if rue.valid:
+                cache[rue.url] = rue
+        return cache        
 
     def _get_listcache(self):
         if self.listcache == None:
@@ -176,26 +54,35 @@ class RejectURLList:
         return self.listcache
 
     def _save_list(self,listcache):
-        rejectlist = []
-        for url in listcache:
-            rejectlist.append("%s,%s"%(url,listcache[url]))
-            
-        self.prefs['rejecturls'] = '\n'.join(rejectlist)
+        #print("_save_list")
+        self.prefs['rejecturls'] = '\n'.join([x.to_line() for x in listcache.values()])
         self.prefs.save_to_db()
         self.listcache = None
         
     def clear_cache(self):
         self.listcache = None
 
+    # true if url is in list.
     def check(self,url):
+        with self.sync_lock:
+            listcache = self._get_listcache()
+            return url in listcache
+        
+    def get_note(self,url):
         with self.sync_lock:
             listcache = self._get_listcache()
             if url in listcache:
-                note = listcache[url]
-                return note
-            
+                return listcache[url].note
             # not found
-            return None
+            return ''
+
+    def get_full_note(self,url):
+        with self.sync_lock:
+            listcache = self._get_listcache()
+            if url in listcache:
+                return listcache[url].fullnote()
+            # not found
+            return ''
 
     def remove(self,url):
         with self.sync_lock:
@@ -205,27 +92,26 @@ class RejectURLList:
                 self._save_list(listcache)
 
     def add_text(self,rejecttext,addreasontext):
-        self.add(self._read_list_from_text(rejecttext,addreasontext).items())
+        self.add(self._read_list_from_text(rejecttext,addreasontext).values())
             
     def add(self,rejectlist,clear=False):
-        # rejectlist=list of (url,note) tuples.
         with self.sync_lock:
             if clear:
-                listcache={}
+                listcache=OrderedDict()
             else:
                 listcache = self._get_listcache()
-            for (url,note) in rejectlist:
-                listcache[url]=note
+            for l in rejectlist:
+                listcache[l.url]=l
             self._save_list(listcache)
 
     def get_list(self):
-        return copy.deepcopy(self._get_listcache())
+        return self._get_listcache().values()
             
     def get_reject_reasons(self):
         return self.prefs['rejectreasons'].splitlines()
 
 rejecturllist = RejectURLList(prefs)
-    
+
 class ConfigWidget(QWidget):
 
     def __init__(self, plugin_action):
@@ -282,12 +168,14 @@ class ConfigWidget(QWidget):
         prefs['updatecover'] = self.basic_tab.updatecover.isChecked()
         prefs['updateepubcover'] = self.basic_tab.updateepubcover.isChecked()
         prefs['keeptags'] = self.basic_tab.keeptags.isChecked()
+        prefs['showmarked'] = self.basic_tab.showmarked.isChecked()
         prefs['urlsfromclip'] = self.basic_tab.urlsfromclip.isChecked()
         prefs['updatedefault'] = self.basic_tab.updatedefault.isChecked()
         prefs['deleteotherforms'] = self.basic_tab.deleteotherforms.isChecked()
         prefs['adddialogstaysontop'] = self.basic_tab.adddialogstaysontop.isChecked()
         prefs['includeimages'] = self.basic_tab.includeimages.isChecked()
         prefs['lookforurlinhtml'] = self.basic_tab.lookforurlinhtml.isChecked()
+        prefs['checkforseriesurlid'] = self.basic_tab.checkforseriesurlid.isChecked()
         prefs['injectseries'] = self.basic_tab.injectseries.isChecked()
 
         if self.readinglist_tab:
@@ -378,13 +266,16 @@ class BasicTab(QWidget):
         self.plugin_action = plugin_action
         QWidget.__init__(self)
         
-        self.l = QVBoxLayout()
-        self.setLayout(self.l)
+        topl = QVBoxLayout()
+        self.setLayout(topl)
 
         label = QLabel('These settings control the basic features of the plugin--downloading FanFiction.')
         label.setWordWrap(True)
-        self.l.addWidget(label)
-        self.l.addSpacing(5)
+        topl.addWidget(label)
+
+        defs_gb = groupbox = QGroupBox("Defaults Options on Download")
+        self.l = QVBoxLayout()
+        groupbox.setLayout(self.l)
 
         tooltip = "On each download, FFDL offers an option to select the output format. <br />This sets what that option will default to."
         horz = QHBoxLayout()
@@ -429,7 +320,9 @@ class BasicTab(QWidget):
         self.updateepubcover.setChecked(prefs['updateepubcover'])
         self.l.addWidget(self.updateepubcover)
 
-        self.l.addSpacing(10)        
+        cali_gb = groupbox = QGroupBox("Updating Calibre Options")
+        self.l = QVBoxLayout()
+        groupbox.setLayout(self.l)
 
         self.deleteotherforms = QCheckBox('Delete other existing formats?',self)
         self.deleteotherforms.setToolTip('Check this to automatically delete all other ebook formats when updating an existing book.\nHandy if you have both a Nook(epub) and Kindle(mobi), for example.')
@@ -446,7 +339,24 @@ class BasicTab(QWidget):
         self.keeptags.setChecked(prefs['keeptags'])
         self.l.addWidget(self.keeptags)
 
-        self.l.addSpacing(10)        
+        self.checkforseriesurlid = QCheckBox("Check for existing Series Anthology books?",self)
+        self.checkforseriesurlid.setToolTip("Check for existings Series Anthology books using each new story's series URL before downloading.\nOffer to skip downloading if a Series Anthology is found.")
+        self.checkforseriesurlid.setChecked(prefs['checkforseriesurlid'])
+        self.l.addWidget(self.checkforseriesurlid)
+
+        self.lookforurlinhtml = QCheckBox("Search EPUB text for Story URL?",self)
+        self.lookforurlinhtml.setToolTip("Look for first valid story URL inside EPUB text if not found in metadata.\nSomewhat risky, could find wrong URL depending on EPUB content.\nAlso finds and corrects bad ffnet URLs from ficsaver.com files.")
+        self.lookforurlinhtml.setChecked(prefs['lookforurlinhtml'])
+        self.l.addWidget(self.lookforurlinhtml)
+
+        self.showmarked = QCheckBox("Show added/updated books when finished?",self)
+        self.showmarked.setToolTip("Show added/updated books only when finished.\nYou can also manually search for 'marked:ffdl_success'.\n'marked:ffdl_failed' is also available, or search 'marked:ffdl' for both.")
+        self.showmarked.setChecked(prefs['showmarked'])
+        self.l.addWidget(self.showmarked)
+
+        gui_gb = groupbox = QGroupBox("GUI Options")
+        self.l = QVBoxLayout()
+        groupbox.setLayout(self.l)
 
         self.urlsfromclip = QCheckBox('Take URLs from Clipboard?',self)
         self.urlsfromclip.setToolTip('Prefill URLs from valid URLs in Clipboard when Adding New.')
@@ -464,7 +374,9 @@ class BasicTab(QWidget):
         self.adddialogstaysontop.setChecked(prefs['adddialogstaysontop'])
         self.l.addWidget(self.adddialogstaysontop)
 
-        self.l.addSpacing(10)        
+        misc_gb = groupbox = QGroupBox("Misc Options")
+        self.l = QVBoxLayout()
+        groupbox.setLayout(self.l)
 
         # this is a cheat to make it easier for users to realize there's a new include_images features.
         self.includeimages = QCheckBox("Include images in EPUBs?",self)
@@ -472,38 +384,43 @@ class BasicTab(QWidget):
         self.includeimages.setChecked(prefs['includeimages'])
         self.l.addWidget(self.includeimages)
 
-        self.lookforurlinhtml = QCheckBox("Search EPUB text for Story URL?",self)
-        self.lookforurlinhtml.setToolTip("Look for first valid story URL inside EPUB text if not found in metadata.\nSomewhat risky, could find wrong URL depending on EPUB content.\nAlso finds and corrects bad ffnet URLs from ficsaver.com files.")
-        self.lookforurlinhtml.setChecked(prefs['lookforurlinhtml'])
-        self.l.addWidget(self.lookforurlinhtml)
-
         self.injectseries = QCheckBox("Inject calibre Series when none found?",self)
         self.injectseries.setToolTip("If no series is found, inject the calibre series (if there is one) so it appears on the FFDL title page(not cover).")
         self.injectseries.setChecked(prefs['injectseries'])
         self.l.addWidget(self.injectseries)
 
-        self.l.addSpacing(10)        
+        rej_gb = groupbox = QGroupBox("Reject List")
+        self.l = QVBoxLayout()
+        groupbox.setLayout(self.l)
 
-        horz = QHBoxLayout()
-        
         self.rejectlist = QPushButton('Edit Reject URL List', self)
         self.rejectlist.setToolTip("Edit list of URLs FFDL will automatically Reject.")
         self.rejectlist.clicked.connect(self.show_rejectlist)
-        horz.addWidget(self.rejectlist)
+        self.l.addWidget(self.rejectlist)
         
         self.reject_urls = QPushButton('Add Reject URLs', self)
         self.reject_urls.setToolTip("Add additional URLs to Reject as text.")
         self.reject_urls.clicked.connect(self.add_reject_urls)
-        horz.addWidget(self.reject_urls)
+        self.l.addWidget(self.reject_urls)
         
         self.reject_reasons = QPushButton('Edit Reject Reasons List', self)
         self.reject_reasons.setToolTip("Customize the Reasons presented when Rejecting URLs")
         self.reject_reasons.clicked.connect(self.show_reject_reasons)
-        horz.addWidget(self.reject_reasons)
+        self.l.addWidget(self.reject_reasons)
+
+        topl.addWidget(defs_gb)
+
+        horz = QHBoxLayout()
+        topl.addLayout(horz)
+        horz.addWidget(cali_gb)
+        horz.addWidget(rej_gb)
         
-        self.l.addLayout(horz)
-        
-        self.l.insertStretch(-1)
+        horz = QHBoxLayout()
+        topl.addLayout(horz)
+        horz.addWidget(gui_gb)
+        horz.addWidget(misc_gb)
+
+        topl.insertStretch(-1)
         
     def set_collisions(self):
         prev=self.collision.currentText()
@@ -520,12 +437,8 @@ class BasicTab(QWidget):
         ShowDefaultsIniDialog(self.windowIcon(),text,self).exec_()
 
     def show_rejectlist(self):
-        rejectlist = []
-        for (url,note) in rejecturllist.get_list().items():
-            rejectlist.append((None,url,note,note))
-            
         d = RejectListDialog(self,
-                             rejectlist,
+                             rejecturllist.get_list(),
                              rejectreasons=rejecturllist.get_reject_reasons(),
                              header="Edit Reject URLs List",
                              show_delete=False,
@@ -535,11 +448,7 @@ class BasicTab(QWidget):
         if d.result() != d.Accepted:
             return
         
-        rejectlist=[]
-        for (bookid,url,note) in d.get_reject_list():
-            rejectlist.append((url,note))
-
-        rejecturllist.add(rejectlist,clear=True)
+        rejecturllist.add(d.get_reject_list(),clear=True)
         
     def show_reject_reasons(self):
         d = EditTextDialog(self,
@@ -554,11 +463,11 @@ class BasicTab(QWidget):
                 
     def add_reject_urls(self):
         d = EditTextDialog(self,
-                           "http://example.com?story.php?sid=5,Reason why I rejected it",
+                           "http://example.com/story.php?sid=5,Reason why I rejected it\nhttp://example.com/story.php?sid=6,Title by Author - Reason why I rejected it",
                            icon=self.windowIcon(),
                            title="Add Reject URLs",
-                           label="Add Reject URLs. Use: <b>http://...,note</b><br>Invalid story URLs will be ignored.",
-                           tooltip="One URL per line, everything after <b>,</b> will be put in the note.",
+                           label="Add Reject URLs. Use: <b>http://...,note</b> or <b>http://...,title by author - note</b><br>Invalid story URLs will be ignored.",
+                           tooltip="One URL per line:\n<b>http://...,note</b>\n<b>http://...,title by author - note</b>",
                            rejectreasons=rejecturllist.get_reject_reasons(),
                            reasonslabel='Add this reason to all URLs added:')
         d.exec_()

calibre-plugin/dialogs.py

@@ -7,29 +7,29 @@ __license__   = 'GPL v3'
 __copyright__ = '2011, Jim Miller'
 __docformat__ = 'restructuredtext en'
 
-import traceback
+import traceback, re
 from functools import partial
 
+import urllib
+import email
+
 from PyQt4 import QtGui
-from PyQt4.Qt import (QDialog, QTableWidget, QMessageBox, QVBoxLayout, QHBoxLayout,
-                      QGridLayout, QPushButton, QProgressDialog, QString, QLabel,
-                      QCheckBox, QIcon, QTextCursor, QTextEdit, QLineEdit, QInputDialog,
-                      QComboBox, QClipboard, QVariant, QProgressDialog, QTimer,
-                      QDialogButtonBox, QPixmap, Qt, QAbstractItemView, SIGNAL,
-                      QTableWidgetItem )
+from PyQt4.Qt import (QDialog, QTableWidget, QVBoxLayout, QHBoxLayout, QGridLayout,
+                      QPushButton, QString, QLabel, QCheckBox, QIcon, QLineEdit,
+                      QComboBox, QVariant, QProgressDialog, QTimer, QDialogButtonBox,
+                      QPixmap, Qt, QAbstractItemView, SIGNAL, QTextEdit, pyqtSignal,
+                      QGroupBox, QFrame)
 
-from calibre.gui2 import error_dialog, warning_dialog, question_dialog, info_dialog
 from calibre.gui2.dialogs.confirm_delete import confirm
 from calibre.gui2.complete2 import EditWithComplete
 
-from calibre import confirm_config_name
-from calibre.gui2 import dynamic
-
-from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapters,writers,exceptions
 from calibre_plugins.fanfictiondownloader_plugin.common_utils \
     import (ReadOnlyTableWidgetItem, ReadOnlyTextIconWidgetItem, SizePersistedDialog,
             ImageTitleLayout, get_icon)
 
+from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.geturls import get_urls_from_html, get_urls_from_text
+from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.adapters import getNormalStoryURL
+
 SKIP=u'Skip'
 ADDNEW=u'Add New Book'
 UPDATE=u'Update EPUB if New Chapters'
@@ -48,7 +48,65 @@ collision_order=[SKIP,
 anthology_collision_order=[UPDATE,
                            UPDATEALWAYS,
                            OVERWRITEALWAYS]
+
+gpstyle='QGroupBox {border:0; padding-top:10px; padding-bottom:0px; margin-bottom:0px;}' #  background-color:red;
+
+class RejectUrlEntry:
+
+    matchpat=re.compile(r"^(?P<url>[^,]+)(,(?P<fullnote>(((?P<title>.+) by (?P<auth>.+?)( - (?P<note>.+))?)|.*)))?$")
+    
+    def __init__(self,url_or_line,note=None,title=None,auth=None,
+                 addreasontext=None,fromline=False,book_id=None):
+
+        self.url=url_or_line
+        self.note=note
+        self.title=title
+        self.auth=auth
+        self.valid=False
+        self.book_id=book_id
+
+        if fromline:
+            mc = re.match(self.matchpat,url_or_line)
+            if mc:
+                #print("mc:%s"%mc.groupdict())
+                (url,title,auth,note) = mc.group('url','title','auth','note')
+                if not mc.group('title'):
+                    title=''
+                    auth=''
+                    note=mc.group('fullnote')
+                self.url=url
+                self.note=note
+                self.title=title
+                self.auth=auth
         
+        if not self.note:
+            if addreasontext:
+                self.note = addreasontext
+            else:
+                self.note = ''
+        else:
+            if addreasontext:
+                self.note = self.note + ' - ' + addreasontext
+                
+        self.url = getNormalStoryURL(self.url)
+        self.valid = self.url != None
+                
+    def to_line(self):
+        # always 'url,'
+        return self.url+","+self.fullnote()
+        
+    def fullnote(self):
+        retval = ""
+        if self.title and self.auth:
+            retval = retval + "%s by %s"%(self.title,self.auth)
+            if self.note:
+                retval = retval + " - "
+                
+        if self.note:
+            retval = retval + self.note
+            
+        return retval
+
 # This is a more than slightly kludgey way to get
 # EditWithComplete to *not* alpha-order the reasons, but leave
 # them in the order entered.  If
@@ -73,6 +131,36 @@ class NotGoingToDownload(Exception):
 class DroppableQTextEdit(QTextEdit):
     def __init__(self,parent):
         QTextEdit.__init__(self,parent)
+
+    def dropEvent(self,event):
+        # print("event:%s"%event)
+
+        mimetype='text/uri-list'
+
+        urllist=[]
+        filelist="%s"%event.mimeData().data(mimetype)
+        for f in filelist.splitlines():
+            #print("filename:%s"%f)
+            if f.endswith(".eml"):
+                fhandle = urllib.urlopen(f)
+                #print("file:\n%s\n\n"%fhandle.read())
+                msg = email.message_from_file(fhandle)
+                if msg.is_multipart():
+                    for part in msg.walk():
+                        #print("part type:%s"%part.get_content_type())
+                        if part.get_content_type() == "text/html":
+                            #print("URL list:%s"%get_urls_from_data(part.get_payload(decode=True)))
+                            urllist.extend(get_urls_from_html(part.get_payload(decode=True)))
+                        if part.get_content_type() == "text/plain":
+                            #print("part content:text/plain")
+                            # print("part content:%s"%part.get_payload(decode=True))
+                            urllist.extend(get_urls_from_text(part.get_payload(decode=True)))
+                else:
+                    urllist.extend(get_urls_from_text("%s"%msg))
+        
+        if urllist:
+            self.append("\n".join(urllist))
+        return QTextEdit.dropEvent(self,event)
         
     def canInsertFromMimeData(self, source):
         if source.hasUrls():
@@ -88,25 +176,11 @@ class DroppableQTextEdit(QTextEdit):
                             
 class AddNewDialog(SizePersistedDialog):
 
-    def __init__(self, gui, prefs, icon, url_list_text, merge=False, newmerge=False):
-        SizePersistedDialog.__init__(self, gui, 'FanFictionDownLoader plugin:add new dialog')
-        self.gui = gui
-        self.merge = merge
-        self.newmerge = newmerge
+    go_signal = pyqtSignal(object, object, object, object)
 
-        if merge:
-            labeltext = 'Story URL(s) for anthology, one per line:'
-            tooltiptext = 'URLs for stories to include in the anthology, one per line.\nWill take URLs from clipboard, but only valid URLs.'
-            collisiontext = 'If Story Already Exists in Anthology?'
-            collisiontooltip = "What to do if there's already an existing story with the same URL in the anthology."
-        else:
-            labeltext = 'Story URL(s), one per line:'
-            tooltiptext = 'URLs for stories, one per line.\nWill take URLs from clipboard, but only valid URLs.\nAdd [1,5] after the URL to limit the download to chapters 1-5.'
-            collisiontext = 'If Story Already Exists?'
-            collisiontooltip = "What to do if there's already an existing story with the same URL or title and author."
-            
-        if prefs['adddialogstaysontop']:
-            QDialog.setWindowFlags ( self, Qt.Dialog|Qt.WindowStaysOnTopHint )
+    def __init__(self, gui, prefs, icon):
+        SizePersistedDialog.__init__(self, gui, 'FanFictionDownLoader plugin:add new dialog')
+        self.prefs = prefs
         
         self.setMinimumWidth(300)
         self.l = QVBoxLayout()
@@ -115,69 +189,178 @@ class AddNewDialog(SizePersistedDialog):
         self.setWindowTitle('FanFictionDownLoader')
         self.setWindowIcon(icon)
 
-        self.l.addWidget(QLabel(labeltext))
+        self.toplabel=QLabel("Toplabel")
+        self.l.addWidget(self.toplabel)
         self.url = DroppableQTextEdit(self)
-        self.url.setToolTip(tooltiptext)
+        self.url.setToolTip("UrlTooltip")
         self.url.setLineWrapMode(QTextEdit.NoWrap)
-        self.url.setText(url_list_text)
         self.l.addWidget(self.url)
 
-        if not merge:
-            horz = QHBoxLayout()
-            label = QLabel('Output &Format:')
-            horz.addWidget(label)
-            self.fileform = QComboBox(self)
-            self.fileform.addItem('epub')
-            self.fileform.addItem('mobi')
-            self.fileform.addItem('html')
-            self.fileform.addItem('txt')
-            self.fileform.setCurrentIndex(self.fileform.findText(prefs['fileform']))
-            self.fileform.setToolTip('Choose output format to create.  May set default from plugin configuration.')
-            self.fileform.activated.connect(self.set_collisions)
-            
-            label.setBuddy(self.fileform)
-            horz.addWidget(self.fileform)
-            self.l.addLayout(horz)
-
-        if not newmerge:
-            horz = QHBoxLayout()
-            label = QLabel(collisiontext)
-            horz.addWidget(label)
-            self.collision = QComboBox(self)
-            self.collision.setToolTip(collisiontooltip)
-            # add collision options
-            self.set_collisions()
-            i = self.collision.findText(prefs['collision'])
-            if i > -1:
-                self.collision.setCurrentIndex(i)
-            label.setBuddy(self.collision)
-            horz.addWidget(self.collision)
-            self.l.addLayout(horz)
-    
-            horz = QHBoxLayout()
-            self.updatemeta = QCheckBox('Update Calibre &Metadata?',self)
-            self.updatemeta.setToolTip("Update metadata for existing stories in Calibre from web site?\n(Columns set to 'New Only' in the column tabs will only be set for new books.)")
-            self.updatemeta.setChecked(prefs['updatemeta'])
-            horz.addWidget(self.updatemeta)
-    
-            if not merge: # hide if anthology merge.
-                self.updateepubcover = QCheckBox('Update EPUB Cover?',self)
-                self.updateepubcover.setToolTip('Update book cover image from site or defaults (if found) <i>inside</i> the EPUB when EPUB is updated.')
-                self.updateepubcover.setChecked(prefs['updateepubcover'])
-                horz.addWidget(self.updateepubcover)
-            
-            self.l.addLayout(horz)
-
-        button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
-        button_box.accepted.connect(self.accept)
-        button_box.rejected.connect(self.reject)
-        self.l.addWidget(button_box)
+        self.merge = self.newmerge = False
         
-        if url_list_text:
-            button_box.button(QDialogButtonBox.Ok).setFocus()
+        # elements to hide when doing merge.
+        self.mergehide = []
+        # elements to show again when doing *update* merge
+        self.mergeupdateshow = []
 
+        self.groupbox = QGroupBox("Show Download Options")
+        self.groupbox.setCheckable(True)
+        self.groupbox.setChecked(False)
+        self.groupbox.setFlat(True)
+        print("style:%s"%self.groupbox.styleSheet())
+        self.groupbox.setStyleSheet(gpstyle);
+
+        self.gbf = QFrame()
+        self.gbl = QVBoxLayout()
+        self.gbl.addWidget(self.gbf)
+        self.groupbox.setLayout(self.gbl)
+        self.gbl = QVBoxLayout()
+        self.gbf.setLayout(self.gbl)
+        self.l.addWidget(self.groupbox)
+
+        self.gbf.setVisible(False)
+        self.groupbox.toggled.connect(self.gbf.setVisible)
+        
+        horz = QHBoxLayout()
+        label = QLabel('Output &Format:')
+        self.mergehide.append(label)
+        
+        self.fileform = QComboBox(self)
+        self.fileform.addItem('epub')
+        self.fileform.addItem('mobi')
+        self.fileform.addItem('html')
+        self.fileform.addItem('txt')
+        self.fileform.setToolTip('Choose output format to create.  May set default from plugin configuration.')
+        self.fileform.activated.connect(self.set_collisions)
+        
+        horz.addWidget(label)
+        label.setBuddy(self.fileform)
+        horz.addWidget(self.fileform)
+        self.gbl.addLayout(horz)
+        self.mergehide.append(self.fileform)
+
+        horz = QHBoxLayout()
+        self.collisionlabel = QLabel("CollisionLabel")
+        horz.addWidget(self.collisionlabel)
+        self.collision = QComboBox(self)
+        self.collision.setToolTip("CollisionToolTip")
+        # add collision options
+        self.set_collisions()
+        i = self.collision.findText(prefs['collision'])
+        if i > -1:
+            self.collision.setCurrentIndex(i)
+        self.collisionlabel.setBuddy(self.collision)
+        horz.addWidget(self.collision)
+        self.gbl.addLayout(horz)
+        self.mergehide.append(self.collisionlabel)
+        self.mergehide.append(self.collision)
+        self.mergeupdateshow.append(self.collisionlabel)
+        self.mergeupdateshow.append(self.collision)
+
+        horz = QHBoxLayout()
+        self.updatemeta = QCheckBox('Update Calibre &Metadata?',self)
+        self.updatemeta.setToolTip("Update metadata for existing stories in Calibre from web site?\n(Columns set to 'New Only' in the column tabs will only be set for new books.)")
+        self.updatemeta.setChecked(prefs['updatemeta'])
+        horz.addWidget(self.updatemeta)
+        self.mergehide.append(self.updatemeta)
+        self.mergeupdateshow.append(self.updatemeta)
+
+        self.updateepubcover = QCheckBox('Update EPUB Cover?',self)
+        self.updateepubcover.setToolTip('Update book cover image from site or defaults (if found) <i>inside</i> the EPUB when EPUB is updated.')
+        self.updateepubcover.setChecked(prefs['updateepubcover'])
+        horz.addWidget(self.updateepubcover)
+        self.mergehide.append(self.updateepubcover)
+        
+        self.gbl.addLayout(horz)
+
+        self.button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
+        self.button_box.accepted.connect(self.ok_clicked)
+        self.button_box.rejected.connect(self.reject)
+        self.l.addWidget(self.button_box)
+
+    # invoke the 
+    def ok_clicked(self):
+        self.dialog_closing(None) # save persistent size.
+        self.hide()
+        self.go_signal.emit( self.get_ffdl_options(),
+                             self.get_urlstext(),
+                             self.merge,
+                             self.extrapayload )
+
+    def show_dialog(self,
+                    url_list_text,
+                    callback,
+                    show=True,
+                    merge=False,
+                    newmerge=True,
+                    extraoptions={},
+                    extrapayload=None):
+        # rather than mutex in ffdl_plugin, just bail here if it's
+        # already in use.
+        if self.isVisible(): return
+
+        try:
+            self.go_signal.disconnect()
+        except:
+            pass # if not already connected.
+        self.go_signal.connect(callback)
+
+        self.merge = merge
+        self.newmerge = newmerge
+        self.extraoptions = extraoptions
+        self.extrapayload = extrapayload
+
+        self.groupbox.setVisible(not(self.merge and self.newmerge))
+        
+        if self.merge:
+            self.toplabel.setText('Story URL(s) for anthology, one per line:')
+            self.url.setToolTip('URLs for stories to include in the anthology, one per line.\nWill take URLs from clipboard, but only valid URLs.')
+            self.collisionlabel.setText('If Story Already Exists in Anthology?')
+            self.collision.setToolTip("What to do if there's already an existing story with the same URL in the anthology.")
+            for widget in self.mergehide:
+                widget.setVisible(False)
+            if not self.newmerge:
+                for widget in self.mergeupdateshow:
+                    widget.setVisible(True)
+        else:
+            for widget in self.mergehide:
+                widget.setVisible(True)
+            self.toplabel.setText('Story URL(s), one per line:')
+            self.url.setToolTip('URLs for stories, one per line.\nWill take URLs from clipboard, but only valid URLs.\nAdd [1,5] after the URL to limit the download to chapters 1-5.')
+            self.collisionlabel.setText('If Story Already Exists?')
+            self.collision.setToolTip("What to do if there's already an existing story with the same URL or title and author.")
+
+        # Need to re-able after hiding/showing
+        self.setAcceptDrops(True)
+        self.url.setFocus()
+            
+        if self.prefs['adddialogstaysontop']:
+            QDialog.setWindowFlags ( self, Qt.Dialog | Qt.WindowStaysOnTopHint )
+        else:
+            QDialog.setWindowFlags ( self, Qt.Dialog )
+
+        if not self.merge:
+            self.fileform.setCurrentIndex(self.fileform.findText(self.prefs['fileform']))
+
+        # add collision options
+        self.set_collisions()
+        
+        i = self.collision.findText(self.prefs['collision'])
+        if i > -1:
+            self.collision.setCurrentIndex(i)
+        self.updatemeta.setChecked(self.prefs['updatemeta'])
+            
+        if not self.merge:
+            self.updateepubcover.setChecked(self.prefs['updateepubcover'])
+                
+        self.url.setText(url_list_text)
+        if url_list_text:
+            self.button_box.button(QDialogButtonBox.Ok).setFocus()
         # restore saved size.
         self.resize_dialog()
+        
+        if show: # so anthology update can be modal still.
+            self.show()
         #self.resize(self.sizeHint())
 
     def set_collisions(self):
@@ -195,27 +378,21 @@ class AddNewDialog(SizePersistedDialog):
             self.collision.setCurrentIndex(i)
         
     def get_ffdl_options(self):
+        retval =  {
+            'fileform': unicode(self.fileform.currentText()),
+            'collision': unicode(self.collision.currentText()),
+            'updatemeta': self.updatemeta.isChecked(),
+            'updateepubcover': self.updateepubcover.isChecked(),
+                }
+        
         if self.merge:
+            retval['fileform']=='epub'
+            retval['updateepubcover']=True
             if self.newmerge:
-                updatemeta=True
-                collision=ADDNEW
-            else:
-                updatemeta=self.updatemeta.isChecked()
-                collision=unicode(self.collision.currentText())
-                
-            return {
-                'fileform': 'epub',
-                'collision': collision,
-                'updatemeta': updatemeta,
-                'updateepubcover': True,
-                }
-        else:
-            return {
-                'fileform': unicode(self.fileform.currentText()),
-                'collision': unicode(self.collision.currentText()),
-                'updatemeta': self.updatemeta.isChecked(),
-                'updateepubcover': self.updateepubcover.isChecked(),
-                }
+                retval['updatemeta']=True
+                retval['collision']=ADDNEW
+            
+        return dict(retval.items() + self.extraoptions.items() )
 
     def get_urlstext(self):
         return unicode(self.url.toPlainText())
@@ -234,7 +411,6 @@ class CollectURLDialog(SizePersistedDialog):
     '''
     def __init__(self, gui, title, url_text, epubmerge_plugin=None): 
         SizePersistedDialog.__init__(self, gui, 'FanFictionDownLoader plugin:get story urls')
-        self.gui = gui
         self.status=False
         self.anthology=False
 
@@ -288,7 +464,6 @@ class UserPassDialog(QDialog):
     '''
     def __init__(self, gui, site, exception=None):
         QDialog.__init__(self, gui)
-        self.gui = gui
         self.status=False
 
         self.l = QGridLayout()
@@ -347,7 +522,6 @@ class LoopProgressDialog(QProgressDialog):
                                  QString(), 0, len(book_list), gui)
         self.setWindowTitle(win_title)
         self.setMinimumWidth(500)
-        self.gui = gui
         self.book_list = book_list
         self.foreach_function = foreach_function
         self.finish_function = finish_function
@@ -396,7 +570,6 @@ class LoopProgressDialog(QProgressDialog):
 
     def do_when_finished(self):
         self.hide()
-        self.gui = None        
         # Queues a job to process these books in the background.
         self.finish_function(self.book_list)
 
@@ -445,7 +618,6 @@ class UpdateExistingDialog(SizePersistedDialog):
     def __init__(self, gui, header, prefs, icon, books,
                  save_size_name='fanfictiondownloader_plugin:update list dialog'):
         SizePersistedDialog.__init__(self, gui, save_size_name)
-        self.gui = gui
         
         self.setWindowTitle(header)
         self.setWindowIcon(icon)
@@ -463,11 +635,7 @@ class UpdateExistingDialog(SizePersistedDialog):
 
         button_layout = QVBoxLayout()
         books_layout.addLayout(button_layout)
-        # self.move_up_button = QtGui.QToolButton(self)
-        # self.move_up_button.setToolTip('Move selected books up the list')
-        # self.move_up_button.setIcon(QIcon(I('arrow-up.png')))
-        # self.move_up_button.clicked.connect(self.books_table.move_rows_up)
-        # button_layout.addWidget(self.move_up_button)
+
         spacerItem = QtGui.QSpacerItem(20, 40, QtGui.QSizePolicy.Minimum, QtGui.QSizePolicy.Expanding)
         button_layout.addItem(spacerItem)
         self.remove_button = QtGui.QToolButton(self)
@@ -477,16 +645,28 @@ class UpdateExistingDialog(SizePersistedDialog):
         button_layout.addWidget(self.remove_button)
         spacerItem1 = QtGui.QSpacerItem(20, 40, QtGui.QSizePolicy.Minimum, QtGui.QSizePolicy.Expanding)
         button_layout.addItem(spacerItem1)
-        # self.move_down_button = QtGui.QToolButton(self)
-        # self.move_down_button.setToolTip('Move selected books down the list')
-        # self.move_down_button.setIcon(QIcon(I('arrow-down.png')))
-        # self.move_down_button.clicked.connect(self.books_table.move_rows_down)
-        # button_layout.addWidget(self.move_down_button)
 
         options_layout = QHBoxLayout()
 
+        groupbox = QGroupBox("Show Download Options")
+        groupbox.setCheckable(True)
+        groupbox.setChecked(False)
+        groupbox.setFlat(True)
+        groupbox.setStyleSheet(gpstyle)
+
+        gbf = QFrame()
+        gbl = QVBoxLayout()
+        gbl.addWidget(gbf)
+        groupbox.setLayout(gbl)
+        gbl = QHBoxLayout()
+        gbf.setLayout(gbl)
+        options_layout.addWidget(groupbox)
+
+        gbf.setVisible(False)
+        groupbox.toggled.connect(gbf.setVisible)
+        
         label = QLabel('Output &Format:')
-        options_layout.addWidget(label)
+        gbl.addWidget(label)
         self.fileform = QComboBox(self)
         self.fileform.addItem('epub')
         self.fileform.addItem('mobi')
@@ -496,10 +676,10 @@ class UpdateExistingDialog(SizePersistedDialog):
         self.fileform.setToolTip('Choose output format to create.  May set default from plugin configuration.')
         self.fileform.activated.connect(self.set_collisions)
         label.setBuddy(self.fileform)
-        options_layout.addWidget(self.fileform)
+        gbl.addWidget(self.fileform)
         
         label = QLabel('Update Mode:')
-        options_layout.addWidget(label)
+        gbl.addWidget(label)
         self.collision = QComboBox(self)
         self.collision.setToolTip("What sort of update to perform.  May set default from plugin configuration.")
         # add collision options
@@ -507,19 +687,20 @@ class UpdateExistingDialog(SizePersistedDialog):
         i = self.collision.findText(prefs['collision'])
         if i > -1:
             self.collision.setCurrentIndex(i)
-        # self.collision.setToolTip('Overwrite will replace the existing story.  Add New will create a new story with the same title and author.')
         label.setBuddy(self.collision)
-        options_layout.addWidget(self.collision)
+        gbl.addWidget(self.collision)
 
         self.updatemeta = QCheckBox('Update Calibre &Metadata?',self)
         self.updatemeta.setToolTip("Update metadata for existing stories in Calibre from web site?\n(Columns set to 'New Only' in the column tabs will only be set for new books.)")
         self.updatemeta.setChecked(prefs['updatemeta'])
-        options_layout.addWidget(self.updatemeta)
+        gbl.addWidget(self.updatemeta)
                 
         self.updateepubcover = QCheckBox('Update EPUB Cover?',self)
         self.updateepubcover.setToolTip('Update book cover image from site or defaults (if found) <i>inside</i> the EPUB when EPUB is updated.')
         self.updateepubcover.setChecked(prefs['updateepubcover'])
-        options_layout.addWidget(self.updateepubcover)
+        gbl.addWidget(self.updateepubcover)
+
+
         
         button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
         button_box.accepted.connect(self.accept)
@@ -557,84 +738,6 @@ class UpdateExistingDialog(SizePersistedDialog):
             'updateepubcover': self.updateepubcover.isChecked(),
             }
 
-def display_story_list(gui, header, prefs, icon, books,
-                       label_text='',
-                       save_size_name='fanfictiondownloader_plugin:display list dialog',
-                       offer_skip=False):
-    all_good = True
-    for b in books:
-        if not b['good']:
-            all_good=False
-            break
-        
-    ## 
-    if all_good and not dynamic.get(confirm_config_name(save_size_name), True):
-        return True
-        pass
-            ## fake accept?
-    d = DisplayStoryListDialog(gui, header, prefs, icon, books,
-                               label_text,
-                               save_size_name,
-                               offer_skip and all_good)
-    d.exec_()
-    return d.result() == d.Accepted
-
-class DisplayStoryListDialog(SizePersistedDialog):
-    def __init__(self, gui, header, prefs, icon, books,
-                 label_text='',
-                 save_size_name='fanfictiondownloader_plugin:display list dialog',
-                 offer_skip=False):
-        SizePersistedDialog.__init__(self, gui, save_size_name)
-        self.name = save_size_name
-        self.gui = gui
-        
-        self.setWindowTitle(header)
-        self.setWindowIcon(icon)
-        
-        layout = QVBoxLayout(self)
-        self.setLayout(layout)
-        title_layout = ImageTitleLayout(self, 'images/icon.png',
-                                        header)
-        layout.addLayout(title_layout)
-
-        self.books_table = StoryListTableWidget(self)
-        layout.addWidget(self.books_table)
-
-        options_layout = QHBoxLayout()
-        self.label = QLabel(label_text)
-        #self.label.setOpenExternalLinks(True)
-        #self.label.setWordWrap(True)
-        options_layout.addWidget(self.label)
-        
-        if offer_skip:
-            spacerItem1 = QtGui.QSpacerItem(2, 4, QtGui.QSizePolicy.Expanding, QtGui.QSizePolicy.Minimum)
-            options_layout.addItem(spacerItem1)
-            self.again = QCheckBox('Show this again?',self)
-            self.again.setChecked(True)
-            self.again.stateChanged.connect(self.toggle)
-            self.again.setToolTip('Uncheck to skip review and update stories immediately when no problems.')
-            options_layout.addWidget(self.again)
-        
-        button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
-        button_box.accepted.connect(self.accept)
-        button_box.rejected.connect(self.reject)
-
-        options_layout.addWidget(button_box)
-        
-        layout.addLayout(options_layout)
-        
-        # Cause our dialog size to be restored from prefs or created on first usage
-        self.resize_dialog()
-        self.books_table.populate_table(books)
-
-    def get_books(self):
-        return self.books_table.get_books()
-
-    def toggle(self, *args):
-        dynamic[confirm_config_name(self.name)] = self.again.isChecked()
-
-    
-
 class StoryListTableWidget(QTableWidget):
 
     def __init__(self, parent):
@@ -694,11 +797,9 @@ class StoryListTableWidget(QTableWidget):
         self.setItem(row, 2, AuthorTableWidgetItem(", ".join(book['author']), ", ".join(book['author_sort'])))
         
         url_cell = ReadOnlyTableWidgetItem(book['url'])
-        #url_cell.setData(Qt.UserRole, QVariant(book['url']))
         self.setItem(row, 3, url_cell)
         
         comment_cell = ReadOnlyTableWidgetItem(book['comment'])
-        #comment_cell.setData(Qt.UserRole, QVariant(book))
         self.setItem(row, 4, comment_cell)
 
     def get_books(self):
@@ -732,56 +833,6 @@ class StoryListTableWidget(QTableWidget):
         self.selectRow(row)
         self.scrollToItem(self.currentItem())
 
-    def move_rows_up(self):
-        self.setFocus()
-        rows = self.selectionModel().selectedRows()
-        if len(rows) == 0:
-            return
-        first_sel_row = rows[0].row()
-        if first_sel_row <= 0:
-            return
-        # Workaround for strange selection bug in Qt which "alters" the selection
-        # in certain circumstances which meant move down only worked properly "once"
-        selrows = []
-        for row in rows:
-            selrows.append(row.row())
-        selrows.sort()
-        for selrow in selrows:
-            self.swap_row_widgets(selrow - 1, selrow + 1)
-        scroll_to_row = first_sel_row - 1
-        if scroll_to_row > 0:
-            scroll_to_row = scroll_to_row - 1
-        self.scrollToItem(self.item(scroll_to_row, 0))
-
-    def move_rows_down(self):
-        self.setFocus()
-        rows = self.selectionModel().selectedRows()
-        if len(rows) == 0:
-            return
-        last_sel_row = rows[-1].row()
-        if last_sel_row == self.rowCount() - 1:
-            return
-        # Workaround for strange selection bug in Qt which "alters" the selection
-        # in certain circumstances which meant move down only worked properly "once"
-        selrows = []
-        for row in rows:
-            selrows.append(row.row())
-        selrows.sort()
-        for selrow in reversed(selrows):
-            self.swap_row_widgets(selrow + 2, selrow)
-        scroll_to_row = last_sel_row + 1
-        if scroll_to_row < self.rowCount() - 1:
-            scroll_to_row = scroll_to_row + 1
-        self.scrollToItem(self.item(scroll_to_row, 0))
-
-    def swap_row_widgets(self, src_row, dest_row):
-        self.blockSignals(True)
-        self.insertRow(dest_row)
-        for col in range(0, self.columnCount()):
-            self.setItem(dest_row, col, self.takeItem(src_row, col))
-        self.removeRow(src_row)
-        self.blockSignals(False)
-
 class RejectListTableWidget(QTableWidget):
 
     def __init__(self, parent,rejectreasons=[]):
@@ -789,85 +840,55 @@ class RejectListTableWidget(QTableWidget):
         self.setSelectionBehavior(QAbstractItemView.SelectRows)
         self.rejectreasons = rejectreasons
 
-    def on_headersection_clicked(self):
-        self.setSortingEnabled(True)
-        
     def populate_table(self, reject_list):
         self.clear()
         self.setAlternatingRowColors(True)
         self.setRowCount(len(reject_list))
-        header_labels = ['URL', 'Note']
+        header_labels = ['URL', 'Title', 'Author', 'Note']
         self.setColumnCount(len(header_labels))
         self.setHorizontalHeaderLabels(header_labels)
         self.horizontalHeader().setStretchLastSection(True)
         #self.verticalHeader().setDefaultSectionSize(24)
         self.verticalHeader().hide()
 
-        # need sortingEnbled to sort, but off to up & down.
-        self.connect(self.horizontalHeader(),
-                     SIGNAL('sectionClicked(int)'),
-                     self.on_headersection_clicked)
-
+        # it's generally recommended to enable sort after pop, not
+        # before.  But then it needs to be sorted on a column and I'd
+        # rather keep the order given.
+        self.setSortingEnabled(True)
         # row is just row number.
         for row, rejectrow in enumerate(reject_list):
+            #print("populating table:%s"%rejectrow.to_line())
             self.populate_table_row(row,rejectrow)
 
         self.resizeColumnsToContents()
-        self.setMinimumColumnWidth(1, 100)
-        self.setMinimumColumnWidth(2, 100)
+        self.setMinimumColumnWidth(0, 100)
+        self.setMinimumColumnWidth(3, 100)
         self.setMinimumSize(300, 0)
 
     def setMinimumColumnWidth(self, col, minimum):
         if self.columnWidth(col) < minimum:
             self.setColumnWidth(col, minimum)
 
-    def populate_table_row(self, row, rejectrow):
+    def populate_table_row(self, row, rej):
 
-        (bookid,url,titleauth,oldrejnote) = rejectrow
-        if oldrejnote:
-            noteprefix = note = oldrejnote
-            # incase the existing note ends with one of the known reasons.
-            for reason in self.rejectreasons:
-                if noteprefix.endswith(' - '+reason):
-                    noteprefix = noteprefix[:-len(' - '+reason)]
-                    break
-        else:
-            noteprefix = note = titleauth
-            
-        if len(noteprefix) > 0:
-            noteprefix = noteprefix+' - '
-            
-        url_cell = ReadOnlyTableWidgetItem(url)
-        url_cell.setData(Qt.UserRole, QVariant(bookid))
-        url_cell.setToolTip('URL to add to the Reject List.')
+        url_cell = ReadOnlyTableWidgetItem(rej.url)
+        url_cell.setData(Qt.UserRole, QVariant(rej.book_id))
         self.setItem(row, 0, url_cell)
-
+        self.setItem(row, 1, ReadOnlyTableWidgetItem(rej.title))
+        self.setItem(row, 2, ReadOnlyTableWidgetItem(rej.auth))
+        
         note_cell = EditWithComplete(self)
-
         note_cell.lineEdit().mcompleter.model().set_items = \
             partial(complete_model_set_items_kludge,
                     note_cell.lineEdit().mcompleter.model())
         
-        items = [note]+[ noteprefix+x for x in self.rejectreasons ]
+        items = [rej.note]+self.rejectreasons
         note_cell.update_items_cache(items)
-        note_cell.show_initial_value(note)
+        note_cell.show_initial_value(rej.note)
         note_cell.set_separator(None)
         note_cell.setToolTip('Select or Edit Reject Note.')
-        self.setCellWidget(row, 1, note_cell)
+        self.setCellWidget(row, 3, note_cell)
         
-        # note_cell = QTableWidgetItem(note)
-        # note_cell.setToolTip('Double-click to edit note.')
-        # self.setItem(row, 1, note_cell)
-      
-    def get_reject_list(self):
-        rejectrows = []
-        for row in range(self.rowCount()):
-            bookid = self.item(row, 0).data(Qt.UserRole).toPyObject()
-            url = unicode(self.item(row, 0).text())
-            note = unicode(self.cellWidget(row, 1).currentText()).strip()
-            rejectrows.append((bookid,url,note))
-        return rejectrows
-
     def remove_selected_rows(self):
         self.setFocus()
         rows = self.selectionModel().selectedRows()
@@ -890,57 +911,6 @@ class RejectListTableWidget(QTableWidget):
         self.selectRow(row)
         self.scrollToItem(self.currentItem())
 
-    def move_rows_up(self):
-        self.setFocus()
-        rows = self.selectionModel().selectedRows()
-        if len(rows) == 0:
-            return
-        first_sel_row = rows[0].row()
-        if first_sel_row <= 0:
-            return
-        # Workaround for strange selection bug in Qt which "alters" the selection
-        # in certain circumstances which meant move down only worked properly "once"
-        selrows = []
-        for row in rows:
-            selrows.append(row.row())
-        selrows.sort()
-        for selrow in selrows:
-            self.swap_row_widgets(selrow - 1, selrow + 1)
-        scroll_to_row = first_sel_row - 1
-        if scroll_to_row > 0:
-            scroll_to_row = scroll_to_row - 1
-        self.scrollToItem(self.item(scroll_to_row, 0))
-
-    def move_rows_down(self):
-        self.setFocus()
-        rows = self.selectionModel().selectedRows()
-        if len(rows) == 0:
-            return
-        last_sel_row = rows[-1].row()
-        if last_sel_row == self.rowCount() - 1:
-            return
-        # Workaround for strange selection bug in Qt which "alters" the selection
-        # in certain circumstances which meant move down only worked properly "once"
-        selrows = []
-        for row in rows:
-            selrows.append(row.row())
-        selrows.sort()
-        for selrow in reversed(selrows):
-            self.swap_row_widgets(selrow + 2, selrow)
-        scroll_to_row = last_sel_row + 1
-        if scroll_to_row < self.rowCount() - 1:
-            scroll_to_row = scroll_to_row + 1
-        self.scrollToItem(self.item(scroll_to_row, 0))
-
-    def swap_row_widgets(self, src_row, dest_row):
-        self.blockSignals(True)
-        self.setSortingEnabled(False)
-        self.insertRow(dest_row)
-        for col in range(0, self.columnCount()):
-            self.setItem(dest_row, col, self.takeItem(src_row, col))
-        self.removeRow(src_row)
-        self.blockSignals(False)
-
 class RejectListDialog(SizePersistedDialog):
     def __init__(self, gui, reject_list,
                  rejectreasons=[],
@@ -950,7 +920,6 @@ class RejectListDialog(SizePersistedDialog):
                  show_all_reasons=True,
                  save_size_name='ffdl:reject list dialog'):
         SizePersistedDialog.__init__(self, gui, save_size_name)
-        self.gui = gui
       
         self.setWindowTitle(header)
         self.setWindowIcon(get_icon(icon))
@@ -970,21 +939,13 @@ class RejectListDialog(SizePersistedDialog):
         rejects_layout.addLayout(button_layout)
         spacerItem = QtGui.QSpacerItem(20, 40, QtGui.QSizePolicy.Minimum, QtGui.QSizePolicy.Expanding)
         button_layout.addItem(spacerItem)
-        # self.move_up_button = QtGui.QToolButton(self)
-        # self.move_up_button.setToolTip('Move selected books up the list')
-        # self.move_up_button.setIcon(QIcon(I('arrow-up.png')))
-        # self.move_up_button.clicked.connect(self.books_table.move_rows_up)
-        # button_layout.addWidget(self.move_up_button)
+        
         self.remove_button = QtGui.QToolButton(self)
         self.remove_button.setToolTip('Remove selected URL(s) from the list')
         self.remove_button.setIcon(get_icon('list_remove.png'))
         self.remove_button.clicked.connect(self.remove_from_list)
         button_layout.addWidget(self.remove_button)
-        # self.move_down_button = QtGui.QToolButton(self)
-        # self.move_down_button.setToolTip('Move selected books down the list')
-        # self.move_down_button.setIcon(QIcon(I('arrow-down.png')))
-        # self.move_down_button.clicked.connect(self.books_table.move_rows_down)
-        # button_layout.addWidget(self.move_down_button)
+
         spacerItem1 = QtGui.QSpacerItem(20, 40, QtGui.QSizePolicy.Minimum, QtGui.QSizePolicy.Expanding)
         button_layout.addItem(spacerItem1)
 
@@ -1031,10 +992,30 @@ class RejectListDialog(SizePersistedDialog):
         self.rejects_table.remove_selected_rows()
 
     def get_reject_list(self):
-        return self.rejects_table.get_reject_list()
+        rejectrows = []
+        for row in range(self.rejects_table.rowCount()):
+            url = unicode(self.rejects_table.item(row, 0).text()).strip()
+            book_id = self.rejects_table.item(row, 0).data(Qt.UserRole).toPyObject()
+            title = unicode(self.rejects_table.item(row, 1).text()).strip()
+            auth = unicode(self.rejects_table.item(row, 2).text()).strip()
+            note = unicode(self.rejects_table.cellWidget(row, 3).currentText()).strip()
+            rejectrows.append(RejectUrlEntry(url,note,title,auth,self.get_reason_text(),book_id=book_id))
+        return rejectrows
+
+    def get_reject_list_ids(self):
+        rejectrows = []
+        for row in range(self.rejects_table.rowCount()):
+            book_id = self.rejects_table.item(row, 0).data(Qt.UserRole).toPyObject()
+            if book_id:
+                rejectrows.append(book_id)
+        return rejectrows
 
     def get_reason_text(self):
-        return unicode(self.reason_edit.currentText()).strip()
+        try:
+            return unicode(self.reason_edit.currentText()).strip()
+        except:
+            # doesn't have self.reason_edit when editing existing list.
+            return None
     
     def get_deletebooks(self):
         return self.deletebooks.isChecked()

calibre-plugin/ffdl_plugin.py

@@ -41,12 +41,13 @@ from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.epubutils impo
 from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.geturls import get_urls_from_page
 
 from calibre_plugins.fanfictiondownloader_plugin.ffdl_util import (get_ffdl_adapter, get_ffdl_config, get_ffdl_personalini)
-from calibre_plugins.fanfictiondownloader_plugin.config import (prefs, permitted_values, rejecturllist)
+from calibre_plugins.fanfictiondownloader_plugin.config import (permitted_values, rejecturllist)
+from calibre_plugins.fanfictiondownloader_plugin.prefs import prefs
 from calibre_plugins.fanfictiondownloader_plugin.dialogs import (
-    AddNewDialog, UpdateExistingDialog, display_story_list, DisplayStoryListDialog,
+    AddNewDialog, UpdateExistingDialog,
     LoopProgressDialog, UserPassDialog, AboutDialog, CollectURLDialog, RejectListDialog,
     OVERWRITE, OVERWRITEALWAYS, UPDATE, UPDATEALWAYS, ADDNEW, SKIP, CALIBREONLY,
-    NotGoingToDownload )
+    NotGoingToDownload, RejectUrlEntry )
 
 # because calibre immediately transforms html into zip and don't want
 # to have an 'if html'.  db.has_format is cool with the case mismatch,
@@ -101,7 +102,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         # are not found in the zip file will result in null QIcons.
         icon = get_icon('images/icon.png')
 
-        #self.qaction.setText('FFDL')
+        self.qaction.setText('FanFictionDL')
         
         # The qaction is automatically created from the action_spec defined
         # above
@@ -125,6 +126,10 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         # been displayed once.
         self.rebuild_menus()
 
+        self.add_new_dialog = AddNewDialog(self.gui,
+                                           prefs,
+                                           self.qaction.icon())
+
     ## Kludgey, yes, but with the real configuration inside the
     ## library now, how else would a user be able to change this
     ## setting if it's crashing calibre?
@@ -135,7 +140,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
             file_path = os.path.join(calibre_config_dir,
                                      *("plugins/fanfictiondownloader_macmenuhack.txt".split('/')))
             file_path = os.path.abspath(file_path)
-            print("macmenuhack file_path:%s"%file_path)
+            print("Plugin %s macmenuhack file_path:%s"%(self.name,file_path))
             self.macmenuhack = os.access(file_path, os.F_OK)
             return self.macmenuhack
 
@@ -398,17 +403,17 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
             # want title/author, too, for rejects.
             self.populate_book_from_calibre_id(book,db)
         if book['url']:
-            # get existing note, if there is one.
-            book['oldrejnote']=rejecturllist.check(book['url'])
+            # get existing note, if on rejected list.
+            book['oldrejnote']=rejecturllist.get_note(book['url'])
             
     def reject_list_urls_finish(self, book_list):
 
-        # construct reject list of tuples:
-        # (calibre_id, url, "title, authors", old reject note).
-        reject_list = [ ( x['calibre_id'],x['url'],
-                          "%s by %s"%(x['title'],
-                                      ', '.join(x['author'])),
-                          x['oldrejnote'])
+        # construct reject list of objects
+        reject_list = [ RejectUrlEntry(x['url'],
+                                       x['oldrejnote'],
+                                       x['title'],
+                                       ', '.join(x['author']),
+                                       book_id=x['calibre_id'])
                         for x in book_list if x['good'] ]
         if reject_list:
             d = RejectListDialog(self.gui,reject_list,
@@ -418,62 +423,31 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
             if d.result() != d.Accepted:
                 return
 
-            bookids=[]
-            rejectlist=[]
-            addreasontext=d.get_reason_text()
-            for (bookid,url,note) in d.get_reject_list():
-                bookids.append(bookid)
-                if addreasontext and note:
-                    note = note +" - "+addreasontext
-                elif addreasontext:
-                    note = addreasontext
-                rejectlist.append((url,note))
-                print("Adding (%s) to Reject List: %s"%(url,note))
-                
-            rejecturllist.add(rejectlist)
+            rejecturllist.add(d.get_reject_list())
 
             if d.get_deletebooks():
-                self.gui.iactions['Remove Books'].delete_books()
+                self.gui.iactions['Remove Books'].do_library_delete(d.get_reject_list_ids())
             
         else:
             message="<p>Rejecting FFDL URLs: None of the books selected have FanFiction URLs.</p><p>Proceed to Remove?</p>"
             if confirm(message,'fanfictiondownloader_reject_non_fanfiction', self.gui):
                 self.gui.iactions['Remove Books'].delete_books()
-                
-    def add_dialog(self,url_list_text=None,merge=False,anthology_url=None):
 
-        #print("add_dialog()")
+    def add_dialog(self,url_list_text=None,merge=False,anthology_url=None):
+        'Both new individual stories and new anthologies are created here.'
 
         if not url_list_text:
             url_list = self.get_urls_clip()
             url_list_text = "\n".join(url_list)
         
-        # self.gui is the main calibre GUI. It acts as the gateway to access
-        # all the elements of the calibre user interface, it should also be the
-        # parent of the dialog
-        # AddNewDialog just collects URLs, format and presents buttons.
-        d = AddNewDialog(self.gui,
-                         prefs,
-                         self.qaction.icon(),
-                         url_list_text,
-                         merge=merge,
-                         newmerge=merge # if here, it's a new anthology.
-                         )
-        d.exec_()
-        if d.result() != d.Accepted:
-            return
-        
-        url_list = split_text_to_urls(d.get_urlstext())
-        add_books = self.convert_urls_to_books(url_list)
-        #print("add_books:%s"%add_books)
-        #print("options:%s"%d.get_ffdl_options())
-
-        options = d.get_ffdl_options()
-        options['version'] = self.version
-        options['anthology_url']=anthology_url
-        print(self.version)
-        
-        self.prep_downloads( options, add_books, merge=merge )
+        # AddNewDialog collects URLs, format and presents buttons.
+        # add_new_dialog is modeless and reused, both for new stories
+        # and anthologies, and for updating existing anthologies.
+        self.add_new_dialog.show_dialog(url_list_text,
+                                        self.prep_downloads,
+                                        merge=merge,
+                                        newmerge=True,
+                                        extraoptions={'anthology_url':anthology_url})
 
     def update_anthology(self):
         if not self.get_epubmerge_plugin():
@@ -487,7 +461,6 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         if len(self.gui.library_view.get_selected_ids()) != 1:
             self.gui.status_bar.show_message(_('Can only update 1 anthology at a time'), 3000)
             return
-        #print("update_existing()")        
         
         db = self.gui.current_db
         book_id = self.gui.library_view.get_selected_ids()[0]
@@ -529,29 +502,34 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         url_list_text = "\n".join(url_list)
         
         #print("urlmapfile:%s"%urlmapfile)
+        
+        # AddNewDialog collects URLs, format and presents buttons.
+        # add_new_dialog is modeless and reused, both for new stories
+        # and anthologies, and for updating existing anthologies.
+        self.add_new_dialog.show_dialog(url_list_text,
+                                        self.prep_anthology_downloads,
+                                        show=False,
+                                        merge=True,
+                                        newmerge=False,
+                                        extrapayload=urlmapfile,
+                                        extraoptions={'tdir':tdir,
+                                                      'mergebook':mergebook})
+        # Need to use AddNewDialog modal here because it's an update
+        # of an existing book.  Don't want the user deleting it or
+        # switching libraries on us.
+        self.add_new_dialog.exec_()
 
-        # self.gui is the main calibre GUI. It acts as the gateway to access
-        # all the elements of the calibre user interface, it should also be the
-        # parent of the dialog
-        # AddNewDialog just collects URLs, format and presents buttons.
-        d = AddNewDialog(self.gui,
-                         prefs,
-                         self.qaction.icon(),
-                         url_list_text,
-                         merge=True,
-                         newmerge=False
-                         )
-        d.exec_()
-        if d.result() != d.Accepted:
-            return
 
-        url_list = split_text_to_urls(d.get_urlstext())
-            
-        update_books = self.convert_urls_to_books(url_list)
+    def prep_anthology_downloads(self, options, update_books,
+                                 merge=False, urlmapfile=None):
+        
+        if isinstance(update_books,basestring):
+            url_list = split_text_to_urls(update_books)
+            update_books = self.convert_urls_to_books(url_list)
         
         for j, book in enumerate(update_books):
             url = book['url']
-            book['mergeorder'] = j
+            book['listorder'] = j
             if url in urlmapfile:
                 #print("found epub for %s"%url)
                 book['epub_for_update']=urlmapfile[url]
@@ -573,13 +551,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                 print("Canceling anthology update due to removed stories.")
                 return
 
-        options = d.get_ffdl_options()
-        options['version'] = self.version
-        options['tdir'] = tdir
-        #options['collision'] = UPDATEALWAYS
-        print(self.version)
-
-        options['mergebook'] = mergebook
+        # Now that we've 
         self.prep_downloads( options, update_books, merge=True )
         
     def update_dialog(self):
@@ -596,6 +568,9 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         book_list = map( partial(self.make_book_id_only), self.gui.library_view.get_selected_ids() )
         #book_ids = self.gui.library_view.get_selected_ids()
 
+        for j, book in enumerate(book_list):
+            book['listorder'] = j
+            
         LoopProgressDialog(self.gui,
                            book_list,
                            partial(self.populate_book_from_calibre_id, db=self.gui.current_db),
@@ -627,8 +602,6 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         # only if there's some good ones.
         if 0 < len(filter(lambda x : x['good'], update_books)):
             options = d.get_ffdl_options()
-            options['version'] = self.version
-            print(self.version)
             self.prep_downloads( options, update_books )
             
     def get_urls_clip(self,storyurls=True):
@@ -644,9 +617,16 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         # No need to do anything with perfs here, but we could.
         prefs
 
-    def prep_downloads(self, options, books, merge=False):
+    def prep_downloads(self, options, books, merge=False, extrapayload=None):
         '''Fetch metadata for stories from servers, launch BG job when done.'''
 
+        if isinstance(books,basestring):
+            url_list = split_text_to_urls(books)
+            books = self.convert_urls_to_books(url_list)
+
+        options['version'] = self.version
+        print(self.version)
+        
         #print("prep_downloads:%s"%books)
 
         if 'tdir' not in options: # if merging an anthology, there's alread a tdir.
@@ -681,14 +661,16 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
 
         url = book['url']
         print("url:%s"%url)
+        mi = None
 
         if not merge: # skip reject list when merging.
-            rejnote = rejecturllist.check(url)
-            if rejnote:
+            if rejecturllist.check(url):
+                rejnote = rejecturllist.get_full_note(url)
                 if question_dialog(self.gui, 'Reject URL?',
-                                   '<p>Reject URL?</p>'+
-                                   '<p>%s is on the Reject URL list:<br />"%s"</p>'%(url,rejnote)+
-                                   "<p>Click 'No' to download anyway.</p>",
+                                   '<h3>Reject URL?</h3>'+
+                                   '<p><b>%s</b> is on your Reject URL list:</p><p>"<b>%s</b>"</p>'%(url,rejnote)+
+                                   "<p>Click '<b>Yes</b>' to Reject.</p>"+
+                                   "<p>Click '<b>No</b>' to download anyway.</p>",
                                    show_copy_button=False):
                     book['comment'] = "Story on Reject URLs list (%s)."%rejnote
                     book['good']=False
@@ -697,9 +679,10 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                     return
                 else:
                     if question_dialog(self.gui, 'Remove Reject URL?',
-                                       "<p>Remove URL from Reject List?</p>"+
-                                       '<p>%s is on the Reject URL list:<br />"%s"</p>'%(url,rejnote)+
-                                       "<p>Click 'Yes' to remove it from the list and download,<br /> 'No' to download, but leave it on the Reject list.</p>",
+                                       "<h3>Remove URL from Reject List?</h3>"+
+                                       '<p><b>%s</b> is on your Reject URL list:</p><p>"<b>%s</b>"</p>'%(url,rejnote)+
+                                       "<p>Click '<b>Yes</b>' to remove it from the list,</p>"+
+                                       "<p>Click '<b>No</b>' to leave it on the list.</p>",
                                        show_copy_button=False):
                         rejecturllist.remove(url)
             
@@ -749,6 +732,31 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         # let other exceptions percolate up.
         story = adapter.getStoryMetadataOnly()
 
+        series = story.getMetadata('series')
+        if not merge and series and prefs['checkforseriesurlid']:
+            # try to find *series anthology* by *seriesUrl* identifier url or uri first.
+            searchstr = 'identifiers:"~ur(i|l):=%s"'%story.getMetadata('seriesUrl').replace(":","|")
+            identicalbooks = db.search_getting_ids(searchstr, None)
+            # print("searchstr:%s"%searchstr)
+            # print("identicalbooks:%s"%identicalbooks)
+            if len(identicalbooks) > 0 and question_dialog(self.gui, 'Skip Story?',
+                                                           '<h3>Skip Anthology Story?</h3>'+
+                                                           '<p>"<b>%s</b>" is in series "<b><a href="%s">%s</a></b>" that you have an anthology book for.</p>'%
+                                                           (story.getMetadata('title'),story.getMetadata('seriesUrl'),series[:series.index(' [')])+
+                                                           "<p>Click '<b>Yes</b>' to Skip.</p>"+
+                                                           "<p>Click '<b>No</b>' to download anyway.</p>",
+                                                           show_copy_button=False):
+                book['comment'] = "Story in Series Anthology(%s)."%series
+                book['title'] = story.getMetadata('title')
+                book['author'] = [story.getMetadata('author')]
+                book['good']=False
+                book['icon']='rotate-right.png'
+                book['status'] = 'Skipped'
+                return
+                
+
+        ################################################################################################################################################33
+
         # set PI version instead of default.
         if 'version' in options:
             story.setMetadata('version',options['version'])
@@ -838,6 +846,42 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                     book['icon'] = 'edit-redo.png'
                     book['status'] = 'Update'
     
+                if book_id and mi: # book_id and mi only set if matched by title/author.
+                    liburl = self.get_story_url(db,book_id)
+                    if book['url'] != liburl:
+                        if collision in (OVERWRITE,OVERWRITEALWAYS):
+                            updat="overwrit"
+                        else:
+                            updat="updat"
+                        if not question_dialog(self.gui, 'Change Story URL?',
+                                               '<h3>Change Story URL?</h3>'+
+                                               '<p><b>%s</b> by <b>%s</b> is already in your library with a different source URL:</p>'%
+                                                   (mi.title,', '.join(mi.author))+
+                                               '<p>In library: <a href="%(liburl)s">%(liburl)s</a></p><p>New URL: <a href="%(newurl)s">%(newurl)s</a></p>'%
+                                                   {'liburl':liburl,'newurl':book['url']}+
+                                               "<p>Click '<b>Yes</b>' to %se book with new URL.</p>"%updat+
+                                               "<p>Click '<b>No</b>' to skip %sing this book.</p>"%updat,
+                                               show_copy_button=False):
+                            if question_dialog(self.gui, 'Download as New Book?',
+                                               '<h3>Download as New Book?</h3>'+
+                                               '<p><b>%s</b> by <b>%s</b> is already in your library with a different source URL.</p>'%
+                                               (mi.title,', '.join(mi.author))+
+                                               '<p>You chose not to update the existing book.  Do you want to add a new book for this URL?</p>'+
+                                               '<p>New URL: <a href="%(newurl)s">%(newurl)s</a></p>'%
+                                               {'newurl':book['url']}+
+                                               "<p>Click '<b>Yes</b>' to a new book with new URL.</p>"+
+                                               "<p>Click '<b>No</b>' to skip URL.</p>",
+                                               show_copy_button=False):
+                                book_id = None
+                                mi = None
+                                book['calibre_id'] = None
+                            else:
+                                book['comment'] = "Update declined by user due to differing story URL(%s)"%liburl
+                                book['good']=False
+                                book['icon']='rotate-right.png'
+                                book['status'] = 'Different URL'
+                                return
+        
             if book_id != None and collision != ADDNEW:
                 if collision in (CALIBREONLY):
                     book['comment'] = 'Metadata collected.'
@@ -872,8 +916,9 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                         raise NotGoingToDownload("Not Overwriting, web site is not newer.",'edit-undo.png')
         
                 # For update, provide a tmp file copy of the existing epub so
-                # it can't change underneath us.
-                if collision in (UPDATE,UPDATEALWAYS) and \
+                # it can't change underneath us.  Now also overwrite for logpage preserve.
+                if collision in (UPDATE,UPDATEALWAYS,OVERWRITE,OVERWRITEALWAYS) and \
+                        fileform == 'epub' and \
                         db.has_format(book['calibre_id'],'EPUB',index_is_id=True):
                     tmp = PersistentTemporaryFile(prefix='old-%s-'%book['calibre_id'],
                                                   suffix='.epub',
@@ -881,7 +926,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                     db.copy_format_to(book_id,fileform,tmp,index_is_id=True)
                     print("existing epub tmp:"+tmp.name)
                     book['epub_for_update'] = tmp.name
-    
+
             if book_id != None and prefs['injectseries']:
                 mi = db.get_metadata(book_id,index_is_id=True)
                 if not book['series'] and mi.series != None:
@@ -930,18 +975,28 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                 break
         else:
             ## No good stories to try to download, go straight to
-            ## list.
-            d = DisplayStoryListDialog(self.gui,
-                                       'Nothing to Download',
-                                       prefs,
-                                       self.qaction.icon(),
-                                       book_list,
-                                       label_text='None of the URLs/stories given can be/need to be downloaded.'
-                                       )
-            d.exec_()
-
-            self.update_error_column(book_list,options)
+            ## updating error col.
+            msg = '''
+<p>None of the <b>%d</b> URLs/stories given can be/need to be downloaded.</p>
+<p>See log for details.</p>
+<p>Proceed with updating your library(Error Column, if configured)?</p>
+'''%len(book_list)
+    
+            htmllog='<html><body><table border="1"><tr><th>Status</th><th>Title</th><th>Author</th><th>Comment</th><th>URL</th></tr>'
+            for book in book_list:
+                if 'status' in book:
+                    status = book['status']
+                else:
+                    status = 'Bad'
+                htmllog = htmllog + '<tr><td>' + '</td><td>'.join([escapehtml(status),escapehtml(book['title']),escapehtml(", ".join(book['author'])),escapehtml(book['comment']),book['url']]) + '</td></tr>'
             
+            htmllog = htmllog + '</table></body></html>'
+
+            payload = ([], book_list, options)
+            self.gui.proceed_question(self.update_error_column,
+                                      payload, htmllog,
+                                      'FFDL log', 'FFDL download ended', msg,
+                                      show_copy_button=False)
             return
             
         func = 'arbitrary_n'
@@ -961,6 +1016,18 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                               'collision':ADDNEW,
                               'updatemeta':True,
                               'updateepubcover':True}):
+        custom_columns = self.gui.library_view.model().custom_columns
+        if book['calibre_id'] and prefs['errorcol'] != '' and prefs['errorcol'] in custom_columns:
+            label = custom_columns[prefs['errorcol']]['label']
+            if not book['good']:
+                print("record/update error message column %s %s"%(book['title'],book['url']))
+                db.set_custom(book['calibre_id'], book['comment'], label=label, commit=True) # book['comment']
+            else:
+                db.set_custom(book['calibre_id'], '', label=label, commit=True) # book['comment']
+                
+        if not book['good']:
+            return # only update errorcol on error.
+        
         print("add/update %s %s"%(book['title'],book['url']))
         mi = self.make_mi_from_book(book)
                 
@@ -979,8 +1046,10 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         add_ids = [ x['calibre_id'] for x in add_list ]
         update_list = filter(lambda x : x['good'] and not x['added'], book_list)
         update_ids = [ x['calibre_id'] for x in update_list ]
-        all_ids = add_ids
-        all_ids.extend(update_ids)
+        all_ids = add_ids + update_ids
+
+        failed_list = filter(lambda x : not x['good'] , book_list)
+        failed_ids = [ x['calibre_id'] for x in failed_list ]
 
         if options['collision'] != CALIBREONLY and \
                 (prefs['addtolists'] or prefs['addtoreadlists']):
@@ -999,19 +1068,26 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         
         if self.gui.cover_flow:
             self.gui.cover_flow.dataChanged()
-        
+
+        if showlist: # don't use with anthology
+            db = self.gui.current_db
+            marked_ids = dict()
+            marked_text = "ffdl_success"
+            for index, book_id in enumerate(all_ids):
+                marked_ids[book_id] = '%s_%04d' % (marked_text, index)
+            for index, book_id in enumerate(failed_ids):
+                marked_ids[book_id] = 'ffdl_failed_%04d' % index
+            # Mark the results in our database
+            db.set_marked_ids(marked_ids)
+            
+            if prefs['showmarked']: # show add/update
+                # Search to display the list contents
+                self.gui.search.set_search_string('marked:' + marked_text)
+                # Sort by our marked column to display the books in order
+                self.gui.library_view.sort_by_named_field('marked', True)
+                
         self.gui.status_bar.show_message(_('Finished Adding/Updating %d books.'%(len(update_list) + len(add_list))), 3000)
             
-        if showlist and (len(update_list) + len(add_list) != len(book_list)):
-            d = DisplayStoryListDialog(self.gui,
-                                       'Updates completed, final status',
-                                       prefs,
-                                       self.qaction.icon(),
-                                       book_list,
-                                       label_text='Stories have be added or updated in Calibre, some had additional problems.'
-                                       )
-            d.exec_()
-    
         print("all done, remove temp dir.")
         remove_dir(options['tdir'])
 
@@ -1030,6 +1106,8 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         book_list = job.result
         good_list = filter(lambda x : x['good'], book_list)
         bad_list = filter(lambda x : not x['good'], book_list)
+        good_list = sorted(good_list,key=lambda x : x['listorder'])
+        bad_list = sorted(bad_list,key=lambda x : x['listorder'])
         #print("book_list:%s"%book_list)
         payload = (good_list, bad_list, options)
         
@@ -1051,7 +1129,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
             msg = msg + '<p>Proceed with updating this anthology and your library?</p>'
     
             htmllog='<html><body><table border="1"><tr><th>Status</th><th>Title</th><th>Author</th><th>Comment</th><th>URL</th></tr>'
-            for book in sorted(good_list+bad_list,key=lambda x : x['mergeorder']):
+            for book in sorted(good_list+bad_list,key=lambda x : x['listorder']):
                 if 'status' in book:
                     status = book['status']
                 else:
@@ -1106,10 +1184,10 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         (good_list,bad_list,options) = payload
         total_good = len(good_list)
 
-        print("merge titles:\n%s"%"\n".join([ "%s %s"%(x['title'],x['mergeorder']) for x in good_list ]))
+        print("merge titles:\n%s"%"\n".join([ "%s %s"%(x['title'],x['listorder']) for x in good_list ]))
 
-        good_list = sorted(good_list,key=lambda x : x['mergeorder'])
-        bad_list = sorted(bad_list,key=lambda x : x['mergeorder'])
+        good_list = sorted(good_list,key=lambda x : x['listorder'])
+        bad_list = sorted(bad_list,key=lambda x : x['listorder'])
         
         self.gui.status_bar.show_message(_('Merging %s books.'%total_good))
 
@@ -1117,22 +1195,19 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         existingbook = None
         if 'mergebook' in options:
             existingbook = options['mergebook']
-        mergebook = self.merge_meta_books(existingbook,good_list)
+        #print("existingbook:\n%s"%existingbook)
+        mergebook = self.merge_meta_books(existingbook,good_list,options['fileform'])
 
         if 'mergebook' in options:
             mergebook['calibre_id'] = options['mergebook']['calibre_id']
-            #mergeid = options['mergebook']['calibre_id']
 
         if 'anthology_url' in options:
             mergebook['url'] = options['anthology_url']
-            
-        print("mergebook:\n%s"%mergebook)
+                    
+        #print("mergebook:\n%s"%mergebook)
         
         if mergebook['good']: # there shouldn't be any !'good' books at this point.
             # if still 'good', make a temp file to write the output to.
-            # For HTML format users, make the filename inside the zip something reasonable.
-            # For crazy long titles/authors, limit it to 200chars.
-            # For weird/OS-unsafe characters, use file safe only.
             tmp = PersistentTemporaryFile(suffix='.'+options['fileform'],
                                           dir=options['tdir'])
             print("title:"+mergebook['title'])
@@ -1152,26 +1227,23 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
     def do_download_list_update(self, payload):
         
         (good_list,bad_list,options) = payload
-        total_good = len(good_list)
+        good_list = sorted(good_list,key=lambda x : x['listorder'])
+        bad_list = sorted(bad_list,key=lambda x : x['listorder'])
 
-        self.gui.status_bar.show_message(_('Adding/Updating %s books.'%total_good))
+        self.gui.status_bar.show_message(_('FFDL Adding/Updating books.'))
 
-        if total_good > 0:
+        if good_list or (bad_list and prefs['errorcol'] != '' and prefs['errorcol'] in self.gui.library_view.model().custom_columns):
             LoopProgressDialog(self.gui,
-                               good_list,
+                               good_list+bad_list,
                                partial(self.update_books_loop, options=options, db=self.gui.current_db),
                                partial(self.update_books_finish, options=options),
                                init_label="Updating calibre for FanFiction stories...",
                                win_title="Update calibre for FanFiction stories",
                                status_prefix="Updated")
 
-        total_bad = len(bad_list)
-
-        if total_bad > 0:
-            self.update_error_column(bad_list,options)
-
-    def update_error_column(self,book_list,options):
+    def update_error_column(self,payload):
         '''Update custom error column if configured.'''
+        (empty_list,book_list,options)=payload
         custom_columns = self.gui.library_view.model().custom_columns
         if prefs['errorcol'] != '' and prefs['errorcol'] in custom_columns:
             self.previous = self.gui.library_view.currentIndex() # used by update_books_finish.
@@ -1246,16 +1318,10 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
             existingepub = db.format(book_id,'EPUB',index_is_id=True, as_file=True)
             epubmi = get_metadata(existingepub,'EPUB')
             if epubmi.cover_data[1] is not None:
-                db.set_cover(book_id, epubmi.cover_data[1])
-
-        # set author link if found.  All current adapters have authorUrl, except anonymous on AO3.
-        if 'authorUrl' in book['all_metadata']:
-            authurls = book['all_metadata']['authorUrl'].split(", ")
-            for i, auth in enumerate(book['author']):
-                #print("===Update author url for %s to %s"%(auth,authurls[i]))
-                autid=db.get_author_id(auth)
-                db.set_link_field_for_author(autid, unicode(authurls[i]),
-                                             commit=False, notify=False)
+                try:
+                    db.set_cover(book_id, epubmi.cover_data[1])
+                except:
+                    print("Failed to set_cover, skipping")
 
         # implement 'newonly' flags here by setting to the current
         # value again.
@@ -1331,9 +1397,17 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                     else:
                         coldef = custom_columns[custcol]
                         label = coldef['label']
-                    
+
                     if flag == 'r' or book['added']: # flag 'n' isn't actually needed--*always* set if configured and new book.
-                        db.set_custom(book_id, book['all_metadata'][meta], label=label, commit=False)
+                        if coldef['datatype'] in ('int','float'): # for favs, etc--site specific metadata.
+                            if 'anthology_meta_list' in book and meta in book['anthology_meta_list']:
+                                # re-split list, strip commas, convert to floats, sum up.
+                                val = sum([ float(x.replace(",","")) for x in book['all_metadata'][meta].split(", ") ])
+                            else:
+                                val = unicode(book['all_metadata'][meta]).replace(",","")
+                        else:
+                            val = book['all_metadata'][meta]
+                        db.set_custom(book_id, val, label=label, commit=False)
 
                     if flag == 'a':
                         vallist = []
@@ -1351,6 +1425,16 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                             
                         db.set_custom(book_id, ", ".join(vallist), label=label, commit=False)
 
+        # set author link if found.  All current adapters have authorUrl, except anonymous on AO3.
+        # Moved down so author's already in the DB.
+        if 'authorUrl' in book['all_metadata']:
+            authurls = book['all_metadata']['authorUrl'].split(", ")
+            for i, auth in enumerate(book['author']):
+                #print("===Update author url for %s to %s"%(auth,authurls[i]))
+                autid=db.get_author_id(auth)
+                db.set_link_field_for_author(autid, unicode(authurls[i]),
+                                             commit=False, notify=False)
+
         db.commit()
 
         if 'Generate Cover' in self.gui.iactions and (book['added'] or not prefs['gcnewonly']):
@@ -1394,11 +1478,6 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                 realmi = db.get_metadata(book_id, index_is_id=True)
                 gc_plugin.generate_cover_for_book(realmi,saved_setting_name=setting_name)
 
-        ## if error column set.
-        if prefs['errorcol'] != '' and prefs['errorcol'] in custom_columns:
-            label = custom_columns[prefs['errorcol']]['label']
-            db.set_custom(book['calibre_id'], '', label=label, commit=True) # book['comment']
-
     def get_clean_reading_lists(self,lists):
         if lists == None or lists.strip() == "" :
             return []
@@ -1491,15 +1570,15 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                 book['good'] = False
                 book['comment'] = "Same story already included."
             uniqueurls.add(book['url'])
-            book['mergeorder']=i # BG d/l jobs don't come back in order.
-                                 # Didn't matter until anthologies.
+            book['listorder']=i # BG d/l jobs don't come back in order.
+                                # Didn't matter until anthologies & 'marked' successes
             books.append(book)
         return books
 
     def convert_url_to_book(self, url):
         book = self.make_book()
         # look here for [\d,\d] at end of url, and remove?
-        mc = re.match(r"^(?P<url>.*?)(?:\[(?P<begin>\d+)?(?P<comma>,)?(?P<end>\d+)?\])?$",url)
+        mc = re.match(r"^(?P<url>.*?)(?:\[(?P<begin>\d+)?(?P<comma>[,-])?(?P<end>\d+)?\])?$",url)
         #print("url:(%s) begin:(%s) end:(%s)"%(mc.group('url'),mc.group('begin'),mc.group('end')))
         url = mc.group('url')
         book['begin'] = mc.group('begin')
@@ -1589,7 +1668,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                     # identifiers have :->| in uri.
                     # print("uri from ident uri:%s"%identifiers['uri'].replace('|',':'))
                     return identifiers['uri'].replace('|',':')
-            elif path.lower().endswith('.epub'):
+            elif path and path.lower().endswith('.epub'):
                 existingepub = path
                 
             ## only epub has URL in it--at least where I can easily find it.
@@ -1608,12 +1687,13 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
     def is_good_downloader_url(self,url):
         return adapters.getNormalStoryURL(url)
 
-    def merge_meta_books(self,existingbook,book_list):
+    def merge_meta_books(self,existingbook,book_list,fileform):
         book = self.make_book()
         book['author'] = []
         book['tags'] = []
         book['url'] = ''
         book['all_metadata'] = {}
+        book['anthology_meta_list'] = {}
         book['comment'] = ''
         book['added'] = True
         book['good'] = True
@@ -1680,25 +1760,39 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                         book['all_metadata'][k]=book['all_metadata'][k]+"\n\n"+v
                     else:
                         book['all_metadata'][k]=book['all_metadata'][k]+", "+v
+                        # flag psuedo list element.  Used so numeric
+                        # cust cols can convert back to numbers and
+                        # add.
+                        book['anthology_meta_list'][k]=True
     
         if existingbook:
             book['title'] = deftitle = existingbook['title']
             book['comments'] = existingbook['comments']
         else:
-            book['title'] = deftitle = book_list[0]['title']+" Anthology"
+            book['title'] = deftitle = book_list[0]['title']
             book['comments'] = "Anthology containing:\n" + \
                 "\n".join([ "%s by %s"%(b['title'],', '.join(b['author'])) for b in book_list ])
             # book['all_metadata']['description']
         
-        # if all same series, use series for name.  But only if all and not previous named
-        if len(serieslist) == len(book_list):
-            series = serieslist[0]
-            book['title'] = series+" Anthology"
-            for sr in serieslist:
-                if series != sr:
-                    book['title'] = deftitle;
-                    break    
+            # if all same series, use series for name.  But only if all and not previous named
+            if len(serieslist) == len(book_list):
+                series = serieslist[0]
+                book['title'] = series
+                for sr in serieslist:
+                    if series != sr:
+                        book['title'] = deftitle;
+                        break
             
+            configuration = get_ffdl_config(book['url'],fileform)
+            print("anthology_title_pattern:%s"%configuration.getConfig('anthology_title_pattern'))
+            if configuration.getConfig('anthology_title_pattern'):
+                tmplt = Template(configuration.getConfig('anthology_title_pattern'))
+                book['title'] = tmplt.safe_substitute({'title':book['title']}).encode('utf8')
+            else:
+                # No setting, do fall back default.  Shouldn't happen,
+                # should always have a version in defaults.
+                book['title'] = book['title']+" Anthology"
+        
         book['all_metadata']['title'] = book['title'] # because custom columns are set from all_metadata
         book['all_metadata']['author'] = ", ".join(book['author'])
         book['author_sort']=book['author']
@@ -1706,6 +1800,8 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
             if v in book['tags']:
                 book['tags'].remove(v)
         book['tags'].append('Anthology')
+        book['all_metadata']['anthology'] = "true"
+        
         return book
 
 def split_text_to_urls(urls):

calibre-plugin/ffdl_util.py

@@ -11,7 +11,7 @@ from StringIO import StringIO
 
 from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapters, exceptions
 from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.configurable import Configuration
-from calibre_plugins.fanfictiondownloader_plugin.config import (prefs)
+from calibre_plugins.fanfictiondownloader_plugin.prefs import (prefs)
 
 def get_ffdl_personalini():
     if prefs['includeimages']:
@@ -24,7 +24,7 @@ make_firstimage_cover:true
     else:
         return prefs['personal.ini']
 
-def get_ffdl_config(url,fileform="EPUB",personalini=None):
+def get_ffdl_config(url,fileform="epub",personalini=None):
     if not personalini:
         personalini = get_ffdl_personalini()
     site='unknown'
@@ -38,6 +38,6 @@ def get_ffdl_config(url,fileform="EPUB",personalini=None):
 
     return configuration
 
-def get_ffdl_adapter(url,fileform="EPUB",personalini=None):
+def get_ffdl_adapter(url,fileform="epub",personalini=None):
     return adapters.getAdapter(get_ffdl_config(url,fileform,personalini),url)
     

calibre-plugin/jobs.py

@@ -153,6 +153,22 @@ def do_download_for_worker(book,options):
         elif options['collision'] in (ADDNEW, SKIP, OVERWRITE, OVERWRITEALWAYS) or \
                 ('epub_for_update' not in book and options['collision'] in (UPDATE, UPDATEALWAYS)):
 
+            # preserve logfile even on overwrite.
+            if 'epub_for_update' in book:
+                (urlignore,
+                 chaptercountignore,
+                 oldchaptersignore,
+                 oldimgsignore,
+                 oldcoverignore,
+                 calibrebookmarkignore,
+                 # only logfile set in adapter, so others aren't used.
+                 adapter.logfile) = get_update_data(book['epub_for_update'])
+
+                # change the existing entries id to notid so
+                # write_epub writes a whole new set to indicate overwrite.
+                if adapter.logfile:
+                    adapter.logfile = adapter.logfile.replace("span id","span notid")
+            
             print("write to %s"%outfile)
             writer.writeStory(outfilename=outfile, forceOverwrite=True)
             book['comment'] = 'Download %s completed, %s chapters.'%(options['fileform'],story.getMetadata("numChapters"))

calibre-plugin/prefs.py

@@ -0,0 +1,142 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2013, Jim Miller'
+__docformat__ = 'restructuredtext en'
+
+import copy
+
+from calibre.utils.config import JSONConfig
+from calibre.gui2.ui import get_gui
+
+from calibre_plugins.fanfictiondownloader_plugin.dialogs import OVERWRITE
+from calibre_plugins.fanfictiondownloader_plugin.common_utils import get_library_uuid
+PREFS_NAMESPACE = 'FanFictionDownLoaderPlugin'
+PREFS_KEY_SETTINGS = 'settings'
+
+# Set defaults used by all.  Library specific settings continue to
+# take from here.
+default_prefs = {}
+default_prefs['personal.ini'] = get_resources('plugin-example.ini')
+default_prefs['rejecturls'] = ''
+default_prefs['rejectreasons'] = '''Sucked
+Boring
+Dup from another site'''
+
+default_prefs['updatemeta'] = True
+default_prefs['updatecover'] = False
+default_prefs['updateepubcover'] = False
+default_prefs['keeptags'] = False
+default_prefs['showmarked'] = False
+default_prefs['urlsfromclip'] = True
+default_prefs['updatedefault'] = True
+default_prefs['fileform'] = 'epub'
+default_prefs['collision'] = OVERWRITE
+default_prefs['deleteotherforms'] = False
+default_prefs['adddialogstaysontop'] = False
+default_prefs['includeimages'] = False
+default_prefs['lookforurlinhtml'] = False
+default_prefs['checkforseriesurlid'] = True
+default_prefs['injectseries'] = False
+
+default_prefs['send_lists'] = ''
+default_prefs['read_lists'] = ''
+default_prefs['addtolists'] = False
+default_prefs['addtoreadlists'] = False
+default_prefs['addtolistsonread'] = False
+
+default_prefs['gcnewonly'] = False
+default_prefs['gc_site_settings'] = {}
+default_prefs['allow_gc_from_ini'] = True
+
+default_prefs['countpagesstats'] = []
+
+default_prefs['errorcol'] = ''
+default_prefs['custom_cols'] = {}
+default_prefs['custom_cols_newonly'] = {}
+default_prefs['allow_custcol_from_ini'] = True
+
+default_prefs['std_cols_newonly'] = {}
+
+# This is where all preferences for this plugin *were* stored
+# Remember that this name (i.e. plugins/fanfictiondownloader_plugin) is also
+# in a global namespace, so make it as unique as possible.
+# You should always prefix your config file name with plugins/,
+# so as to ensure you dont accidentally clobber a calibre config file
+old_prefs = JSONConfig('plugins/fanfictiondownloader_plugin')
+
+def set_library_config(library_config,db):
+    db.prefs.set_namespaced(PREFS_NAMESPACE,
+                            PREFS_KEY_SETTINGS,
+                            library_config)
+
+def get_library_config(db):
+    library_id = get_library_uuid(db)
+    library_config = None
+    # Check whether this is a configuration needing to be migrated
+    # from json into database.  If so: get it, set it, rename it in json.
+    if library_id in old_prefs:
+        #print("get prefs from old_prefs")
+        library_config = old_prefs[library_id]
+        set_library_config(library_config,db)
+        old_prefs["migrated to library db %s"%library_id] = old_prefs[library_id]
+        del old_prefs[library_id]
+
+    if library_config is None:
+        #print("get prefs from db")
+        library_config = db.prefs.get_namespaced(PREFS_NAMESPACE, PREFS_KEY_SETTINGS,
+                                                 copy.deepcopy(default_prefs))
+    return library_config
+
+# fake out so I don't have to change the prefs calls anywhere.  The
+# Java programmer in me is offended by op-overloading, but it's very
+# tidy.
+class PrefsFacade():
+    def _get_db(self):
+        if self.passed_db:
+            return self.passed_db
+        else:
+            # In the GUI plugin we want current db so we detect when
+            # it's changed.  CLI plugin calls need to pass db in.
+            return get_gui().current_db
+    
+    def __init__(self,passed_db=None):
+        self.default_prefs = default_prefs
+        self.libraryid = None
+        self.current_prefs = None
+        self.passed_db=passed_db
+        
+    def _get_prefs(self):
+        libraryid = get_library_uuid(self._get_db())
+        if self.current_prefs == None or self.libraryid != libraryid:
+            #print("self.current_prefs == None(%s) or self.libraryid != libraryid(%s)"%(self.current_prefs == None,self.libraryid != libraryid))
+            self.libraryid = libraryid
+            self.current_prefs = get_library_config(self._get_db())
+        return self.current_prefs
+        
+    def __getitem__(self,k):            
+        prefs = self._get_prefs()
+        if k not in prefs:
+            # pulls from default_prefs.defaults automatically if not set
+            # in default_prefs
+            return self.default_prefs[k]
+        return prefs[k]
+
+    def __setitem__(self,k,v):
+        prefs = self._get_prefs()
+        prefs[k]=v
+        # self._save_prefs(prefs)
+
+    def __delitem__(self,k):
+        prefs = self._get_prefs()
+        if k in prefs:
+            del prefs[k]
+
+    def save_to_db(self):
+        set_library_config(self._get_prefs(),self._get_db())
+        
+prefs = PrefsFacade()
+

defaults.ini

@@ -1,4 +1,4 @@
-# Copyright 2012 Fanficdownloader team
+# Copyright 2013 Fanficdownloader team
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -44,6 +44,9 @@ language_label:Language
 characters_label:Characters
 ships_label:Relationships
 series_label:Series
+seriesUrl_label:Series URL
+## seriesHTML is series as a link to seriesUrl.
+seriesHTML_label:Series
 ## Completed/In-Progress
 status_label:Status
 ## Dates story first published, last updated, and downloaded(last with time).
@@ -81,7 +84,7 @@ dateUpdated_format:%%Y-%%m-%%d
 ## You can include extra text or HTML that will be included as-is in
 ## the title page. Eg: titlepage_entries: ...,<br />,summary,<br />,...
 ## All current formats already include title and author.
-titlepage_entries: series,category,genre,language,characters,ships,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,description
+titlepage_entries: seriesHTML,category,genre,language,characters,ships,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,description
 
 ## Try to collect series name and number of this story in series.
 ## Some sites (ab)use 'series' for reading lists and personal
@@ -166,6 +169,8 @@ extratags: FanFiction
 ## *Five* part lines.  Effect only when trailing conditional key=>regexp matches
 ## metakey[,metakey]=>pattern=>replacement[&&conditionalkey=>regexp]
 ## Note that if metakey == conditionalkey the conditional is ignored.
+## You can use \s in the replacement to add explicit spaces.  (The config parser
+## tends to discard trailing spaces.)
 #replace_metadata:
 # genre,category=>Sci-Fi=>SF
 # Puella Magi Madoka Magica.* => Madoka
@@ -186,6 +191,10 @@ extratags: FanFiction
 ## summary.
 keep_summary_html:true
 
+## If set true, any style attributes on tags in the story HTML will be
+## kept.  Useful for keeping extra colors & formatting from original.
+#keep_style_attr: false
+
 ## Don't like the numbers at the start of chapter titles on some
 ## sites?  You can use strip_chapter_numbers to strip them off.  Just
 ## want to make them all look the same?  Strip them off, then add them
@@ -270,7 +279,7 @@ output_css:
 
 [txt]
 ## Add URLs since there aren't links.
-titlepage_entries: series,category,genre,language,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,storyUrl, authorUrl, description
+titlepage_entries: series,seriesUrl,category,genre,language,characters,ships,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,storyUrl, authorUrl, description
 
 ## Width to word wrap text output.  0 indicates no wrapping.
 wrap_width: 78
@@ -304,13 +313,13 @@ include_logpage: false
 ## if in the list.  You can include extra text or HTML that will be
 ## included as-is in each log entry. Eg: logpage_entries: ...,<br />,
 ## summary,<br />,...
-logpage_entries: dateCreated,datePublished,dateUpdated,numChapters,numWords,status,title,author,description,category,genre,rating,warnings
+logpage_entries: dateCreated,datePublished,dateUpdated,numChapters,numWords,status,series,title,author,description,category,genre,rating,warnings
 
 ## epub->mobi conversions typically don't like tables.
 titlepage_use_table: false
 
 ## When using tables, make these span both columns.
-wide_titlepage_entries: description, storyUrl, author URL
+wide_titlepage_entries: description, storyUrl, authorUrl, seriesUrl
 
 ## output background color--only used by html and epub (and ignored in
 ## epub by many readers).  Included below in output_css--will be
@@ -369,7 +378,7 @@ output_css:
 ## It can be either a 'file:' or 'http:' url.
 ## Note that if you enable make_firstimage_cover in [epub], but want
 ## to use default_cover_image for a specific site, use the site:format
-## section, for example: [www.ficwad.com:epub]
+## section, for example: [ficwad.com:epub]
 ## default_cover_image is a python string Template string with
 ## ${title}, ${author} etc, same as titlepage_entries. Unless
 ## allow_unsafe_filename is true, invalid filename chars will be
@@ -504,6 +513,15 @@ extraships:Severus Snape/Hermione Granger
 #username:YourName
 #password:yourpassword
 
+[asr3.slashzone.org]
+## Site dedicated to these categories/characters/ships
+extracategories:The Sentinel
+
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
 [bloodties-fans.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:Blood Ties
@@ -568,6 +586,11 @@ extraships:Spike/Buffy
 #username:YourName
 #password:yourpassword
 
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
 [dramione.org]
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter
@@ -599,7 +622,7 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 ## composite metadata entries.  dramione.org, for example, adds
 ## 'cliches' and then defines as the composite of hermiones,dracos in
 ## include_in_cliches.
-extra_valid_entries:themes,hermiones,dracos,timeline,cliches
+extra_valid_entries:themes,hermiones,dracos,timeline,cliches,read,reviews
 include_in_cliches:hermiones,dracos
 
 ## For another example, you could, by uncommenting this line, include
@@ -644,6 +667,11 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter
 
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -651,11 +679,22 @@ extracategories:Harry Potter
 #username:YourName
 #password:yourpassword
 
-## Some sites also require the user to confirm they are adult for
-## adult content.  In commandline version, this should go in your
-## personal.ini, not defaults.ini.
+[fanfic.potterheadsanonymous.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
 [fanfiction.portkey.org]
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter
@@ -699,6 +738,10 @@ extracharacters:Hermione Granger
 ## personal.ini, not defaults.ini.
 #is_adult:true
 
+## Extra metadata that this adapter knows about.  See [dramione.org]
+## for examples of how to use them.
+extra_valid_entries:read,reviews
+
 [hlfiction.net]
 ## Site dedicated to these categories/characters/ships
 extracategories:Highlander
@@ -759,6 +802,8 @@ extracategories:West Wing
 ## Site dedicated to these categories/characters/ships
 extracategories:NCIS
 
+[netraptor.org]
+
 [nfacommunity.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:NCIS
@@ -841,6 +886,22 @@ extracategories:Supernatural
 extracharacters:Sam,Dean
 extraships:Sam/Dean
 
+[scarhead.net]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
 [sg1-heliopolis.com]
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
@@ -915,6 +976,15 @@ extracategories:Harry Potter
 ## personal.ini, not defaults.ini.
 #is_adult:true
 
+[tokra.fandomnet.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Stargate: SG-1
+
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
 [www.adastrafanfic.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:Star Trek
@@ -969,6 +1039,13 @@ extracategories:InuYasha
 extracharacters:Sesshoumaru,Kagome
 extraships:Sesshoumaru/Kagome
 
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
 [www.dotmoon.net]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -1138,6 +1215,11 @@ cover_exclusion_regexp:/stories/999/images/.*?_trophy.png
 ## Site dedicated to these categories/characters/ships
 extracategories:NCIS
 
+[www.nickandgreg.net]
+## Site dedicated to these categories/characters/ships
+extracategories:CSI
+extraships:Nick Stokes/Greg Sanders
+
 [www.phoenixsong.net]
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter
@@ -1150,6 +1232,12 @@ extraships:Harry Potter/Ginny Weasley
 #username:YourName
 #password:yourpassword
 
+## phoenixsong.net, oddly, can have high rated chapters (login
+## required) in the middle of a lower rated story.  Use this to force
+## FFDL to always login to phoenixsong.net so those stories download
+## correctly.
+#force_login:true
+
 [www.potionsandsnitches.net]
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter
@@ -1200,6 +1288,17 @@ extracharacters:Kurt Hummel,Blaine Anderson
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+[www.simplyundeniable.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
 [www.sinful-desire.org]
 ## Site dedicated to these categories/characters/ships
 extracategories:Supernatural
@@ -1219,12 +1318,12 @@ extraships:Harry Potter/Ginny Weasley
 # www.squidge.org/peja calls it Fandom <shrug>
 category_label:Fandom
 # Remove numWords -- www.squidge.org/peja word counts are inaccurate
-titlepage_entries: series,category,genre,language,characters,ships,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,site,description
+titlepage_entries: seriesHTML,category,genre,language,characters,ships,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,site,description
 
 [www.squidge.org/peja:txt]
 ## Add URLs since there aren't links.
 # Remove numWords -- www.squidge.org/peja word counts are inaccurate
-titlepage_entries: series,category,genre,language,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,site,storyUrl, authorUrl, description
+titlepage_entries: series,seriesUrl,category,genre,language,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,site,storyUrl, authorUrl, description
 
 [www.storiesofarda.com]
 ## Site dedicated to these categories/characters/ships
@@ -1346,3 +1445,47 @@ extracategories:Stargate: Atlantis
 ## extratags params in all other sections.  Only commandline options
 ## beat overrides.
 #extratags:fanficdownloader
+
+
+[teststory:defaults]
+valid_entries:title,author_list,authorId_list,authorUrl_list,storyUrl,
+ datePublished,dateUpdated,numWords,status,language,series,seriesUrl,
+ rating,category_list,genre_list,warnings_list,characters_list
+
+# {{storyId}} is a special case--it's the only one that works.
+title:Test Story Title {{storyId}}
+author_list:Test Author aa
+authorId_list:1
+authorUrl_list:http://test1.com?authid=1
+storyUrl:http://test1.com?sid={{storyId}}
+datePublished:1975-03-15
+dateUpdated:1975-04-15
+numWords:123,456
+status:In-Progress
+language:English
+
+chaptertitles:Prologue
+
+## Add additional sections with different numbers to get different
+## parameters for different story urls.
+## test1.com?sid=1000
+[teststory:1000]
+# note the leading commas when doing add_to_ with valid_entries and *_list
+add_to_valid_entries:,favs
+title:Testing New Feature {{storyId}}
+author_list:Bob Smith
+authorId_list:45
+authorUrl_list:http://test1.com?authid=45
+datePublished:2013-03-15
+dateUpdated:2013-04-15
+numWords:1456
+favs:56
+series:The Great Test [4]
+seriesUrl:http://test1.com?seriesid=1
+rating:Tweenie
+category_list:Harry Potter,Furbie,Crossover,Puella Magi Madoka Magica/魔法少女まどか★マギカ,Magical Girl Lyrical Nanoha
+genre_list:Fantasy,Comedy,Sci-Fi,Noir
+warnings_list:Swearing,Violence
+characters_list:Bob Smith,George Johnson,Fred Smythe
+
+chaptertitles:Prologue,Chapter 1\, Xenos on Cinnabar,Chapter 2\, Sinmay on Kintikin,3. Chapter 3

downloader.py

@@ -32,10 +32,25 @@ if sys.version_info >= (2, 7):
     loghandler.setFormatter(logging.Formatter("(=====)(levelname)s:%(message)s"))
     rootlogger.addHandler(loghandler)
 
-from fanficdownloader import adapters,writers,exceptions
-from fanficdownloader.configurable import Configuration
-from fanficdownloader.epubutils import get_dcsource_chaptercount, get_update_data
-from fanficdownloader.geturls import get_urls_from_page
+try:
+    from calibre.constants import numeric_version as calibre_version
+    is_calibre = True
+except:
+    is_calibre = False
+
+# using try/except directly was masking errors during development.
+if is_calibre:
+    # running under calibre
+    from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapters,writers,exceptions
+    from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.configurable import Configuration
+    from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.epubutils import get_dcsource_chaptercount, get_update_data
+    from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.geturls import get_urls_from_page
+else:
+    from fanficdownloader import adapters,writers,exceptions
+    from fanficdownloader.configurable import Configuration
+    from fanficdownloader.epubutils import get_dcsource_chaptercount, get_update_data
+    from fanficdownloader.geturls import get_urls_from_page
+    
 
 if sys.version_info < (2, 5):
     print "This program requires Python 2.5 or newer."
@@ -48,15 +63,23 @@ def writeStory(config,adapter,writeformat,metaonly=False,outstream=None):
     del writer
     return output_filename
 
-def main():
+def main(argv,
+         parser=None,
+         passed_defaultsini=None,
+         passed_personalini=None):
    # read in args, anything starting with -- will be treated as --<varible>=<value>
-   usage = "usage: %prog [options] storyurl"
-   parser = OptionParser(usage)
+   if not parser:
+       parser = OptionParser("usage: %prog [options] storyurl")
    parser.add_option("-f", "--format", dest="format", default="epub",
-                     help="write story as FORMAT, epub(default), text or html", metavar="FORMAT")
+                     help="write story as FORMAT, epub(default), mobi, text or html", metavar="FORMAT")
+
+   if passed_defaultsini:
+       config_help="read config from specified file(s) in addition to calibre plugin personal.ini, ~/.fanficdownloader/personal.ini, and ./personal.ini"
+   else:
+       config_help="read config from specified file(s) in addition to ~/.fanficdownloader/defaults.ini, ~/.fanficdownloader/personal.ini, ./defaults.ini, and ./personal.ini"
    parser.add_option("-c", "--config",
                      action="append", dest="configfile", default=None,
-                     help="read config from specified file(s) in addition to ~/.fanficdownloader/defaults.ini, ~/.fanficdownloader/personal.ini, ./defaults.ini, ./personal.ini", metavar="CONFIG")
+                     help=config_help, metavar="CONFIG")
    parser.add_option("-b", "--begin", dest="begin", default=None,
                      help="Begin with Chapter START", metavar="START")
    parser.add_option("-e", "--end", dest="end", default=None,
@@ -79,11 +102,14 @@ def main():
    parser.add_option("-l", "--list",
                      action="store_true", dest="list",
                      help="Get list of valid story URLs from page given.",)
+   parser.add_option("-n", "--normalize-list",
+                     action="store_true", dest="normalize",default=False,
+                     help="Get list of valid story URLs from page given, but normalized to standard forms.",)
    parser.add_option("-d", "--debug",
                      action="store_true", dest="debug",
                      help="Show debug output while downloading.",)
    
-   (options, args) = parser.parse_args()
+   (options, args) = parser.parse_args(argv)
 
    if not options.debug:
        logger = logging.getLogger("fanficdownloader")
@@ -107,12 +133,18 @@ def main():
 
    conflist = []
    homepath = join(expanduser("~"),".fanficdownloader")
+
+   if passed_defaultsini:
+       configuration.readfp(passed_defaultsini)
    
    if isfile(join(homepath,"defaults.ini")):
        conflist.append(join(homepath,"defaults.ini"))
    if isfile("defaults.ini"):
        conflist.append("defaults.ini")
        
+   if passed_personalini:
+       configuration.readfp(passed_personalini)
+   
    if isfile(join(homepath,"personal.ini")):
        conflist.append(join(homepath,"personal.ini"))
    if isfile("personal.ini"):
@@ -148,8 +180,8 @@ def main():
            (var,val) = opt.split('=')
            configuration.set("overrides",var,val)
 
-   if options.list:
-       retlist = get_urls_from_page(args[0], configuration)
+   if options.list or options.normalize:
+       retlist = get_urls_from_page(args[0], configuration, normalize=options.normalize)
        print "\n".join(retlist)
                
        return
@@ -240,5 +272,5 @@ def main():
 if __name__ == "__main__":
     #import time
     #start = time.time()
-    main()
+    main(sys.argv[1:])
     #print("Total time seconds:%f"%(time.time()-start))

fanficdownloader/adapters/__init__.py

@@ -71,7 +71,6 @@ import adapter_thehexfilesnet
 import adapter_dokugacom
 import adapter_iketernalnet
 import adapter_onedirectionfanfictioncom
-import adapter_prisonbreakficnet
 import adapter_storiesofardacom
 import adapter_samdeanarchivenu
 import adapter_destinysgatewaycom
@@ -88,7 +87,6 @@ import adapter_pretendercentrecom
 import adapter_darksolaceorg
 import adapter_finestoriescom
 import adapter_hpfanficarchivecom
-import adapter_svufictioncom
 import adapter_twilightarchivescom
 import adapter_wizardtalesnet
 import adapter_nhamagicalworldsus
@@ -103,7 +101,6 @@ import adapter_merlinficdtwinscouk
 import adapter_thehookupzonenet
 import adapter_bloodtiesfancom
 import adapter_indeathnet
-import adapter_jlaunlimitedcom
 import adapter_qafficcom
 import adapter_efpfanficnet
 import adapter_potterficscom
@@ -115,6 +112,13 @@ import adapter_imagineeficcom
 import adapter_buffynfaithnet
 import adapter_psychficcom
 import adapter_hennethannunnet
+import adapter_tokrafandomnetcom
+import adapter_netraptororg
+import adapter_asr3slashzoneorg
+import adapter_nickandgregnet
+import adapter_potterheadsanonymouscom
+import adapter_simplyundeniablecom
+import adapter_scarheadnet
 
 ## This bit of complexity allows adapters to be added by just adding
 ## importing.  It eliminates the long if/else clauses we used to need
@@ -153,7 +157,7 @@ def getAdapter(config,url):
 
     #logger.debug("trying url:"+url)
     (cls,fixedurl) = getClassFor(url)
-    logger.debug("fixedurl:"+fixedurl)
+    #logger.debug("fixedurl:"+fixedurl)
     if cls:
         adapter = cls(config,fixedurl) # raises InvalidStoryURL
         return adapter

fanficdownloader/adapters/adapter_adastrafanficcom.py

@@ -195,6 +195,7 @@ class AdAstraFanficComSiteAdapter(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             

fanficdownloader/adapters/adapter_archiveofourownorg.py

@@ -1,4 +1,4 @@
-# -*- coding: utf-8 -*-
+#  -*- coding: utf-8 -*-
 
 # Copyright 2011 Fanficdownloader team
 #
@@ -292,9 +292,10 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
             a = metasoup.find('dd',{'class':"series"})
             b = a.find('a', href=re.compile(r"/series/\d+"))
             series_name = b.string
-            series_url = 'http://'+self.host+'/fanfic/'+b['href']
+            series_url = 'http://'+self.host+b['href']
             series_index = int(a.text.split(' ')[1])
             self.setSeries(series_name, series_index)
+            self.story.setMetadata('seriesUrl',series_url)
             
         except:
             # I find it hard to care if the series parsing fails

fanficdownloader/adapters/adapter_asr3slashzoneorg.py

@@ -0,0 +1,225 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return Asr3SlashzoneOrgAdapter
+
+class Asr3SlashzoneOrgAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/archive/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','asr3')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%d/%m/%y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'asr3.slashzone.org'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/archive/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/archive/viewstory.php?sid=")+r"\d+$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=3"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid & error in url.
+                addurl = addurl.replace("&","&")
+                url = self.url+'&index=1'+addurl
+                logger.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        #print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/archive/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Rating
+        rate = stripHTML(soup.find('div',{'id':'pagetitle'}))
+        rate = rate[rate.rindex('[')+1:rate.rindex(']')]
+        self.story.setMetadata('rating', rate)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/archive/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        metadiv = soup.find('div',{'class':'content'})
+        smalldiv = metadiv.find('div',{'class':'small'})
+
+        categorys = smalldiv.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+        for category in categorys:
+            self.story.addToList('category',category.string)
+        
+        chars = smalldiv.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+        for char in chars:
+            self.story.addToList('characters',char.string)
+
+        ships = smalldiv.parent.findAll('a',href=re.compile(r'browse\.php\?type=class&type_id=2&classid=1'))
+        for ship in ships:
+            self.story.addToList('ships',ship.string)
+
+        metatext = stripHTML(smalldiv)
+
+        if 'Completed: Yes' in metatext:
+            self.story.setMetadata('status', 'Completed')
+        else:
+            self.story.setMetadata('status', 'In-Progress')
+
+        wordstart=metatext.rindex('Word count:')+12
+        words = metatext[wordstart:metatext.index(' ',wordstart)]
+        self.story.setMetadata('numWords', words)
+
+        datesdiv = soup.find('div',{'class':'bottom'})
+        dates = stripHTML(datesdiv).split()
+        # Published: 04/26/2011 Updated: 03/06/2013
+        self.story.setMetadata('datePublished', makeDate(dates[1], self.dateformat))
+        self.story.setMetadata('dateUpdated', makeDate(dates[3], self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/archive/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            i=1
+            for a in storyas:
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
+                        break
+                    i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+        # remove 'small' leaving only summary.
+        smalldiv.extract()
+        self.setDescription(url,metadiv)
+        
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulSoup(self._fetchUrl(url))
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_bloodtiesfancom.py

@@ -211,7 +211,7 @@ class BloodTiesFansComAdapter(BaseSiteAdapter): # XXX
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
         self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/fiction/'+a['href'])
         self.story.setMetadata('author',a.string)
 
         # Find the chapters:
@@ -311,6 +311,7 @@ class BloodTiesFansComAdapter(BaseSiteAdapter): # XXX
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             

fanficdownloader/adapters/adapter_castlefansorg.py

@@ -184,7 +184,7 @@ class CastleFansOrgAdapter(BaseSiteAdapter): # XXX
         # Find authorid and URL from... author url.
         a = pagetitle.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
         self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/fanfic/'+a['href'])
         self.story.setMetadata('author',a.string)
 
         # Find the chapters:
@@ -284,6 +284,7 @@ class CastleFansOrgAdapter(BaseSiteAdapter): # XXX
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             

fanficdownloader/adapters/adapter_chaossycophanthexcom.py

@@ -212,6 +212,7 @@ class ChaosSycophantHexComAdapter(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             

fanficdownloader/adapters/adapter_darksolaceorg.py

@@ -49,7 +49,6 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
         logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-		
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/elysian/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -59,7 +58,7 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
 
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%d %B %Y"
+        self.dateformat = "%B %d, %Y"
             
     @staticmethod # must be @staticmethod, don't remove it.
     def getSiteDomain():
@@ -79,7 +78,8 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
     ## Login seems to be reasonably standard across eFiction sites.
     def needToLoginCheck(self, data):
         if 'This story contains adult content not suitable for children' in data \
-                or "That password doesn't match the one in our database" in data:
+                or "That password doesn't match the one in our database" in data \
+                or "Registered Users Only" in data:
             return True
         else:
             return False
@@ -94,17 +94,16 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
             params['penname'] = self.getConfig("username")
             params['password'] = self.getConfig("password")
         params['rememberme'] = '1'
-        params['sid'] = ''
-        params['intent'] = ''
+        params['action'] = 'login'
         params['submit'] = 'Submit'
     
         loginUrl = 'http://' + self.getSiteDomain() + '/elysian/user.php'
         logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['penname']))
     
-        d = self._fetchUrl(loginUrl, params)
+        d = self._postUrl(loginUrl, params)
     
-        if "User Account Page" not in d : #Member Account
+        if "Member Account" not in d : #User Account Page
             logger.info("Failed to login to URL %s as %s, or have no authorization to access the story" % (loginUrl, params['penname']))
             raise exceptions.FailedToLogin(url,params['penname'])
             return False
@@ -113,9 +112,19 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
 
     ## Getting the chapter list and the meta data, plus 'is adult' checking.
     def extractChapterUrlsAndMetadata(self):
+        
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=5"
+        else:
+            addurl=""
+
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
-        url = self.url
+        url = self.url+'&index=1'+addurl
         logger.debug("URL: "+url)
 
         try:
@@ -128,9 +137,30 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
 
         if self.needToLoginCheck(data):
             # need to log in for this one.
-            addurl="&ageconsent=ok"
             self.performLogin(url)
-            data = self._fetchUrl(url+addurl)
+            data = self._fetchUrl(url)
+            
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid & error in url.
+                addurl = addurl.replace("&","&")
+                url = self.url+'&index=1'+addurl
+                logger.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
             
         if "Access denied. This story has not been validated by the adminstrators of this site." in data:
             raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
@@ -142,33 +172,41 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
         # Now go hunting for all the meta data and the chapter list.
         
         ## Title and author
-        a = soup.find('div', {'id' : 'pagetitle'})
+        div = soup.find('div', {'id' : 'pagetitle'})
         
-        aut = a.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        aut = div.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
         self.story.setMetadata('authorId',aut['href'].split('=')[1])
         self.story.setMetadata('authorUrl','http://'+self.host+'/elysian/'+aut['href'])
         self.story.setMetadata('author',aut.string)
         aut.extract()
-        
-        self.story.setMetadata('title',a.string[:(len(a.string)-3)])
+
+        # first a tag in pagetitle is title
+        self.story.setMetadata('title',stripHTML(div.find('a')))
 
         # Find the chapters:
-        chapters=soup.find('select', {'name' : 'chapter'})
-        if chapters != None:
-            for chapter in chapters.findAll('option'):
-                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/elysian/viewstory.php?sid='+self.story.getMetadata('storyId')+'&chapter='+chapter['value']))
-        else:
-            self.chapterUrls.append((self.story.getMetadata('title'),url))
+        # chapters=soup.find('select', {'name' : 'chapter'})
+        # if chapters != None:
+        #     for chapter in chapters.findAll('option'):
+        #         self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/elysian/viewstory.php?sid='+self.story.getMetadata('storyId')+'&chapter='+chapter['value']))
+        # else:
+        #     self.chapterUrls.append((self.story.getMetadata('title'),url+"&chapter=1"))
 
+        for chapa in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+
+                                                       self.story.getMetadata('storyId')+'&chapter=\d+')):
+            self.chapterUrls.append((stripHTML(chapa),'http://'+self.host+'/elysian/'+chapa['href']))
+        
         self.story.setMetadata('numChapters',len(self.chapterUrls))
         
         asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
         
-        for list in asoup.findAll('div', {'class' : re.compile('listbox\s+')}):
-            a = list.find('a', href=re.compile(r'viewstory.php\?sid='))
-            if a != None:
-                if 'viewstory.php?sid='+self.story.getMetadata('storyId') in a['href']:
-                    break
+        # for metalist in asoup.findAll('div', {'class' : re.compile('listbox\s+')}):
+        #     a = metalist.find('a', href=re.compile(r'viewstory.php\?sid='))
+        #     if a != None:
+        #         if 'viewstory.php?sid='+self.story.getMetadata('storyId') in a['href']:
+        #             break
+
+        metalist = asoup.find('a', href=re.compile(r'viewstory.php\?sid='+
+                                                   self.story.getMetadata('storyId')+'($|[^\d])')).parent.parent
 
         # eFiction sites don't help us out a lot with their meta data
         # formating, so it's a little ugly.
@@ -182,7 +220,7 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
                 
 
         # <span class="label">Rated:</span> NC-17<br /> etc
-        labels = list.findAll('span', {'class' : 'classification'})
+        labels = metalist.findAll('span', {'class' : 'label'})
         for labelspan in labels:
             label = labelspan.text
             value = labelspan.nextSibling
@@ -190,7 +228,7 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
             if 'Summary' in label:
                 ## Everything until the next span class='label'
                 svalue = ""
-                while not (defaultGetattr(value,'class') == 'classification' or "Chapters: " in stripHTML(value)):
+                while not (defaultGetattr(value,'class') == 'label' or "Chapters: " in stripHTML(value)):
                     svalue += str(value)
                     value = value.nextSibling
                 self.setDescription(url,svalue)
@@ -238,7 +276,7 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
 
         try:
             # Find Series name from series URL.
-            a = list.find('a', href=re.compile(r"series.php\?seriesid=\d+"))
+            a = metalist.find('a', href=re.compile(r"series.php\?seriesid=\d+"))
             series_name = a.string
             series_url = 'http://'+self.host+'/elysian/'+a['href']
 
@@ -252,6 +290,7 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if ('viewstory.php?sid='+self.story.getMetadata('storyId')) in a['href']:
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             
@@ -264,8 +303,7 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
 
         logger.debug('Getting chapter text from: %s' % url)
 
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        soup = bs.BeautifulSoup(self._fetchUrl(url))
         
         div = soup.find('div', {'id' : 'story'})
 

fanficdownloader/adapters/adapter_destinysgatewaycom.py

@@ -218,6 +218,7 @@ class DestinysGatewayComAdapter(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             

fanficdownloader/adapters/adapter_dokugacom.py

@@ -81,6 +81,44 @@ class DokugaComAdapter(BaseSiteAdapter):
     def getSiteURLPattern(self):
         return r"http://"+self.getSiteDomain()+"/(fanfiction|spark)?/story/\d+/?\d+?$"
 
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'The author has disabled anonymous viewing for this story.' in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url,soup):
+        params = {}
+
+        if self.password:
+            params['username'] = self.username
+            params['passwd'] = self.password
+        else:
+            params['username'] = self.getConfig("username")
+            params['passwd'] = self.getConfig("password")
+        params['Submit'] = 'Submit'
+
+        # copy all hidden input tags to pick up appropriate tokens.
+        for tag in soup.findAll('input',{'type':'hidden'}):
+            params[tag['name']] = tag['value']
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/fanfiction'
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['username']))
+    
+        d = self._postUrl(loginUrl, params)
+
+        if "Your session has expired. Please log in again." in d:
+            d = self._postUrl(loginUrl, params)
+
+        if "Logout" not in d : #Member Account
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['username']))
+            raise exceptions.FailedToLogin(url,params['username'])
+            return False
+        else:
+            return True
 
     ## Getting the chapter list and the meta data, plus 'is adult' checking.
     def extractChapterUrlsAndMetadata(self):
@@ -97,12 +135,18 @@ class DokugaComAdapter(BaseSiteAdapter):
                 raise exceptions.StoryDoesNotExist(self.url)
             else:
                 raise e
+                
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url,soup)
+            data = self._fetchUrl(url)
+            soup = bs.BeautifulSoup(data)
             
         if "Access denied. This story has not been validated by the adminstrators of this site." in data:
             raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
         # print data
 
         # Now go hunting for all the meta data and the chapter list.
@@ -218,10 +262,6 @@ class DokugaComAdapter(BaseSiteAdapter):
             a=div.text.split('Words ')
             if len(a)==2: self.story.setMetadata('numWords', a[1])
             
-        
-
-
-            
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 

fanficdownloader/adapters/adapter_dracoandginnycom.py

@@ -273,6 +273,7 @@ class DracoAndGinnyComAdapter(BaseSiteAdapter):
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             

fanficdownloader/adapters/adapter_dramioneorg.py

@@ -237,6 +237,9 @@ class DramioneOrgAdapter(BaseSiteAdapter):
             if 'Word count' in label:
                 self.story.setMetadata('numWords', value)
 
+            if 'Read' in label:
+                self.story.setMetadata('read', value)
+
             if 'Categories' in label:
                 cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
                 for cat in cats:
@@ -274,6 +277,7 @@ class DramioneOrgAdapter(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             
@@ -281,6 +285,14 @@ class DramioneOrgAdapter(BaseSiteAdapter):
             # I find it hard to care if the series parsing fails
             pass
             
+        try:
+            self.story.setMetadata('reviews',
+                                   stripHTML(soup.find('h2',{'id':'pagetitle'}).
+                                             findAll('a', href=re.compile(r'^reviews.php'))[1]))
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 

fanficdownloader/adapters/adapter_efictionestelielde.py

@@ -195,6 +195,7 @@ class EfictionEstelielDeAdapter(BaseSiteAdapter):
                             name=seriessoup.find('div', {'id' : 'pagetitle'})
                             name.find('a').extract()
                             self.setSeries(name.text.split(' by[')[0], i)
+                            self.story.setMetadata('seriesUrl',series_url)
                             i=0
                             break
                         i+=1

fanficdownloader/adapters/adapter_efpfanficnet.py

@@ -280,6 +280,7 @@ class EFPFanFicNet(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId'))+'&i=1':
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             

fanficdownloader/adapters/adapter_erosnsapphosycophanthexcom.py

@@ -230,6 +230,7 @@ class ErosnSapphoSycophantHexComAdapter(BaseSiteAdapter):
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             

fanficdownloader/adapters/adapter_fanfictionnet.py

@@ -132,14 +132,28 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         ## For 1, use the second link.
         ## For 2, fetch the crossover page and pull the two categories from there.
 
-        categories = soup.findAll('a',{'class':'xcontrast_txt'})
+        categories = soup.find('div',{'id':'pre_story_links'}).findAll('a',{'class':'xcontrast_txt'})
+        #print("xcontrast_txt a:%s"%categories)
         if len(categories) > 1:
-            self.story.addToList('category',stripHTML(categories[-1]))
+            # Strangely, the ones with *two* links are the
+            # non-crossover categories.  Each is in a category itself
+            # of Book, Movie, etc.
+            self.story.addToList('category',stripHTML(categories[1]))
         elif 'Crossover' in categories[0]['href']:
             caturl = "http://%s%s"%(self.getSiteDomain(),categories[0]['href'])
             catsoup = bs.BeautifulSoup(self._fetchUrl(caturl))
             for a in catsoup.findAll('a',href=re.compile(r"^/crossovers/")):
                 self.story.addToList('category',stripHTML(a))
+            else:
+                # Fall back.  I ran across a story with a Crossver
+                # category link to a broken page once.
+                # http://www.fanfiction.net/s/2622060/1/
+                # Naruto + Harry Potter Crossover
+                logger.info("Fall back category collection")
+                for c in stripHTML(categories[0]).replace(" Crossover","").split(' + '):
+                    self.story.addToList('category',c)
+                
+                
             
         a = soup.find('a', href='http://www.fictionratings.com/')
         rating = a.string
@@ -150,7 +164,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
 
         # after Rating, the same bit of text containing id:123456 contains
         # Complete--if completed.
-        gui_table1i = soup.find(id="gui_table1i")
+        gui_table1i = soup.find('table',{'cellpadding':'5'})
 
         self.story.setMetadata('title', stripHTML(gui_table1i.find('b'))) # title appears to be only(or at least first) bold tag in gui_table1i
 
@@ -158,12 +172,17 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         if summarydiv:
             self.setDescription(url,stripHTML(summarydiv))
             
-        
-        metatext = stripHTML(gui_table1i.find('div', {'style':'color:gray;'})).replace('Hurt/Comfort','Hurt-Comfort')
+
+        grayspan = gui_table1i.find('span', {'class':'xgray xcontrast_txt'})
+        # for b in grayspan.findAll('button'):
+        #     b.extract()
+        metatext = stripHTML(grayspan).replace('Hurt/Comfort','Hurt-Comfort')
+        #logger.debug("metatext:(%s)"%metatext)
         metalist = metatext.split(" - ")
-        #logger.debug("metatext:(%s)"%metalist)
+        #logger.debug("metalist:(%s)"%metalist)
 
         # Rated: Fiction K - English - Words: 158,078 - Published: 02-04-11
+        # Rated: Fiction T - English - Adventure/Sci-Fi - Naruto U. - Chapters: 22 - Words: 114,414 - Reviews: 395 - Favs: 779 - Follows: 835 - Updated: 03-21-13 - Published: 04-28-12 - id: 8067258 
 
         # rating is obtained above more robustly.
         if metalist[0].startswith('Rated:'):
@@ -253,12 +272,18 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
                         ## slow_down_sleep_time setting is.
         data = self._fetchUrl(url)
 
+        if "Please email this error message in full to <a href='mailto:support@fanfiction.com'>support@fanfiction.com</a>" in data:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  FanFiction.net Site Error!" % url)
+        
         # some ancient stories have body tags inside them that cause
         # soup parsing to discard the content.  For story text we
         # don't care about anything before "<div class='storytextp"
         # (there's a space after storytextp, so no close quote(')) and
         # this kills any body tags.
-        data = data[data.index("<div class='storytextp"):]
+        if "<div class='storytextp" not in data:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+        else:
+            data = data[data.index("<div class='storytextp"):]
         data.replace("<body","<notbody").replace("<BODY","<NOTBODY")
         
         soup = bs.BeautifulSoup(data)

fanficdownloader/adapters/adapter_fimfictionnet.py

@@ -28,8 +28,6 @@ from .. import BeautifulSoup as bs
 from ..htmlcleanup import stripHTML
 from .. import exceptions as exceptions
 
-from ..bbcodeutils.bbcodeparser import bbcodeparser
-
 from base_adapter import BaseSiteAdapter,  makeDate
 
 def getClass():
@@ -181,10 +179,11 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
 
             self.setCoverImage(self.url,coverurl)
 
-        # the fimfic API gives bbcode for desc, not html.
-        # btw, bbcode honors newlines, html doesn't.  change newlines to br tags.
-        self.setDescription(self.url,
-                            bbcodeparser().parse(storyMetadata["description"]).html(doDeepCopy=False).replace('\r','').replace('\n','<br />'))
+        # fimf has started including extra stuff inside the description div.
+        descdivstr = "%s"%soup.find("div", {"class":"description"})
+        hrstr="<hr />"
+        descdivstr = '<div class="description">'+descdivstr[descdivstr.index(hrstr)+len(hrstr):]
+        self.setDescription(self.url,descdivstr)
         
         # Dates are in Unix time
         # Take the publish date from the first chapter posted
@@ -193,7 +192,7 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
         rawDateUpdated = storyMetadata["date_modified"]
         self.story.setMetadata("dateUpdated", datetime.fromtimestamp(rawDateUpdated))
 
-        chars = soup.find("div", {"class":"story"})
+        chars = soup.find("div", {"class":"inner_data"})
         # fimfic stopped putting the char name on or around the char
         # icon now for some reason.  Pull it from the image name with
         # some heuristics.

fanficdownloader/adapters/adapter_finestoriescom.py

@@ -187,6 +187,7 @@ class FineStoriesComAdapter(BaseSiteAdapter):
             a = lc4.find('a', href=re.compile(r"/library/show_series.php\?id=\d+"))
             i = a.parent.text.split('(')[1].split(')')[0]
             self.setSeries(a.text, i)
+            self.story.setMetadata('seriesUrl','http://'+self.host+a['href'])
         except:
             pass
         try:

fanficdownloader/adapters/adapter_grangerenchantedcom.py

@@ -227,6 +227,9 @@ class GrangerEnchantedCom(BaseSiteAdapter):
             if 'Word count' in label:
                 self.story.setMetadata('numWords', value)
 
+            if 'Read' in label:
+                self.story.setMetadata('read', value)
+
             if 'Categories' in label:
                 cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
                 for cat in cats:
@@ -275,9 +278,17 @@ class GrangerEnchantedCom(BaseSiteAdapter):
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
-            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+
+        try:
+            self.story.setMetadata('reviews',
+                                   stripHTML(soup.find('div',{'id':'sort'}).
+                                             findAll('a', href=re.compile(r'^reviews.php'))[1]))
         except:
             # I find it hard to care if the series parsing fails
             pass

fanficdownloader/adapters/adapter_hlfictionnet.py

@@ -206,6 +206,7 @@ class HLFictionNetAdapter(BaseSiteAdapter):
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if ('viewstory.php?sid='+self.story.getMetadata('storyId')) in a['href']:
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             

fanficdownloader/adapters/adapter_hpfanficarchivecom.py

@@ -193,6 +193,7 @@ class HPFanficArchiveComAdapter(BaseSiteAdapter):
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             

fanficdownloader/adapters/adapter_imagineeficcom.py

@@ -264,6 +264,7 @@ class ImagineEFicComAdapter(BaseSiteAdapter):
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             

fanficdownloader/adapters/adapter_ksarchivecom.py

@@ -285,6 +285,7 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             

fanficdownloader/adapters/adapter_libraryofmoriacom.py

@@ -133,7 +133,7 @@ class LibraryOfMoriaComAdapter(BaseSiteAdapter):
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
         self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/a/'+a['href'])
         self.story.setMetadata('author',a.string)
 
         # Find the chapters:
@@ -225,6 +225,7 @@ class LibraryOfMoriaComAdapter(BaseSiteAdapter):
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             

fanficdownloader/adapters/adapter_lumossycophanthexcom.py

@@ -212,6 +212,7 @@ class LumosSycophantHexComAdapter(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             

fanficdownloader/adapters/adapter_merlinficdtwinscouk.py

@@ -268,6 +268,7 @@ class MerlinFicDtwinsCoUk(BaseSiteAdapter):
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             

fanficdownloader/adapters/adapter_midnightwhispersca.py

@@ -260,6 +260,7 @@ class MidnightwhispersCaAdapter(BaseSiteAdapter): # XXX
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             

fanficdownloader/adapters/adapter_mugglenetcom.py

@@ -66,14 +66,18 @@ class MuggleNetComAdapter(BaseSiteAdapter): # XXX
             
     @staticmethod # must be @staticmethod, don't remove it.
     def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'fanfiction.mugglenet.com' # XXX
+        # The site domain.
+        return 'fanfiction.mugglenet.com'
+
+    @classmethod
+    def getAcceptDomains(cls):
+        return ['fanfiction.mugglenet.com','fanfic.mugglenet.com']
 
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+        return re.escape("http://")+r"fanfic(tion)?\.mugglenet\.com"+re.escape("/viewstory.php?sid=")+r"\d+$"
 
     ## Login seems to be reasonably standard across eFiction sites.
     def needToLoginCheck(self, data):
@@ -306,6 +310,7 @@ class MuggleNetComAdapter(BaseSiteAdapter): # XXX
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             

fanficdownloader/adapters/adapter_nationallibrarynet.py

@@ -163,6 +163,7 @@ class NationalLibraryNetAdapter(BaseSiteAdapter):
             
             if 'Series' in label:
                 self.setSeries(stripHTML(value.nextSibling), value.nextSibling.nextSibling.string[2:])
+                self.story.setMetadata('seriesUrl','http://'+self.host+'/'+value.nextSibling['href'])
                 
         asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
         story=asoup.find('a', href=re.compile(r'viewstory.php\?storyid='+self.story.getMetadata('storyId')))

fanficdownloader/adapters/adapter_ncisficcom.py

@@ -164,6 +164,7 @@ class NCISFicComAdapter(BaseSiteAdapter):
             if 'Series' in label:
                 if "No Series" not in value.nextSibling.string:
                     self.setSeries(stripHTML(value.nextSibling), value.nextSibling.nextSibling.string[2:])
+                    self.story.setMetadata('seriesUrl','http://'+self.host+'/'+value.nextSibling['href'])
                 
         asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
         story=asoup.find('a', href=re.compile(r'viewstory.php\?storyid='+self.story.getMetadata('storyId')))

fanficdownloader/adapters/adapter_ncisfictionnet.py

@@ -193,7 +193,7 @@ class NCISFictionNetAdapter(BaseSiteAdapter):
             series_name = a.find('a').string
             i = a.text.split('#')[1]
             self.setSeries(series_name, i)
-            
+            self.story.setMetadata('seriesUrl','http://'+self.host+'/'+a.find('a')['href'])
         except:
             # I find it hard to care if the series parsing fails
             pass

fanficdownloader/adapters/adapter_netraptororg.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-# Copyright 2012 Fanficdownloader team
+# Copyright 2013 Fanficdownloader team
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -28,11 +28,9 @@ from .. import exceptions as exceptions
 from base_adapter import BaseSiteAdapter,  makeDate
 
 def getClass():
-    return PrisonBreakFicNetAdapter
+    return NetRaptorOrgAdapter
 
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class PrisonBreakFicNetAdapter(BaseSiteAdapter):
+class NetRaptorOrgAdapter(BaseSiteAdapter):
 
     def __init__(self, config, url):
         BaseSiteAdapter.__init__(self, config, url)
@@ -51,31 +49,29 @@ class PrisonBreakFicNetAdapter(BaseSiteAdapter):
         logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        self._setURL('http://' + self.getSiteDomain() + '/fanfiction/viewstory.php?sid='+self.story.getMetadata('storyId'))
         
         # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','pbf')
+        self.story.setMetadata('siteabbrev','netrap')
 
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%B %d, %Y"
+        self.dateformat = "%d/%m/%y"
             
     @staticmethod # must be @staticmethod, don't remove it.
     def getSiteDomain():
         # The site domain.  Does have www here, if it uses it.
-        return 'www.prisonbreakfic.net'
+        return 'netraptor.org'
 
     def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+        return "http://"+self.getSiteDomain()+"/fanfiction/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+        return re.escape("http://"+self.getSiteDomain()+"/fanfiction/viewstory.php?sid=")+r"\d+$"
 
     ## Getting the chapter list and the meta data, plus 'is adult' checking.
     def extractChapterUrlsAndMetadata(self):
 
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
         url = self.url+'&index=1'
         logger.debug("URL: "+url)
 
@@ -86,6 +82,10 @@ class PrisonBreakFicNetAdapter(BaseSiteAdapter):
                 raise exceptions.StoryDoesNotExist(self.url)
             else:
                 raise e
+
+
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
             
         # use BeautifulSoup HTML parser to make everything easier to find.
         soup = bs.BeautifulSoup(data)
@@ -94,19 +94,20 @@ class PrisonBreakFicNetAdapter(BaseSiteAdapter):
         # Now go hunting for all the meta data and the chapter list.
         
         ## Title
-        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        pagetitle = soup.find('div',{'id':'pagetitle'})
+        a = pagetitle.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
         self.story.setMetadata('title',a.string)
         
         # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        a = pagetitle.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
         self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/fanfiction/'+a['href'])
         self.story.setMetadata('author',a.string)
 
         # Find the chapters:
         for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
             # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']))
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/fanfiction/'+chapter['href']))
 
         self.story.setMetadata('numChapters',len(self.chapterUrls))
 
@@ -119,10 +120,7 @@ class PrisonBreakFicNetAdapter(BaseSiteAdapter):
                 return d[k]
             except:
                 return ""
-                
-# summary, rated, word count, categories, characters, genre, warnings, completed, published, updated, seires
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
+				
         labels = soup.findAll('span',{'class':'label'})
         for labelspan in labels:
             value = labelspan.nextSibling
@@ -145,21 +143,18 @@ class PrisonBreakFicNetAdapter(BaseSiteAdapter):
 
             if 'Categories' in label:
                 cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-                for cat in cats:
+                catstext = [cat.string for cat in cats]
+                for cat in catstext:
                     self.story.addToList('category',cat.string)
 
             if 'Characters' in label:
                 chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-                for char in chars:
+                charstext = [char.string for char in chars]
+                for char in charstext:
                     self.story.addToList('characters',char.string)
 
-            if 'Pairing' in label:
-                ships = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=3')) # XXX
-                for ship in ships:
-                    self.story.addToList('ships',ship.string.replace(" and ","/"))
-
             if 'Genre' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) # XXX
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
                 for genre in genres:
                     self.story.addToList('genre',genre.string)
 
@@ -184,7 +179,7 @@ class PrisonBreakFicNetAdapter(BaseSiteAdapter):
             # Find Series name from series URL.
             a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
             series_name = a.string
-            series_url = 'http://'+self.host+'/'+a['href']
+            series_url = 'http://'+self.host+'/fanfiction/'+a['href']
 
             # use BeautifulSoup HTML parser to make everything easier to find.
             seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
@@ -193,6 +188,7 @@ class PrisonBreakFicNetAdapter(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             
@@ -205,8 +201,7 @@ class PrisonBreakFicNetAdapter(BaseSiteAdapter):
 
         logger.debug('Getting chapter text from: %s' % url)
 
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        soup = bs.BeautifulSoup(self._fetchUrl(url))
         
         div = soup.find('div', {'id' : 'story'})
 

fanficdownloader/adapters/adapter_nfacommunitycom.py

@@ -264,6 +264,7 @@ class NfaCommunityComAdapter(BaseSiteAdapter): # XXX
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             

fanficdownloader/adapters/adapter_nhamagicalworldsus.py

@@ -112,7 +112,8 @@ class NHAMagicalWorldsUsAdapter(BaseSiteAdapter):
 		
         for info in asoup.findAll('table', {'width' : '100%', 'bordercolor' : re.compile(r'#')}):
             a = info.find('a')
-            if ('viewstory.php?sid='+self.story.getMetadata('storyId')) in a['href']:
+            if 'viewstory.php?sid='+self.story.getMetadata('storyId') == a['href'] or \
+                    ('viewstory.php?sid='+self.story.getMetadata('storyId')+'&') in a['href']:
                 self.story.setMetadata('title',a.string)
                 break
 		
@@ -142,14 +143,14 @@ class NHAMagicalWorldsUsAdapter(BaseSiteAdapter):
         for cat in cats:
             self.story.addToList('category',cat.string)
 					
-        a = info.find('a', href=re.compile(r'reviews.php\?sid='+self.story.getMetadata('storyId')))
+        a = info.find('a', href=re.compile(r'viewuser.php'))
         val = a.nextSibling
         svalue = ""
         while not defaultGetattr(val) == 'br':
             val = val.nextSibling
         val = val.nextSibling
         while not defaultGetattr(val) == 'br':
-            svalue += str(val)
+            svalue += unicode(val)
             val = val.nextSibling
         self.setDescription(url,svalue)
         

fanficdownloader/adapters/adapter_nickandgregnet.py

@@ -0,0 +1,174 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return NickAndGregNetAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class NickAndGregNetAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
+        self._setURL('http://' + self.getSiteDomain() + '/desert_archive/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','nag')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%Y/%m/%d"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.nickandgreg.net'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/desert_archive/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/desert_archive/viewstory.php?sid=")+r"\d+$"
+    
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&i=1'
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/desert_archive/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        chapters = soup.find('select')
+        for chapter in chapters.findAll('option'):
+            if chapter.text != 'Story Index' and chapter.text != 'Chapters':
+                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/desert_archive/'+chapter['value']))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+        
+        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
+        
+        for div in asoup.findAll('td', {'class' : 'tblborder6'}):
+            a = div.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+            if a != None:
+                break
+        
+        self.setDescription(url,div.find('br').nextSibling)
+        
+        a=div.text.split('Rating:')
+        if len(a) == 2: self.story.setMetadata('rating', a[1].split(' -')[0])
+        
+        a=div.text.split('Characters:')
+        if len(a) == 2: 
+        	for char in a[1].split(' -')[0].split(', '):
+        	    self.story.addToList('characters',char)
+        	    
+        a=div.text.split('Genres:')
+        if len(a) == 2: 
+        	for genre in a[1].split(' -')[0].split(', '):
+        	    self.story.addToList('genre',genre)
+        	    
+        a=div.text.split('Warnings:')
+        if len(a) == 2: 
+        	for warn in a[1].split(' -')[0].split(', '):
+        	    if 'none' not in warn:
+        	        self.story.addToList('warnings',warn)
+        	    
+        a=div.text.split('Completed:')
+        if len(a) ==2:
+            if 'Yes' in a[1]:
+                self.story.setMetadata('status', 'Completed')
+            else:
+                self.story.setMetadata('status', 'In-Progress')
+                
+        a=div.text.split('Published:')
+        if len(a) == 2: self.story.setMetadata('datePublished', makeDate(stripHTML(a[1].split(' -')[0]), self.dateformat))
+        
+        a=div.text.split('Updated:')
+        if len(a) == 2: self.story.setMetadata('dateUpdated', makeDate(stripHTML(a[1].split(' -')[0]), self.dateformat))
+
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('table', {'class' : 'tblborder6'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_onedirectionfanfictioncom.py

@@ -244,6 +244,7 @@ class OneDirectionFanfictionComAdapter(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             

fanficdownloader/adapters/adapter_phoenixsongnet.py

@@ -112,6 +112,8 @@ class PhoenixSongNetAdapter(BaseSiteAdapter):
         logger.debug("URL: "+url)
 
         try:
+            if self.getConfig('force_login'):
+                self.performLogin(url)
             data = self._fetchUrl(url)
         except urllib2.HTTPError, e:
             if e.code == 404:
@@ -123,7 +125,6 @@ class PhoenixSongNetAdapter(BaseSiteAdapter):
             # need to log in for this one.
             self.performLogin(url)
             data = self._fetchUrl(url)
-			
             
         # use BeautifulSoup HTML parser to make everything easier to find.
         soup = bs.BeautifulSoup(data)

fanficdownloader/adapters/adapter_pommedesangcom.py

@@ -275,6 +275,7 @@ class PommeDeSangComAdapter(BaseSiteAdapter):
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if ('viewstory.php?sid='+self.story.getMetadata('storyId')) in a['href']:
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             

fanficdownloader/adapters/adapter_ponyfictionarchivenet.py

@@ -225,6 +225,7 @@ class PonyFictionArchiveNetAdapter(BaseSiteAdapter):
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             

fanficdownloader/adapters/adapter_potionsandsnitchesnet.py

@@ -182,6 +182,7 @@ class PotionsAndSnitchesNetSiteAdapter(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             

fanficdownloader/adapters/adapter_potterheadsanonymouscom.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-# Copyright 2011 Fanficdownloader team
+# Copyright 2013 Fanficdownloader team
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -27,12 +27,12 @@ from .. import exceptions as exceptions
 
 from base_adapter import BaseSiteAdapter,  makeDate
 
-
 def getClass():
-    return JLAUnlimitedComAdapter
+    return PotterHeadsAnonymousComAdapter
 
-
-class JLAUnlimitedComAdapter(BaseSiteAdapter):
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class PotterHeadsAnonymousComAdapter(BaseSiteAdapter):
 
     def __init__(self, config, url):
         BaseSiteAdapter.__init__(self, config, url)
@@ -42,36 +42,69 @@ class JLAUnlimitedComAdapter(BaseSiteAdapter):
                                # Most sites that claim to be
                                # iso-8859-1 (and some that claim to be
                                # utf8) are really windows-1252.
-        self.username = "" # if left empty, site doesn't return any message at all.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
         self.password = ""
         self.is_adult=False
-
+        
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
         logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-
-        self._setURL('http://' + self.getSiteDomain() + '/eFiction1.1/viewstory.php?sid='+self.story.getMetadata('storyId'))
-
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
         # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','jla')
+        self.story.setMetadata('siteabbrev','pha')
 
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%m/%d/%y"
-
+        self.dateformat = "%d %b %Y"
+            
     @staticmethod # must be @staticmethod, don't remove it.
     def getSiteDomain():
         # The site domain.  Does have www here, if it uses it.
-        return 'www.jlaunlimited.com'
+        return 'fanfic.potterheadsanonymous.com'
 
     def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/eFiction1.1/viewstory.php?sid=1234"
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/eFiction1.1/viewstory.php?sid=")+r"\d+$"
-
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
 
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Registered Users Only' in data \
+                or 'There is no such account on our website' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
 
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['cookiecheck'] = '1'
+        params['submit'] = 'Submit'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "Member Account" not in d : #Member Account
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
 
     ## Getting the chapter list and the meta data, plus 'is adult' checking.
     def extractChapterUrlsAndMetadata(self):
@@ -81,7 +114,7 @@ class JLAUnlimitedComAdapter(BaseSiteAdapter):
             # If the title search below fails, there's a good chance
             # you need a different number.  print data at that point
             # and see what the 'click here to continue' url says.
-            addurl = "&ageconsent=ok&warning=4" # XXX
+            addurl = "&ageconsent=ok&warning=4"
         else:
             addurl=""
 
@@ -98,8 +131,21 @@ class JLAUnlimitedComAdapter(BaseSiteAdapter):
             else:
                 raise e
 
-        # Assume that if there is a url with 'warning=#' in the page then it is a 'check' page
-        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)",data)
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+            
+        # Since the warning text can change by warning level, let's
+        # look for the warning pass url.  ksarchive uses
+        # &warning= -- actually, so do other sites.  Must be an
+        # eFiction book.
+            
+        # viewstory.php?sid=1882&warning=4
+        # viewstory.php?sid=1654&ageconsent=ok&warning=5
+        #print data
+        #m = re.search(r"'viewstory.php\?sid=1882(&warning=4)'",data)
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
         if m != None:
             if self.is_adult or self.getConfig("is_adult"):
                 # We tried the default and still got a warning, so
@@ -120,27 +166,32 @@ class JLAUnlimitedComAdapter(BaseSiteAdapter):
                         raise e    
             else:
                 raise exceptions.AdultCheckRequired(self.url)
-                
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
         # use BeautifulSoup HTML parser to make everything easier to find.
         soup = bs.BeautifulSoup(data)
         # print data
 
         # Now go hunting for all the meta data and the chapter list.
 
+        pagetitle = soup.find('div',{'id':'pagetitle'})
+        
         ## Title
-        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')))
+        a = pagetitle.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
         self.story.setMetadata('title',a.string)
         
         # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        a = pagetitle.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
         self.story.setMetadata('authorId',a['href'].split('=')[1])
         self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
         self.story.setMetadata('author',a.string)
 
         # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+")):
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
             # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/eFiction1.1/'+chapter['href']+addurl))
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
 
         self.story.setMetadata('numChapters',len(self.chapterUrls))
 
@@ -177,34 +228,22 @@ class JLAUnlimitedComAdapter(BaseSiteAdapter):
 
             if 'Categories' in label:
                 cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-                catstext = [cat.string for cat in cats]
-                for cat in catstext:
+                for cat in cats:
                     self.story.addToList('category',cat.string)
 
             if 'Characters' in label:
                 chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-                charstext = [char.string for char in chars]
-                for char in charstext:
+                for char in chars:
                     self.story.addToList('characters',char.string)
 
-            ## Not all sites use Genre, but there's no harm to
-            ## leaving it in.  Check to make sure the type_id number
-            ## is correct, though--it's site specific.
             if 'Genre' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) # XXX
-                genrestext = [genre.string for genre in genres]
-                self.genre = ', '.join(genrestext)
-                for genre in genrestext:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+                for genre in genres:
                     self.story.addToList('genre',genre.string)
 
-            ## Not all sites use Warnings, but there's no harm to
-            ## leaving it in.  Check to make sure the type_id number
-            ## is correct, though--it's site specific.
             if 'Warnings' in label:
-                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
-                warningstext = [warning.string for warning in warnings]
-                self.warning = ', '.join(warningstext)
-                for warning in warningstext:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=3'))
+                for warning in warnings:
                     self.story.addToList('warnings',warning.string)
 
             if 'Completed' in label:
@@ -225,7 +264,7 @@ class JLAUnlimitedComAdapter(BaseSiteAdapter):
             # Find Series name from series URL.
             a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
             series_name = a.string
-            series_url = 'http://'+self.host+'/fanfic/'+a['href']
+            series_url = 'http://'+self.host+'/'+a['href']
 
             # use BeautifulSoup HTML parser to make everything easier to find.
             seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
@@ -234,26 +273,25 @@ class JLAUnlimitedComAdapter(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             
         except:
             # I find it hard to care if the series parsing fails
             pass
-
-
+            
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
         logger.debug('Getting chapter text from: %s' % url)
 
-        soup = bs.BeautifulSoup(self._fetchUrl(url),
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-
-        div = soup.find('div', {'id' : 'story'})
         
+        div = soup.find('div', {'id' : 'story'})
+
         if None == div:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-
+    
         return self.utf8FromSoup(url,div)
-

fanficdownloader/adapters/adapter_pretendercentrecom.py

@@ -228,6 +228,7 @@ class PretenderCenterComAdapter(BaseSiteAdapter):
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             

fanficdownloader/adapters/adapter_psychficcom.py

@@ -223,6 +223,7 @@ class PsychFicComAdapter(BaseSiteAdapter):
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             

fanficdownloader/adapters/adapter_qafficcom.py

@@ -236,6 +236,7 @@ class QafFicComAdapter(BaseSiteAdapter):
                             name=seriessoup.find('div', {'id' : 'pagetitle'})
                             name.find('a').extract()
                             self.setSeries(name.text.split(' by[')[0], i)
+                            self.story.setMetadata('seriesUrl',series_url)
                             i=0
                             break
                         i+=1

fanficdownloader/adapters/adapter_samdeanarchivenu.py

@@ -207,6 +207,7 @@ class SamDeanArchiveNuAdapter(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             

fanficdownloader/adapters/adapter_scarheadnet.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-# Copyright 2012 Fanficdownloader team
+# Copyright 2013 Fanficdownloader team
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -28,11 +28,11 @@ from .. import exceptions as exceptions
 from base_adapter import BaseSiteAdapter,  makeDate
 
 def getClass():
-    return SVUFictionComAdapter
+    return ScarHeadNetAdapter
 
 # Class name has to be unique.  Our convention is camel case the
 # sitename with Adapter at the end.  www is skipped.
-class SVUFictionComAdapter(BaseSiteAdapter):
+class ScarHeadNetAdapter(BaseSiteAdapter):
 
     def __init__(self, config, url):
         BaseSiteAdapter.__init__(self, config, url)
@@ -54,16 +54,16 @@ class SVUFictionComAdapter(BaseSiteAdapter):
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
         
         # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','svuf')
+        self.story.setMetadata('siteabbrev','shn')
 
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%b %d, %Y"
+        self.dateformat = "%d/%m/%y"
             
     @staticmethod # must be @staticmethod, don't remove it.
     def getSiteDomain():
         # The site domain.  Does have www here, if it uses it.
-        return 'svufiction.com'
+        return 'scarhead.net'
 
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
@@ -108,13 +108,13 @@ class SVUFictionComAdapter(BaseSiteAdapter):
 
     ## Getting the chapter list and the meta data, plus 'is adult' checking.
     def extractChapterUrlsAndMetadata(self):
-    
+
         if self.is_adult or self.getConfig("is_adult"):
             # Weirdly, different sites use different warning numbers.
             # If the title search below fails, there's a good chance
             # you need a different number.  print data at that point
             # and see what the 'click here to continue' url says.
-            addurl = "&ageconsent=ok&warning=6"
+            addurl = "&ageconsent=ok&warning=5"
         else:
             addurl=""
 
@@ -130,12 +130,21 @@ class SVUFictionComAdapter(BaseSiteAdapter):
                 raise exceptions.StoryDoesNotExist(self.url)
             else:
                 raise e
-                
+
         if self.needToLoginCheck(data):
             # need to log in for this one.
             self.performLogin(url)
             data = self._fetchUrl(url)
-                
+            
+        # Since the warning text can change by warning level, let's
+        # look for the warning pass url.  ksarchive uses
+        # &warning= -- actually, so do other sites.  Must be an
+        # eFiction book.
+            
+        # viewstory.php?sid=1882&warning=4
+        # viewstory.php?sid=1654&ageconsent=ok&warning=5
+        #print data
+        #m = re.search(r"'viewstory.php\?sid=1882(&warning=4)'",data)
         m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
         if m != None:
             if self.is_adult or self.getConfig("is_adult"):
@@ -166,24 +175,19 @@ class SVUFictionComAdapter(BaseSiteAdapter):
         # print data
 
         # Now go hunting for all the meta data and the chapter list.
+
+        pagetitle = soup.find('tr',{'valign':'top'})
         
         ## Title
-        pt = soup.find('div', {'class' : 'title'})
-        a = pt.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        if a.text != "":
-            self.story.setMetadata('title',a.text)
-        else:
-            self.story.setMetadata('title','Banner')
+        a = pagetitle.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
         
         # Find authorid and URL from... author url.
-        a = pt.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        a = pagetitle.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
         self.story.setMetadata('authorId',a['href'].split('=')[1])
         self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
         self.story.setMetadata('author',a.string)
 
-        rating=pt.text.split('[')[1].split(']')[0]
-        self.story.setMetadata('rating', rating)
-
         # Find the chapters:
         for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
             # just in case there's tags, like <i> in chapter titles.
@@ -193,46 +197,70 @@ class SVUFictionComAdapter(BaseSiteAdapter):
 
         # eFiction sites don't help us out a lot with their meta data
         # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-
-        bottom=soup.find('div', {'class' : 'bottom'}).text
-        self.story.setMetadata('numWords', bottom.split('Word count: ')[1].split(' Read:')[0])
-        self.story.setMetadata('datePublished', makeDate(bottom.split('Published: ')[1].split(' Updated:')[0], self.dateformat))
-        self.story.setMetadata('dateUpdated', makeDate(bottom.split('Updated: ')[1], self.dateformat))
-
         
-        content=soup.find('div', {'class' : 'content'})
-        value=content.find('h1').nextSibling
-        svalue = ""
-        while not defaultGetattr(value,'class') == 'smaller':
-            svalue += str(value)
-            value = value.nextSibling
-        self.setDescription(url,svalue)
-        
-        status = content.text.split('Completed: ')[1]
-        if 'Yes' in status:
+        cats = soup.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+        for cat in cats:
+            if '/' == cat.string[0]:
+                self.story.addToList('ships','Harry Potter'+cat.string.split('(')[0])
+            elif 'Harry' in cat.string:
+                self.story.addToList('ships',cat.string.split('(')[0])
+            else:
+                self.story.addToList('category',cat.string)
+            if '(' in cat.string:
+                self.story.addToList('category',cat.string.split('(')[1].split(')')[0])
+            
+                
+            
+
+        chars = soup.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+        for char in chars:
+            self.story.addToList('characters',char.string)
+
+        genres = soup.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
+        for genre in genres:
+            self.story.addToList('genre',genre.string)
+
+        warnings = soup.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+        for warning in warnings:
+            self.story.addToList('warnings',warning.string)
+            
+        textsoup = stripHTML(soup)
+
+        a = textsoup.split('Published: ')[1].split(' ')[0]
+        self.story.setMetadata('datePublished', makeDate(stripHTML(a), self.dateformat))
+        a = textsoup.split('Updated: ')[1].split(' ')[0]
+        self.story.setMetadata('dateUpdated', makeDate(stripHTML(a), self.dateformat))
+        a = textsoup.split('Rating: ')[1].split(' ')[0]
+        self.story.setMetadata('rating', a)
+        a = textsoup.split('Length: ')[1].split('(')[1].split(' ')[0]
+        self.story.setMetadata('numWords', a)
+        a = textsoup.split('Completed: ')[1].split(' ')[0]
+        if 'Yes' in a:
             self.story.setMetadata('status', 'Completed')
         else:
             self.story.setMetadata('status', 'In-Progress')
+        #a = textsoup.split('Summary: ')[1].split('Add Story to Favorites')[0]
+        #self.setDescription(url,a)
         
-        cats = content.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-        for cat in cats:
-            self.story.addToList('category',cat.string)
-            
-        chars = content.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-        for char in chars:
-            self.story.addToList('characters',char.string)
-            
-        genres = content.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
-        for genre in genres:
-            self.story.addToList('genre',genre.string)
-            
+
+        
+        a=soup.find(text=re.compile("Summary: "))
+        i=0
+        svalue = ""
+        while i == 0:
+            try:
+                b = str(a)
+                svalue += b.split('Summary: ')[1]
+            except:
+                svalue += str(a)
+            if a.nextSibling != None:
+                a = a.nextSibling
+            else:
+                a = a.parent.nextSibling
+            if 'Disclaimer: ' in stripHTML(a):
+                i=1
+        self.setDescription(url,svalue)
+
         try:
             # Find Series name from series URL.
             a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
@@ -241,16 +269,14 @@ class SVUFictionComAdapter(BaseSiteAdapter):
 
             # use BeautifulSoup HTML parser to make everything easier to find.
             seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
-            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
             i=1
             for a in storyas:
-                # skip 'report this' and 'TOC' links
-                if 'contact.php' not in a['href'] and 'index' not in a['href']:
-                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                        self.setSeries(series_name, i)
-                        break
-                    i+=1
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
+                    break
+                i+=1
             
         except:
             # I find it hard to care if the series parsing fails
@@ -261,7 +287,7 @@ class SVUFictionComAdapter(BaseSiteAdapter):
 
         logger.debug('Getting chapter text from: %s' % url)
 
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+        soup = bs.BeautifulSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
         
         div = soup.find('div', {'id' : 'story'})

fanficdownloader/adapters/adapter_scarvesandcoffeenet.py

@@ -141,7 +141,7 @@ class ScarvesAndCoffeeNetAdapter(BaseSiteAdapter):
         self.story.setMetadata('title',a.string)
         
         # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        a = soup.find('div',{"id":"pagetitle"}).find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
         self.story.setMetadata('authorId',a['href'].split('=')[1])
         self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
         self.story.setMetadata('author',a.string)
@@ -222,6 +222,7 @@ class ScarvesAndCoffeeNetAdapter(BaseSiteAdapter):
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             

fanficdownloader/adapters/adapter_sg1heliopoliscom.py

@@ -233,6 +233,7 @@ class SG1HeliopolisComAdapter(BaseSiteAdapter):
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             

fanficdownloader/adapters/adapter_simplyundeniablecom.py

@@ -0,0 +1,217 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return SimplyUndeniableComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class SimplyUndeniableComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','sud')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%m/%d/%Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.simplyundeniable.com'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Please log in now' in data \
+                or 'There is no such account on our website' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['cookiecheck'] = '1'
+        params['submit'] = 'Submit'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "My Account" not in d : #Member Account
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        if "Please log in now" in data:
+                raise exceptions.FailedToDownload(self.getSiteDomain() +" says: You need to have access to the restricted section.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('h1')
+        self.story.setMetadata('title',a.text)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
+        
+        for info in asoup.findAll('table', {'cellpadding' : '5'}):
+            a = info.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+            if a != None:
+                self.story.setMetadata('title',a.string)
+                break
+                
+        # Find the chapters:
+        if "Disclaimer" in data:
+            self.chapterUrls.append((self.story.getMetadata('title'),url))
+        else:
+            for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+&i=1$')):
+            # just in case there's tags, like <i> in chapter titles.
+                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+        
+        a = info.find('td', {'valign' : 'top'}).find('p')
+        self.setDescription(url,a)
+        
+        a = info.find('td', {'class' : 'greysm'}).findAll('b')
+        self.story.setMetadata('datePublished', makeDate(stripHTML(a[0].nextSibling), self.dateformat))
+        self.story.setMetadata('dateUpdated', makeDate(stripHTML(a[1].nextSibling), self.dateformat))
+        if 'Yes' in a[2].nextSibling:
+            self.story.setMetadata('status', 'Completed')
+        else:
+            self.story.setMetadata('status', 'In-Progress')
+        self.story.setMetadata('rating', a[3].nextSibling)
+        self.story.setMetadata('numWords', a[4].nextSibling)
+        
+        
+        warnings = info.find('td', {'width' : '45'}).nextSibling.nextSibling.text.split(', ')
+        for warning in warnings:
+            if 'none' not in warning:
+                self.story.addToList('warnings',warning)
+        chars = info.find('td', {'width' : '51'}).nextSibling.nextSibling.text.split(', ')
+        for char in chars:
+            if '&' in char:
+                self.story.addToList('ships',char)
+            else:
+                self.story.addToList('characters',char)
+        genres = info.find('td', {'width' : '36'}).nextSibling.nextSibling.text.split(', ')
+        for genre in genres:
+            self.story.addToList('genre',genre)
+            
+        cat = info.find('a', href=re.compile(r'categories.php'))
+        self.story.addToList('category',cat.string)
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('span', {'class' : 'style'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_sinfuldesireorg.py

@@ -226,6 +226,7 @@ class SinfulDesireOrgAdapter(BaseSiteAdapter):
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             

fanficdownloader/adapters/adapter_siyecouk.py

@@ -110,7 +110,7 @@ class SiyeCoUkAdapter(BaseSiteAdapter): # XXX
 
         ## Title
         titlea = authsoup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',titlea.string)
+        self.story.setMetadata('title',stripHTML(titlea))
         
         # Find the chapters (from soup, not authsoup):
         for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
@@ -215,6 +215,7 @@ class SiyeCoUkAdapter(BaseSiteAdapter): # XXX
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             

fanficdownloader/adapters/adapter_squidgeorgpeja.py

@@ -212,6 +212,7 @@ class SquidgeOrgPejaAdapter(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
                 

fanficdownloader/adapters/adapter_stargateatlantisorg.py

@@ -204,6 +204,7 @@ class StargateAtlantisOrgAdapter(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             

fanficdownloader/adapters/adapter_tenhawkpresentscom.py

@@ -220,6 +220,7 @@ class TenhawkPresentsComSiteAdapter(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             

fanficdownloader/adapters/adapter_test1.py

@@ -47,31 +47,63 @@ class TestSiteAdapter(BaseSiteAdapter):
         return BaseSiteAdapter.getSiteURLPattern(self)+r'/?\?sid=\d+$'
 
     def extractChapterUrlsAndMetadata(self):
+        idstr = self.story.getMetadata('storyId')
+        idnum = int(idstr)
 
-        if self.story.getMetadata('storyId') == '665' and not (self.is_adult or self.getConfig("is_adult")):
+        if idnum >= 1000:
+            logger.warn("storyId:%s - Custom INI data will be used."%idstr)
+
+            sections = ['teststory:%s'%idstr,'teststory:defaults']
+            #print("self.get_config_list(sections,'valid_entries'):%s"%self.get_config_list(sections,'valid_entries'))
+            for key in self.get_config_list(sections,'valid_entries'):
+                if key.endswith("_list"):
+                    nkey = key[:-len("_list")]
+                    #print("addList:%s"%(nkey))
+                    for val in self.get_config_list(sections,key):
+                        #print("addList:%s->%s"%(nkey,val))
+                        self.story.addToList(nkey,val.decode('utf-8').replace('{{storyId}}',idstr))
+                else:
+                    # Special cases:
+                    if key in ['datePublished','dateUpdated']:
+                        self.story.setMetadata(key,makeDate(self.get_config(sections,key),"%Y-%m-%d"))
+                    else:
+                        self.story.setMetadata(key,self.get_config(sections,key).decode('utf-8').replace('{{storyId}}',idstr))
+                    #print("set:%s->%s"%(key,self.story.getMetadata(key)))
+
+            self.chapterUrls = []
+            for (j,chap) in enumerate(self.get_config_list(sections,'chaptertitles'),start=1):
+                self.chapterUrls.append( (chap,self.url+"&chapter=%d"%j) )
+            # self.chapterUrls = [(u'Prologue '+self.crazystring,self.url+"&chapter=1"),
+            #                 ('Chapter 1, Xenos on Cinnabar',self.url+"&chapter=2"),
+            #                 ]
+            self.story.setMetadata('numChapters',len(self.chapterUrls))
+            
+            return
+
+        if idstr == '665' and not (self.is_adult or self.getConfig("is_adult")):
             logger.warn("self.is_adult:%s"%self.is_adult)
             raise exceptions.AdultCheckRequired(self.url)
 
-        if self.story.getMetadata('storyId') == '666':
+        if idstr == '666':
             raise exceptions.StoryDoesNotExist(self.url)
 
-        if self.story.getMetadata('storyId').startswith('670'):
+        if idstr.startswith('670'):
             time.sleep(1.0)
             
-        if self.story.getMetadata('storyId').startswith('671'):
+        if idstr.startswith('671'):
             time.sleep(1.0)
             
         if self.getConfig("username"):
             self.username = self.getConfig("username")
         
-        if self.story.getMetadata('storyId') == '668' and self.username != "Me" :
+        if idstr == '668' and self.username != "Me" :
             raise exceptions.FailedToLogin(self.url,self.username)
 
-        if self.story.getMetadata('storyId') == '664':
-            self.story.setMetadata(u'title',"Test Story Title "+self.story.getMetadata('storyId')+self.crazystring)
+        if idstr == '664':
+            self.story.setMetadata(u'title',"Test Story Title "+idstr+self.crazystring)
             self.story.setMetadata('author','Test Author aa bare amp(&) quote(') amp(&)')
         else:
-            self.story.setMetadata(u'title',"Test Story Title "+self.story.getMetadata('storyId'))
+            self.story.setMetadata(u'title',"Test Story Title "+idstr)
             self.story.setMetadata('author','Test Author aa')
         self.story.setMetadata('storyUrl',self.url)
         self.setDescription(self.url,u'Description '+self.crazystring+u''' Done
@@ -79,13 +111,12 @@ class TestSiteAdapter(BaseSiteAdapter):
 Some more longer description.  "I suck at summaries!"  "Better than it sounds!"  "My first fic"
 ''')
         self.story.setMetadata('datePublished',makeDate("1975-03-15","%Y-%m-%d"))
-        if self.story.getMetadata('storyId') == '669':
+        if idstr == '669':
             self.story.setMetadata('dateUpdated',datetime.datetime.now())
         else:
             self.story.setMetadata('dateUpdated',makeDate("1975-04-15","%Y-%m-%d"))
         self.story.setMetadata('numWords','123456')
 
-        idnum = int(self.story.getMetadata('storyId'))
         if idnum % 2 == 1:
             self.story.setMetadata('status','In-Progress')
         else:
@@ -101,12 +132,14 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
                 }
             self.story.setMetadata('language',langs[idnum%len(langs)])
             self.setSeries('The Great Test',idnum)
+            self.story.setMetadata('seriesUrl','http://test1.com?seriesid=1')
         if idnum == 0:
             self.setSeries("A Nook Hyphen Test "+self.story.getMetadata('dateCreated'),idnum)
+            self.story.setMetadata('seriesUrl','http://test1.com?seriesid=0')
             
         self.story.setMetadata('rating','Tweenie')
 
-        if self.story.getMetadata('storyId') == '673':
+        if idstr == '673':
             self.story.addToList('author','Author From List')
             self.story.addToList('author','Author From List 2')
         
@@ -126,21 +159,37 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
         self.story.addToList('warnings','Swearing')
         self.story.addToList('warnings','Violence')
 
-        self.story.addToList('category','Harry Potter')
-        self.story.addToList('category','Furbie')
-        self.story.addToList('category','Crossover')
-        self.story.addToList('category',u'Puella Magi Madoka Magica/魔法少女まどか★マギカ')
-        self.story.addToList('category',u'Magical Girl Lyrical Nanoha')
-
+        if idstr == '80':
+            self.story.addToList('category',u'Rizzoli & Isles')
+            self.story.addToList('characters','J. Rizzoli')
+        elif idstr == '81':
+            self.story.addToList('category',u'Pitch Perfect')
+            self.story.addToList('characters','Chloe B.')
+        elif idstr == '83':
+            self.story.addToList('category',u'Rizzoli & Isles')
+            self.story.addToList('characters','J. Rizzoli')
+            self.story.addToList('category',u'Pitch Perfect')
+            self.story.addToList('characters','Chloe B.')
+            self.story.addToList('ships','Chloe B. & J. Rizzoli')
+        elif idstr == '82':
+            self.story.addToList('characters','Henry (Once Upon a Time)')        
+            self.story.addToList('category',u'Once Upon a Time (TV)')
+        else:
+            self.story.addToList('category','Harry Potter')
+            self.story.addToList('category','Furbie')
+            self.story.addToList('category','Crossover')
+            self.story.addToList('category',u'Puella Magi Madoka Magica/魔法少女まどか★マギカ')
+            self.story.addToList('category',u'Magical Girl Lyrical Nanoha')
+            self.story.addToList('category',u'Once Upon a Time (TV)')
+            self.story.addToList('characters','Bob Smith')
+            self.story.addToList('characters','George Johnson')
+            self.story.addToList('characters','Fred Smythe')
+            
         self.story.addToList('genre','Fantasy')
         self.story.addToList('genre','Comedy')
         self.story.addToList('genre','Sci-Fi')
         self.story.addToList('genre','Noir')
-        
-        self.story.addToList('characters','Bob Smith')
-        self.story.addToList('characters','George Johnson')
-        self.story.addToList('characters','Fred Smythe')
-        
+                
         self.story.addToList('listX','xVal1')
         self.story.addToList('listX','xVal2')
         self.story.addToList('listX','xVal3')
@@ -200,6 +249,9 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
 <div>
 <h3>Prologue</h3>
 <p>This is a fake adapter for testing purposes.  Different sid's will give different errors:</p>
+<h4>Config(personal.ini)</h4>
+<p>sid&gt;=1000 will use custom test story data from your configuration(personal.ini)</p>
+<p>Hard coded ids:</p>
 <p>http://test1.com?sid=664 - Crazy string title</p>
 <p>http://test1.com?sid=665 - raises AdultCheckRequired</p>
 <p>http://test1.com?sid=666 - raises StoryDoesNotExist</p>
@@ -207,10 +259,6 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
 <p>http://test1.com?sid=668 - raises FailedToLogin unless username='Me'</p>
 <p>http://test1.com?sid=669 - Succeeds with Updated Date=now</p>
 <p>http://test1.com?sid=670 - Succeeds, but sleeps 2sec on each chapter</p>
-
-
-
-
 <p>http://test1.com?sid=671 - Succeeds, but sleeps 2sec metadata only</p>
 <p>http://test1.com?sid=672 - Succeeds, quick meta, sleeps 2sec chapters only</p>
 <p>http://test1.com?sid=673 - Succeeds, multiple authors, extra categories, genres</p>

fanficdownloader/adapters/adapter_thealphagatecom.py

@@ -189,6 +189,7 @@ class TheAlphaGateComAdapter(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             

fanficdownloader/adapters/adapter_thehookupzonenet.py

@@ -183,7 +183,7 @@ class TheHookupZoneNetAdapter(BaseSiteAdapter): # XXX
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
         self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/CriminalMinds/'+a['href'])
         self.story.setMetadata('author',a.string)
 
         # Find the chapters:
@@ -283,6 +283,7 @@ class TheHookupZoneNetAdapter(BaseSiteAdapter): # XXX
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             

fanficdownloader/adapters/adapter_thequidditchpitchorg.py

@@ -263,6 +263,7 @@ class TheQuidditchPitchOrgAdapter(BaseSiteAdapter): # XXX
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             

fanficdownloader/adapters/adapter_thewriterscoffeeshopcom.py

@@ -227,6 +227,7 @@ class TheWritersCoffeeShopComSiteAdapter(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             
@@ -243,16 +244,23 @@ class TheWritersCoffeeShopComSiteAdapter(BaseSiteAdapter):
         # issues with different SGML parsers in python.  This is a
         # nasty hack, but it works.
         data = data[data.index("<body"):]
-        
-        soup = bs.BeautifulStoneSoup(data,
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        span = soup.find('div', {'id' : 'story'})
 
-        if None == span:
+        chapter=bs.BeautifulSoup('<div class="story"></div>')
+        
+        soup = bs.BeautifulSoup(data)
+
+        found=False
+        for div in soup.findAll('div'):
+            if div.has_key('class') and div['class'] == 'notes':
+                chapter.append(div)
+            if div.has_key('id') and div['id'] == 'story':
+                chapter.append(div)
+                found=True
+
+        if not found:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,span)
+
+        return self.utf8FromSoup(url,chapter)
 
 def getClass():
     return TheWritersCoffeeShopComSiteAdapter

fanficdownloader/adapters/adapter_tokrafandomnetcom.py

@@ -0,0 +1,236 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return TokraFandomnetComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class TokraFandomnetComAdapter(BaseSiteAdapter): 
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','tokra')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%m/%d/%Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.  But it
+        # doesn't matter too much anymore.
+        return 'tokra.fandomnet.com'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=3"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid & error in url.
+                addurl = addurl.replace("&","&")
+                url = self.url+'&index=1'+addurl
+                logger.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        #print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Rating
+        rate = stripHTML(soup.find('div',{'id':'pagetitle'}))
+        rate = rate[rate.rindex('[')+1:rate.rindex(']')]
+        self.story.setMetadata('rating', rate)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        metadiv = soup.find('div',{'class':'content'})
+        smalldiv = metadiv.find('div',{'class':'small'})
+
+        # tokra categories -> genre
+        # categories will be filled from ini.
+        genres = smalldiv.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+        for genre in genres:
+            self.story.addToList('genre',genre.string)
+        
+        chars = smalldiv.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+        for char in chars:
+            self.story.addToList('characters',char.string)
+
+        metatext = stripHTML(smalldiv)
+
+        if 'Completed: Yes' in metatext:
+            self.story.setMetadata('status', 'Completed')
+        else:
+            self.story.setMetadata('status', 'In-Progress')
+
+        wordstart=metatext.rindex('Word count:')+12
+        words = metatext[wordstart:metatext.index(' ',wordstart)]
+        self.story.setMetadata('numWords', words)
+
+        datesdiv = soup.find('div',{'class':'bottom'})
+        dates = stripHTML(datesdiv).split()
+        # Published: 04/26/2011 Updated: 03/06/2013
+        self.story.setMetadata('datePublished', makeDate(dates[1], self.dateformat))
+        self.story.setMetadata('dateUpdated', makeDate(dates[3], self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            i=1
+            for a in storyas:
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
+                        break
+                    i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+        # remove 'small' leaving only summary.
+        smalldiv.extract()
+        self.setDescription(url,metadiv)
+        
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulSoup(self._fetchUrl(url))
+        
+        div = soup.find('div', {'class' : 'content'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+
+        # remove some decorations while keeping notes.
+        remove = div.find('div', {'id' : 'pagetitle'})
+        remove.extract()
+    
+        for remove in div.findAll('div', {'class' : 'right'}):
+            remove.extract()
+    
+        for remove in div.findAll('div', {'class' : 'left'}):
+            remove.extract()
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_tthfanficorg.py

@@ -100,8 +100,8 @@ class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter):
     
         if "Stories Published" not in d : #Member Account
             logger.info("Failed to login to URL %s as %s" % (loginUrl,
-                                                              params['penname']))
-            raise exceptions.FailedToLogin(url,params['penname'])
+                                                             params['urealname']))
+            raise exceptions.FailedToLogin(self.url,params['urealname'])
             return False
         else:
             return True
@@ -154,26 +154,6 @@ class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter):
         self.story.setMetadata('author',stripHTML(a))
         authorurl = 'http://'+self.host+a['href']
 
-        ainfo = soup.find('a', href='/StoryInfo-%s-1'%self.story.getMetadata('storyId'))
-        if ainfo != None: # indicates multiple authors/contributors.
-            try:
-                # going to pull part of the meta data from author list page.
-                infourl = 'http://'+self.host+ainfo['href']
-                logger.debug("**StoryInfo** URL: "+infourl)
-                infodata = self._fetchUrl(infourl)
-                infosoup = bs.BeautifulSoup(infodata)
-
-                for a in infosoup.findAll('a',href=re.compile(r"^/Author-\d+")):
-                    self.story.addToList('authorId',a['href'].split('/')[1].split('-')[1])
-                    self.story.addToList('authorUrl','http://'+self.host+a['href'].replace("/Author-","/AuthorStories-"))
-                    self.story.addToList('author',stripHTML(a))                
-                                            
-            except urllib2.HTTPError, e:
-                if e.code == 404:
-                    raise exceptions.StoryDoesNotExist(url)
-                else:
-                    raise e
-    
         try:
             # going to pull part of the meta data from *primary* author list page.
             logger.debug("**AUTHOR** URL: "+authorurl)
@@ -198,6 +178,53 @@ class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter):
         #self.story.setMetadata('description',stripHTML(storydiv.find('div',{'class':'storydesc'})))
         self.story.setMetadata('title',stripHTML(storydiv.find('a',{'class':'storylink'})))
 
+        ainfo = soup.find('a', href='/StoryInfo-%s-1'%self.story.getMetadata('storyId'))
+        if ainfo != None: # indicates multiple authors/contributors.
+            try:
+                # going to pull part of the meta data from author list page.
+                infourl = 'http://'+self.host+ainfo['href']
+                logger.debug("**StoryInfo** URL: "+infourl)
+                infodata = self._fetchUrl(infourl)
+                infosoup = bs.BeautifulSoup(infodata)
+
+                # for a in infosoup.findAll('a',href=re.compile(r"^/Author-\d+")):
+                #     self.story.addToList('authorId',a['href'].split('/')[1].split('-')[1])
+                #     self.story.addToList('authorUrl','http://'+self.host+a['href'].replace("/Author-","/AuthorStories-"))
+                #     self.story.addToList('author',stripHTML(a))
+
+                # second verticaltable is the chapter list.
+                table = infosoup.findAll('table',{'class':'verticaltable'})[1]
+                for a in table.findAll('a',href=re.compile(r"^/Story-"+self.story.getMetadata('storyId'))):
+                    autha = a.findNext('a',href=re.compile(r"^/Author-\d+"))
+                    self.story.addToList('authorId',autha['href'].split('/')[1].split('-')[1])
+                    self.story.addToList('authorUrl','http://'+self.host+autha['href'].replace("/Author-","/AuthorStories-"))
+                    self.story.addToList('author',stripHTML(autha))
+                    # include leading number to match 1. ... 2. ...
+                    self.chapterUrls.append(("%d. %s by %s"%(len(self.chapterUrls)+1,
+                                                             stripHTML(a),
+                                                             stripHTML(autha)),'http://'+self.host+a['href']))
+                                            
+            except urllib2.HTTPError, e:
+                if e.code == 404:
+                    raise exceptions.StoryDoesNotExist(url)
+                else:
+                    raise e
+        else: # single author:
+            # Find the chapter selector 
+            select = soup.find('select', { 'name' : 'chapnav' } )
+        	 
+            if select is None:
+        	   # no selector found, so it's a one-chapter story.
+        	   self.chapterUrls.append((self.story.getMetadata('title'),url))
+            else:
+                allOptions = select.findAll('option')
+                for o in allOptions:
+                    url = "http://"+self.host+o['value']
+                    # just in case there's tags, like <i> in chapter titles.
+                    self.chapterUrls.append((stripHTML(o),url))
+    
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
         verticaltable = soup.find('table', {'class':'verticaltable'})
 
         BtVS = True
@@ -235,29 +262,12 @@ class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter):
         if BtVS:
             self.story.addToList('category','Buffy: The Vampire Slayer')
             
-        # Find the chapter selector 
-        select = soup.find('select', { 'name' : 'chapnav' } )
-    	 
-        if select is None:
-    	   # no selector found, so it's a one-chapter story.
-    	   self.chapterUrls.append((self.story.getMetadata('title'),url))
-        else:
-            allOptions = select.findAll('option')
-            for o in allOptions:
-                url = "http://"+self.host+o['value']
-                # just in case there's tags, like <i> in chapter titles.
-                self.chapterUrls.append((stripHTML(o),url))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
         pseries = soup.find('p', {'style':'margin-top:0px'})
         m = re.match('This story is No\. (?P<num>\d+) in the series &quot;(?P<series>.+)&quot;\.',
                      pseries.text)
         if m:
             self.setSeries(m.group('series'),m.group('num'))
-        
-        return
-
+            self.story.setMetadata('seriesUrl',"http://"+self.host+pseries.find('a')['href'])
 
     def getChapterText(self, url):
         logger.debug('Getting chapter text from: %s' % url)

fanficdownloader/adapters/adapter_twilightarchivescom.py

@@ -163,6 +163,7 @@ class TwilightArchivesComAdapter(BaseSiteAdapter):
                 for a in storyas:
                     if a['href'] == ('/read/'+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
                     

fanficdownloader/adapters/adapter_twilightednet.py

@@ -217,6 +217,7 @@ class TwilightedNetSiteAdapter(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
         

fanficdownloader/adapters/adapter_twiwritenet.py

@@ -246,6 +246,7 @@ class TwiwriteNetSiteAdapter(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             

fanficdownloader/adapters/adapter_walkingtheplankorg.py

@@ -119,7 +119,7 @@ class WalkingThePlankOrgAdapter(BaseSiteAdapter):
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
         self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/archive/'+a['href'])
         self.story.setMetadata('author',a.string)
 
         # Find the chapters:
@@ -206,6 +206,7 @@ class WalkingThePlankOrgAdapter(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             

fanficdownloader/adapters/adapter_whoficcom.py

@@ -204,6 +204,7 @@ class WhoficComSiteAdapter(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             

fanficdownloader/adapters/adapter_wizardtalesnet.py

@@ -277,6 +277,7 @@ class WizardTalesNetAdapter(BaseSiteAdapter):
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             

fanficdownloader/adapters/adapter_wolverineandroguecom.py

@@ -194,6 +194,7 @@ class WolverineAndRogueComAdapter(BaseSiteAdapter):
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             

fanficdownloader/adapters/adapter_wraithbaitcom.py

@@ -126,8 +126,13 @@ class WraithBaitComAdapter(BaseSiteAdapter):
 
         # Find the chapters:
         for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # include author on chapters if multiple authors.
+            if len(alist) > 1:
+                add = " by %s"%stripHTML(chapter.findNext('a', href=re.compile(r"viewuser.php\?uid=\d+")))
+            else:
+                add = ""
             # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+            self.chapterUrls.append((stripHTML(chapter)+add,'http://'+self.host+'/'+chapter['href']+addurl))
 
         self.story.setMetadata('numChapters',len(self.chapterUrls))
 
@@ -196,6 +201,7 @@ class WraithBaitComAdapter(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             

fanficdownloader/adapters/base_adapter.py

@@ -197,7 +197,8 @@ class BaseSiteAdapter(Configurable):
             for index, (title,url) in enumerate(self.chapterUrls):
                 if (self.chapterFirst!=None and index < self.chapterFirst) or \
                         (self.chapterLast!=None and index > self.chapterLast):
-                    self.story.addChapter(removeEntities(title),
+                    self.story.addChapter(url,
+                                          removeEntities(title),
                                           None)
                 else:
                     if self.oldchapters and index < len(self.oldchapters):
@@ -206,7 +207,8 @@ class BaseSiteAdapter(Configurable):
                                                  partial(cachedfetch,self._fetchUrlRaw,self.oldimgs))
                     else:
                         data = self.getChapterText(url)
-                    self.story.addChapter(removeEntities(title),
+                    self.story.addChapter(url,
+                                          removeEntities(title),
                                           removeEntities(data))
             self.storyDone = True
             
@@ -313,6 +315,8 @@ class BaseSiteAdapter(Configurable):
             fetch=self._fetchUrlRaw
 
         acceptable_attributes = ['href','name','class','id']
+        if self.getConfig("keep_style_attr"):
+            acceptable_attributes.append('style')
         #print("include_images:"+self.getConfig('include_images'))
         if self.getConfig('include_images'):
             acceptable_attributes.extend(('src','alt','longdesc'))

fanficdownloader/bbcodeutils/bbcode2html.py

@@ -1,325 +0,0 @@
-#!/usr/bin/python
-# -*- coding: UTF-8 -*-
-#
-# Author:        Pau Sanchez (contact@pausanchez.com)
-# Version:       v1.0
-# Last Modified: 2010/09/15
-# 
-# For the latest version check out:
-#   http://www.codigomanso.com/en/projects
-# 
-# My blog:
-#   http://www.codigomanso.com/en/  - English Version
-#   http://www.codigomanso.com/es/  - Spanish Version
-#
-
-import sys
-import os
-import re
-import urllib
-
-class bbcode2html:
-  '''
-  This class gets a parsed BBCode and transforms it to valid HTML
-
-  Useful functions of this class:
-    html
-    convertToHTML
-
-  Example:
-    > parser = bbcodeparser ()
-    > parser.parse ('[b]bold[/b]')
-    > bbcode2html (parser).html()
-    <b>bold</b>
-
-    # This is faster for huge strings but changes the parser object internally
-    > bbcode2html (parser).html(doDeepCopy = False)
-    <b>bold</b>
-  '''
-  def __init__ (self, parser):
-    self._parser = parser
-    return
-
-  def html (self, allowClassAttr = False, doDeepCopy = True, parser = None):
-    '''
-    Convert current parsed code to HTML
-
-    Example:
-      code = bbcodeparser ('[b]bold[/b]')
-      code.html() -> '<b>bold</b>'
-    '''
-    if parser is None:
-      parser = self._parser
-
-    tokens = parser
-    if instanceof (parser, bbcodeparser):
-      tokens = parser.getTokens()
-
-    return bbcode2html.convertToHTML (tokens, allowClassAttr = allowClassAttr, doDeepCopy = doDeepCopy)
-
-  @staticmethod
-  def htmlString (string):
-    toReplace = {
-      u'<' : '&lt;',
-      u'>' : '&gt;',
-      u'"' : "&quot;",
-      u'&' : "&amp;"
-    }
-    for entity in toReplace:
-      string = string.replace(entity, toReplace[entity])
-    return string
-
-  @staticmethod
-  def getValidTags ():
-    simpleTags = ['b', 'u', 'i', 'sup', 'sub', 'ul', 'ol', 'li', 'table', 'tr', 'th', 'td', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6']
-    validTags = {
-      'p'         : { 'color' : 'color', 'size' : 'size', 'font' : 'font' },
-      'color'     : { 'color' : 'color' },
-      'size'      : { 'size' : 'size' },
-      'font'      : { 'font' : 'font' },
-      'img'       : { 'alt' : 'alt', 'title' : 'title', 'width' : 'width' , 'height' : 'height', 'img' : 'img'},
-      'url'       : { 'href' : 'href', 'url' : 'href', 'link'  : 'href', 'title' : 'title' },
-      's'         : { },
-      'code'      : { },
-      'quote'     : { },
-      'list'      : { 'list' : 'type' },
-      'email'     : { 'email': 'href'},
-      'google'    : { 'google':  'google'},
-      'wikipedia' : { 'wikipedia' : 'wikipedia', 'language' : 'language', 'lang' : 'lang'}
-    }
-
-    for tag in simpleTags:
-      validTags[tag] = { }
-    return validTags
-
-  @staticmethod
-  def convertToHTML (tokens, allowClassAttr = False, validTags = None, doDeepCopy = True):
-    '''
-    Convert internally parsed BBCode to XHTML
-
-    @doDeepCopy 
-      True:  it does a deep copy of tokens so this list will remain unchanged
-      False: tokens will be modified internally, but the output will be produced like 5x faster
-             it's a good idea to use False only when this is the last operation
-    '''
-    # do a deep copy
-    if doDeepCopy:
-      import copy
-      tokens = copy.deepcopy (tokens)
-
-    # filter invalid tags and attributes
-    if validTags is None:
-      validTags = bbcode2html.getValidTags()    
-
-    bbcode2html._filterInvalidTagsAndAttributes (tokens, validTags, allowClassAttr)
-    
-    # Start to convert
-    index       = 0
-    tokenLength = len (tokens)
-
-    # use a list for the output (an order of magnitude faster than using string concatenation)
-    htmlList = []
-    lastListOpener = []
-
-    while index < tokenLength:
-
-      if isinstance (tokens [index], basestring):
-        htmlList.append (bbcode2html.htmlString (tokens [index]))
-        index += 1
-        continue
-
-      token     = tokens[index] 
-      tag       = token['tag']  # opening or closing simple tag. e.g: 'b', '/b', '/u', ...
-      tagName   = (tag[1:] if tag[0] == '/' else tag)
-      tagOpener = (u'/' if tag[0] == '/' else u'')
-      tokenArgs = (token['args'] if 'args' in token else {})
-
-      # opening or closing simple tag COLOR / SIZE
-      if (tagName in ['p', 'color', 'size', 'font']): 
-        style  = ''
-        style += ((u' color: ' + tokenArgs['color'] + u';') if ('color' in tokenArgs) else '')
-        style += ((u' font-size: ' + tokenArgs['size'] + u'pt;') if ('size' in tokenArgs) else '')
-        style += ((u' font-family: ' + tokenArgs['font'] + u';') if ('font' in tokenArgs) else '')
-        style  = style.strip()
-
-        pArgs = {}
-        if style != '':
-          pArgs ['style'] = style
-
-        if 'class' in tokenArgs:
-          pArgs ['class'] = tokenArgs['class']
-
-        if ('args' not in token) and (tagName != 'p'):
-          if (tagOpener == '/'): # if closing tag, close it
-            htmlList.append (u'</span>')
-          index += 1
-          continue
-
-        if tagName != 'p':
-          tag = tagOpener + u'span'
-
-        htmlList.append (bbcode2html.xml (tag, pArgs))
-
-      # IMG tag
-      elif tag == 'img' and (index+2 < tokenLength):
-        if 'img' in tokenArgs:
-          # has the form of <width>x<height> ?
-          sizeMatch = re.match (u'^\s*(\d+)[xX](\d+)\s*$', tokenArgs['img'])
-          if sizeMatch is not None:
-            tokenArgs['width']  = sizeMatch.group(1)
-            tokenArgs['height'] = sizeMatch.group(2)
-          # then assume is the alternative text
-          else:
-            tokenArgs['alt'] = tokenArgs['img']
-          del tokenArgs['img']
-
-        # add the source of the image
-        tokenArgs ['src'] = tokens[index+1]
-
-        # [img]http://www.whatever.com/pic.jpg[/img]
-        htmlList.append (
-          bbcode2html.xml ('img', tokenArgs, soloTag=True)
-        )
-        index += 2 # skip next token and closing tag
-
-      # URL tag
-      elif tag == 'url':
-        if ('args' not in token) and (index+2 < tokenLength):
-          # [url]http://www.google.com[/url]
-          htmlList.append (bbcode2html.xml ('a', { 'href' : tokens[index+1] }))
-        else:
-          # [url=http://www.google.com]Google[/url]
-          # [url link=http://www.google.com title="This is Google"]Google[/url]
-          htmlList.append (bbcode2html.xml ('a', tokenArgs))
-  
-      # URL closing tag (sometimes needed)
-      elif (tag == '/url') or (tag == '/email'):
-        htmlList.append (u'</a>')
-
-      # Email tag
-      elif tag == 'email':
-        if ('args' not in token) and (index+2 < tokenLength):
-          # [email]asdf@asdf.com]
-          htmlList.append (bbcode2html.xml ('a', { 'href' : u'mailto:' + tokens[index+1].strip() }))
-        else:
-          # [email=asdf@asfd.com]john smith[/email]
-          if 'href' in tokenArgs:
-            tokenArgs['href'] = u'mailto:' + tokenArgs['href']
-          htmlList.append (bbcode2html.xml ('a', tokenArgs))
-  
-      elif tagName == 'list':
-        if tagOpener == '/':
-          htmlList.append (bbcode2html.xml (u'/' + lastListOpener.pop()))
-        else:
-          if ('type' not in tokenArgs):
-            htmlList.append (bbcode2html.xml (tagOpener + u'ul', tokenArgs))
-            lastListOpener.append ('ul')
-          else:
-            htmlList.append (bbcode2html.xml (tagOpener + u'ol', tokenArgs))
-            lastListOpener.append ('ol')
-
-      elif tagName == '*':
-        htmlList.append (bbcode2html.xml (tagOpener + u'li', tokenArgs))
-
-      elif (tagName == 's'):
-        tokenArgs['style'] =  'text-decoration: line-through;'
-        htmlList.append (bbcode2html.xml (tagOpener + u'span', tokenArgs))
-
-      elif (tagName == 'code'):
-        htmlList.append (bbcode2html.xml (tagOpener + u'pre', tokenArgs))
-
-      elif (tagName == 'quote'):
-        htmlList.append (bbcode2html.xml (tagOpener + u'blockquote', tokenArgs))
-
-      elif (tagName == 'google'):
-        htmlList.append (
-          bbcode2html.xml (
-            tagOpener + u'a', 
-            {'href' : 'http://www.google.com/search?q=' + urllib.quote_plus (tokens[index+1])},
-            tokens[index+1]
-          )
-        )
-        index += 2
-
-      elif (tagName == 'wikipedia'):
-        subdomain = 'www'
-        for arg  in ['lang', 'language', 'wikipedia']:
-          if arg in tokenArgs:
-            subdomain = tokenArgs[arg] 
-
-        htmlList.append (
-          bbcode2html.xml (
-            tagOpener + u'a', 
-            {'href' : 'http://' + subdomain + '.wikipedia.org/wiki/' + tokens[index+1].replace (' ', '_')},
-            tokens[index+1]
-          )
-        )
-        index += 2
-
-      elif (tagName in validTags):
-        htmlList.append (
-          bbcode2html.xml (tag, tokenArgs)
-        )
-
-      else:
-        # ignore this tag
-        pass 
-
-      index += 1
-
-    return ''.join (htmlList)
-
-  @staticmethod
-  def _filterInvalidTagsAndAttributes (tokens, validTags, allowClassAttr):
-    '''
-    Helper function to filter out invalid attributes from the tokens list
-    '''
-    # add 'class' attribute as valid (mapping 'class' itself)
-    if allowClassAttr:
-      for attr in validTags:
-        validTags[attr]['class'] = 'class'
-
-    # remove invalid attributes from tokens
-    for tindex in range(0, len(tokens)):
-      if isinstance (tokens[tindex], dict) and ('args' in tokens[tindex]) and (tokens[tindex]['tag'] in validTags):
-        validList = validTags[tokens[tindex]['tag']]
-
-        filteredArgs = {}
-        for arg in tokens[tindex]['args']:
-          if arg in validList:
-            # rename the argument
-            filteredArgs[validList[arg]] = tokens[tindex]['args'][arg]
-          else:
-            pass # do not include this arg in the filteredArgs
-
-        tokens[tindex]['args'] = filteredArgs
-
-    return
-
-  @staticmethod
-  def xml (tag, attrs = {}, text = None, soloTag = False):
-    '''
-    Helper function to produce valid XML output
-    '''
-    xml = u'<' + tag.lower()
-
-    # make sure we sort attributes alphabetically (for deterministic output)
-    # Faster but non-deterministic:
-    #  for (key, value) in attrs.iteritems():
-    #    xml += u' ' + key + u'="' + value + u'"'
-    for key in sorted (attrs.keys()):
-      xml += u' ' + key + u'="' + attrs[key] + u'"'
-      
-    # close tag
-    if text is None:
-      if soloTag:
-        xml += u' />'
-      else:
-        xml += u'>'
-    else:
-      xml += u'>' + text + u'</' + tag.lower() + '>'
-
-    return xml
-
-
-

fanficdownloader/bbcodeutils/bbcodebuilder.py

@@ -1,77 +0,0 @@
-#!/usr/bin/python
-# -*- coding: UTF-8 -*-
-#
-# Author:        Pau Sanchez (contact@pausanchez.com)
-# Version:       v1.0
-# Last Modified: 2010/09/15
-# 
-# For the latest version check out:
-#   http://www.codigomanso.com/en/projects
-# 
-# My blog:
-#   http://www.codigomanso.com/en/  - English Version
-#   http://www.codigomanso.com/es/  - Spanish Version
-#
-
-import sys
-import os
-import re
-import hashlib
-
-class bbcodebuilder:
-  '''
-  This class helps to build BBCode programmatically.
-
-  The function names are used as the tag name, then the first parameter
-  is the string that goes inside the tags and any extra parameter is 
-  appended as a parameter to the tag
-
-  Examples:
-    > bbcode = bbcodebuilder()  # create a instance!
-
-    > print bbcode.b ('bold')
-    [b]bold[/b]
-
-    > print bbcode.color ('this goes in red', 'red')
-    [color=red]this goes in red[/color]
-
-    > print bbcode.url ('Google', 'http://www.google.com')
-    [url=http://www.google.com]Google[/url]
-    
-    > print bbcode.alist('item 1', 'item 2')
-    [list=a]
-      [*]item 1
-      [*]item 2
-    [/list]
-
-
-  This solution is based on the recipe found on:
-    http://code.activestate.com/recipes/576831-simple-bbcode-support/
-  '''
-
-  def __getattr__(self, name):
-    '''
-    This is a generic getter that returns a function which gets the first parameter
-    as the string that goes between the tags, and extra parameters as tag parameters.
-
-    The name of the attribute is used as the tag name
-    '''
-    class bbcodebuilder_helper:
-      def __init__(self, name):
-        self._name = name
-
-      def __call__(self, string, *args):
-        return u'[{0}{1}]{2}[/{0}]'.format(self._name, (u'=' + u','.join(map(str, args))) if args else u'', string)
-    
-    return bbcodebuilder_helper (name)
-
-  def list(self, *items):
-    return u'[list]' + u''.join(map(lambda item: u"\n  [*]" + item, items)) + u"\n[/list]"
-
-  def nlist(self, *items):
-    return u'[list=1]' + u''.join(map(lambda item: u"\n  [*]" + item, items)) + u"\n[/list]"
-
-  def alist(self, *items):
-    return u'[list=a]' + u''.join(map(lambda item: u"\n  [*]" + item, items)) + u"\n[/list]"
-
-

fanficdownloader/bbcodeutils/bbcodeparser.py

@@ -1,258 +0,0 @@
-#!/usr/bin/python
-# -*- coding: utf-8 -*-
-#
-# Author:        Pau Sanchez (contact@pausanchez.com)
-# Version:       v1.0
-# Last Modified: 2010/09/15
-# 
-# For the latest version check out:
-#   http://www.codigomanso.com/en/projects
-# 
-# My blog:
-#   http://www.codigomanso.com/en/  - English Version
-#   http://www.codigomanso.com/es/  - Spanish Version
-#
-
-import sys
-import os
-import re
-import hashlib
-
-class bbcodeparser:
-  '''
-  This class parses BBCode into a internal structure to allow later processing and
-  conversion to HTML.
-
-  The parser tries to fix invalid code (like unclosed tags)
-
-  Useful URLs:
-    http://en.wikipedia.org/wiki/BBCode
-    http://www.bbcode.org/reference.php
-
-  Example:
-    > bbcode = bbcodeparser ()
-    > bbcode.parse ('[b]text in bold[/b]').html()
-    <b>text in bold</b>
-
-    # dump HTML
-    > bbcode.parse ('[p][color=red]text in red').html()
-    <p><span style="color:red;">text in red</span></p>
-
-    # dump fixed BBCode
-    > bbcode.parse ('[p][color=red]text in red').bbcode()
-    [p][color=red]text in red[/color][/p]
-
-    > bbcode.parse ('This [b][i]code[/b] will be fixed[/invalid]').bbcode()
-    This [b][i]code[/i][/b] will be fixed
-
-    # dump fixed bbcode
-    > str (bbcodeparse ('This [b][i]code[/b] will be fixed[/invalid]'))
-    This [b][i]code[/i][/b] will be fixed
-  '''
-  _bbcode = ''
-  _tokens = []
-
-  def __init__ (self, bbcode = '', fixInvalidCode = True):
-    ''' Initialize and parse bbcode string (if any is given)
-    '''
-    self.parse (bbcode, fixInvalidCode)
-    return
-
-  def __str__ (self):
-    return self.bbcode()
-
-  def parse (self, bbcode = None, fixInvalidCode = True):
-    '''
-    It will parse and return the token list, trying to fix tags if
-    fixInvalidCode is True
-
-    It will return the current object to allow chaining
-
-    Example:
-      code = bbcode()
-      code.parse ('<b>bold</b>', True) -> 
-      code.parse ('<b>bold<i>italics</b>', True) -> internally will add the missing '</i>'
-    '''
-    if bbcode is not None:
-      self._bbcode = bbcode
-      self._tokens = self.tokenize (bbcode)
-      if fixInvalidCode:
-        self._tokens = self.fixWrongTags (self._tokens)
-
-    return self
-
-  # return ALL tokens
-  def getTokens (self):
-    return self._tokens
-
-  def bbcode (self):
-    '''
-    Dump BBCode again. This is useful for dumping valid BBCode
-    '''
-    bbcode = []
-    for token in self._tokens:
-      if token is None:
-        continue
-
-      if isinstance (token, basestring):
-        bbcode.append (token.replace (u'[', u'\[').replace (u']', u'\]'))
-        continue
-      
-      tag       = token['tag']  # opening or closing simple tag. e.g: 'b', '/b', '/u', ...
-      tagOpener = (u'/' if tag[0] == u'/' else u'')
-
-      if (tagOpener == '/') or ('args' not in token):
-        bbcode.append (u'[' + tag + u']')
-      else:
-        # process args
-        argstr = ''
-
-        # the arg with the same name as the tag repersents the '=whatever'
-        if tag in token['args']:
-          if re.match ('\s|"', token['args'][tag]) is None:
-            argstr = u'=' + token['args'][tag]
-          else:
-            argstr = u'="' + token['args'][tag].replace (u'"', u'\"') + u'"'
-
-        for (k,v) in token['args'].iteritems():
-          if k == tag: # already processed
-            continue
-          argstr += ' ' + k + u'="' + v.replace (u'"', u'\"') + u'"'
-          
-        bbcode.append (u'[' + tag + argstr + ']')
-
-    return u''.join (bbcode)
-
-  def html (self, allowClassAttr = False, doDeepCopy = True):
-    '''
-    Convert current parsed code to HTML
-
-    @allowClassAttr
-      Is something like [b class="asdf"] allowed?
-
-    @doDeepCopy 
-      True:  it does a deep copy of tokens so this list will remain unchanged
-      False: tokens will be modified internally, but the output will be produced like 5x faster
-             it's a good idea to use False when the string parsed is huge and this is the
-             last operation on the string
-
-    Example:
-      code = bbcode ('[b]bold[/b]')
-      code.html() -> '<b>bold</b>'
-    '''
-    from bbcode2html import bbcode2html
-    return bbcode2html.convertToHTML (self._tokens, allowClassAttr = allowClassAttr, doDeepCopy = doDeepCopy)
-
-
-  @staticmethod
-  def fixWrongTags (inTokenList):
-    ''' Add missing tokens that have not been closed properly and try to fix some scenarios
-    '''
-    opened       = []
-    outTokenList = []
-    for token in inTokenList:
-      # normal string... do nothing
-      if isinstance(token, basestring):
-        outTokenList.append (token)
-      else:
-        # if starts with '/' is closing a tag
-        if token['tag'][0] == '/':
-          while (len (opened) > 0) and (opened[-1] != token['tag'][1:]):
-            outTokenList.append ({'tag' : '/' + opened[-1] })
-            del opened[-1]
-
-          if len(opened):
-            del opened[-1]
-            outTokenList.append (token)
-
-        # opening tag
-        else:
-          # if I open the same tag I opened before, close it, and open it again
-          if (len(opened) > 0) and (token['tag'] == opened[-1]):
-            outTokenList.append ({'tag' : '/' + opened[-1] })
-          else:
-            opened.append (token['tag'])
-          outTokenList.append (token)
-
-    # close all elements that have not been closed
-    while len(opened):
-      outTokenList.append ({'tag' : '/' + opened[-1] })
-      del opened[-1]
-
-    return outTokenList
-
-  @staticmethod
-  def tokenize(code):
-    '''
-    Tokenize BBCode tags and parameters
-
-    Return the token list using a internal format. See the example:
-      [
-        { 'tag' : 'p', 'args' : { 'font' : 'arial' } },
-        'This is ',
-        { 'tag' : 'url', 'args' : {'url' : 'http://www.google.com'} },
-        'a link to google',
-        { 'tag' : '/url' },
-        { 'tag' : '/p' }
-      ]
-    '''
-    re_tags    = re.compile (r'(\[[^]]+\])', re.DOTALL | re.UNICODE)
-    re_tagName = re.compile (r'\[([^]=\s]+)([^]]*)\]',  re.DOTALL | re.UNICODE)
-    #re_tagArgs = re.compile (r'\s*([^=]*)=(("([^"]+)")|([^\s]+))',  re.DOTALL | re.UNICODE)
-    re_tagArgs = re.compile (r'\s*([\w]*)=(("([^"]+)")|([^\s]+))',  re.DOTALL | re.UNICODE)
-
-    # get a unique name and replace escaped braces encode utf8 to
-    # prevent CLI/Web from barfing on unicode chars.  Not sure why
-    # this even needs to be 'unique' like this, but that's the way
-    # they wrote it.
-    unique = hashlib.md5(code.encode('utf8')).hexdigest()
-    code   = code.replace ('\[', unique+'_OPEN_BRACE')
-    code   = code.replace ('\]', unique+'_CLOSE_BRACE')
-
-    splitted = re_tags.split(code)
-
-    outTokenList = []
-    for token in splitted:
-      if len(token) == 0:
-        continue
-      
-      if token[0] == '[':
-        match   = re_tagName.match (token)
-        if match:
-          tagName = match.group(1)
-          tagArgs = match.group(2)
-          
-          tagToken = { 'tag' : tagName.lower() }
-  
-          # parse arguments (if any)
-          if len(tagArgs) > 0:
-            allArgs = re_tagArgs.findall(tagArgs)
-  
-            tagArgs = {}
-            for arg in allArgs:
-              # if the argument has no name, use the tagName itself
-              argName  = (arg[0] if arg[0] != '' else tagName)
-              argValue = (arg[3] if (arg[1][0] == '"') else arg[4])
-  
-              tagArgs[argName.lower()] = argValue.replace ('\"', '"')
-  
-            tagToken['args'] = tagArgs
-  
-          outTokenList.append (tagToken)
-
-        # no match, append the text as it is
-        else:
-          outTokenList.append (token)
-      # append the text as it is
-      else:
-        outTokenList.append (token)
-
-    # restore escaped braces back (once code is parsed)
-    restoredTokenList = []
-    for token in outTokenList:
-      if isinstance (token, basestring):
-        token = token.replace (unique+'_OPEN_BRACE', '[').replace (unique+'_CLOSE_BRACE', ']')
-      restoredTokenList.append (token)
-
-    return restoredTokenList
-

fanficdownloader/bbcodeutils/readme.txt

@@ -1,81 +0,0 @@
-AUTHOR
-  Pau Sanchez
-  http://www.codigomanso.com/
-
-VERSION:
-  bbcodeutils v1.0
-  
-LICENSE
-  This code is licensed under Creative Commons Attribution 3.0
-  http://creativecommons.org/licenses/by/3.0/
-
-  You can use this python module or any part of the code you want as long as you add
-  my name as a contributor to your project.
-  
-DESCRIPTION
-  This module can be used to produce HTML from BBCode, to generate BBCode or to fix invalid BBCode.
-
-  The classes are:
-    - bbcodeparser
-    - bbcodebuilder
-    - bbcode2html
-  
-  You can use bbcodeparser to parse BBCode and to produce output in any format you want.
-
-  Open the python file to find more information and examples of use of each class. It can
-  be a good idea to check the test.py for examples
-
-  To run the unit tests:
-    > python test.py
-  
-  To run the performance test:
-    > python test.py BBCodeTests.performanceTest  
-
-
-EXAMPLES OF BBCode:
-
-  [b] -> bold
-  [u] -> underline
-  [i] -> italic
-
-  [center] -> center the text inside
-  [color=XXX] -> change color of text
-  [size=XXX] -> change size of text
-
-  Lists:
-  [ul] -> unordered list
-  [ol] -> ordered list
-  [li] -> list item
-
-  [list] -> start unordered list
-  [*]    -> list item
-  [list=1] -> start a list of numbers
-  [list=a] -> start a list of alphabetic characters
-
-  Advanced:
-  [url] -> link to url
-  [url=http://link/url/]text[/url]  
-  [url link=http://link/url/ title="This is the title"]text[/url]  
-
-  [img]http://to/image[/img]
-  [img=230x330]http://to/image[/img]
-  [img="Alt text here"]http://to/image[/img]
-  [img="Alt text here" width=320 height=240]http://to/image[/img]
-
-  [email]asdf@asdf.com[/email]
-  [email=john@asdf.com]John Smith[/email]
-
-  [google]search this[/google]
-  [wikipedia]Tom Hanks[/wikipedia]
-  [wikipedia lang=es]Tom Hanks[/wikipedia]
-
-  Tables:
-  [table]
-  [tr]
-  [th]
-  [td]
-
-  Advanced:
-  [google]
-  [wikipedia]
-

fanficdownloader/bbcodeutils/test.py

@@ -1,420 +0,0 @@
-#!/usr/bin/python
-# -*- coding: UTF-8 -*-
-#
-# Author:        Pau Sanchez (contact@pausanchez.com)
-# Version:       v1.0
-# Last Modified: 2010/09/15
-# 
-# For the latest version check out:
-#   http://www.codigomanso.com/en/projects
-# 
-# My blog:
-#   http://www.codigomanso.com/en/  - English Version
-#   http://www.codigomanso.com/es/  - Spanish Version
-#
-
-from bbcodeparser import bbcodeparser
-from bbcodebuilder import bbcodebuilder
-
-import random
-import unittest
-
-class BBCodeTests(unittest.TestCase):
-  def setUp (self):
-    self.bbcode = bbcodeparser()
-    return
-
-  def testConstructor (self):
-    self.assertEqual (bbcodeparser ('whatever').html(), 'whatever')
-    self.assertEqual (bbcodeparser ('[b]bold[/b]').html(), '<b>bold</b>')
-    self.assertEqual (str (bbcodeparser ('[b]bold[/b]')), '[b]bold[/b]')
-    return
-
-  def testBold (self):
-    self.assertEqual (self.bbcode.parse ('whatever').html(), 'whatever')
-    self.assertEqual (self.bbcode.parse ('[b]bold[/b]').html(), '<b>bold</b>')
-    self.assertEqual (self.bbcode.parse ('[B]bold[/b]').html(), '<b>bold</b>')
-    self.assertEqual (self.bbcode.parse ('this is [B]bold[/B]').html(), 'this is <b>bold</b>')
-    return
-
-  def testItalic (self):
-    self.assertEqual (self.bbcode.parse ('[i]italic[/i]').html(), '<i>italic</i>')
-    return
-
-  def testUnderline (self):
-    self.assertEqual (self.bbcode.parse ('[u]italic[/u]').html(), '<u>italic</u>')
-    return
-
-  def testURLs (self):
-    self.assertEqual (
-      self.bbcode.parse ('[url]http://www.google.com[/url]').html(),
-      '<a href="http://www.google.com">http://www.google.com</a>'
-    )
-    self.assertEqual (
-      self.bbcode.parse ('[url=http://www.google.com]Google[/url]').html(),
-      '<a href="http://www.google.com">Google</a>'
-    )
-    self.assertEqual (
-      self.bbcode.parse ('[url="http://www.google.com"]Google[/url]').html(),
-      '<a href="http://www.google.com">Google</a>'
-    )
-    self.assertEqual (
-      self.bbcode.parse ('[url="http://www.google.com" title="Search Engine"]Google[/url]').html(),
-      '<a href="http://www.google.com" title="Search Engine">Google</a>'
-    )
-    self.assertEqual (
-      self.bbcode.parse ('[url link="http://www.google.com"]Google[/url]').html(),
-      '<a href="http://www.google.com">Google</a>'
-    )
-    return
-
-  def testPTag (self):
-    self.assertEqual (
-      self.bbcode.parse ('[p color=#0000ff]blue[/p]').html(),
-      u'<p style="color: #0000ff;">blue</p>'
-    )
-    self.assertEqual (
-      self.bbcode.parse ('[p size=12]12pt font[/p]').html(),
-      u'<p style="font-size: 12pt;">12pt font</p>'
-    )
-
-    self.assertEqual (
-      self.bbcode.parse ('[p font=arial]arial[/p]').html(),
-      u'<p style="font-family: arial;">arial</p>'
-    )
-
-    self.assertEqual (
-      self.bbcode.parse ('[p font=arial color=blue size=14]blue 14pt arial').html(),
-      u'<p style="color: blue; font-size: 14pt; font-family: arial;">blue 14pt arial</p>' 
-    )
-
-    self.assertEqual (
-      self.bbcode.parse ('[p class=whatever]text[/p]').html(),
-      u'<p>text</p>' 
-    )
-    return
-
-  def testColorTag (self):
-    self.assertEqual (
-      self.bbcode.parse ('[color=#0000ff]blue[/color]').html(),
-      u'<span style="color: #0000ff;">blue</span>'
-    )
-    return
-
-  def testSizeTag (self):
-    self.assertEqual (
-      self.bbcode.parse ('[size=12]12pt font[/size]').html(),
-      u'<span style="font-size: 12pt;">12pt font</span>'
-    )
-    return
-
-  def testEmail(self):
-    self.assertEqual (
-      self.bbcode.parse ('[email]asdf@asdf.com[/email]').html(),
-      u'<a href="mailto:asdf@asdf.com">asdf@asdf.com</a>'
-    )
-
-    self.assertEqual (
-      self.bbcode.parse ('[email=john@smith.com]John Smith[/email]').html(),
-      u'<a href="mailto:john@smith.com">John Smith</a>'
-    )
-    return
-
-  def testImgTag (self):
-    self.assertEqual (
-      self.bbcode.parse ('[img]http://www.codigomanso.com/image.jpg[/img]').html(),
-      u'<img src="http://www.codigomanso.com/image.jpg" />'
-    )
-
-    self.assertEqual (
-      self.bbcode.parse ('[img="This is the ALT of the image"]http://www.codigomanso.com/image.jpg[/img]').html(),
-      u'<img alt="This is the ALT of the image" src="http://www.codigomanso.com/image.jpg" />'
-    )
-
-    self.assertEqual (
-      self.bbcode.parse ('[img=320x200]http://www.codigomanso.com/image.jpg[/img]').html(),
-      u'<img height="200" src="http://www.codigomanso.com/image.jpg" width="320" />'
-    )
-
-    self.assertEqual (
-      self.bbcode.parse ('[img=320x200 title="Image Test"]http://www.codigomanso.com/image.jpg[/img]').html(),
-      u'<img height="200" src="http://www.codigomanso.com/image.jpg" title="Image Test" width="320" />'
-    )
-
-    self.assertEqual (
-      self.bbcode.parse ('[img="whatever" width=320 height="212" title="Image Test"]http://www.codigomanso.com/image.jpg[/img]').html(),
-      u'<img alt="whatever" height="212" src="http://www.codigomanso.com/image.jpg" title="Image Test" width="320" />'
-    )
-    return
-
-  def testGoogleURL (self):
-    self.assertEqual (
-      self.bbcode.parse ('[google]asdf[/google]').html(),
-      u'<a href="http://www.google.com/search?q=asdf">asdf</a>'
-    )
-    self.assertEqual (
-      self.bbcode.parse ('[google]Tom Hanks[/google]').html(),
-      u'<a href="http://www.google.com/search?q=Tom+Hanks">Tom Hanks</a>'
-    )
-    return
-    
-  def testWikipediaURL (self):
-    self.assertEqual (
-      self.bbcode.parse ('[wikipedia]Tom Hanks[/wikipedia]').html(),
-      u'<a href="http://www.wikipedia.org/wiki/Tom_Hanks">Tom Hanks</a>'
-    )
-
-    self.assertEqual (
-      self.bbcode.parse ('[wikipedia language=en]Tom Hanks[/wikipedia]').html(),
-      u'<a href="http://en.wikipedia.org/wiki/Tom_Hanks">Tom Hanks</a>'
-    )
-    
-    self.assertEqual (
-      self.bbcode.parse ('[wikipedia lang=es]Tom Hanks[/wikipedia]').html(),
-      u'<a href="http://es.wikipedia.org/wiki/Tom_Hanks">Tom Hanks</a>'
-    )
-    
-    self.assertEqual (
-      self.bbcode.parse ('[wikipedia=es]Tom Hanks[/wikipedia]').html(),
-      u'<a href="http://es.wikipedia.org/wiki/Tom_Hanks">Tom Hanks</a>'
-    )
-    return
-    
-  def testListTags (self):
-    self.assertEqual (
-      self.bbcode.parse ('[ul][li]item 1[/li][li]item 2[/li][/ul]').html(),
-      u'<ul><li>item 1</li><li>item 2</li></ul>'
-    )
-    
-    self.assertEqual (
-      self.bbcode.parse ('[ol][li]item 1[/li][li]item 2[/li][/ol]').html(),
-      u'<ol><li>item 1</li><li>item 2</li></ol>'
-    )
-
-    self.assertEqual (
-      self.bbcode.parse ('[list][li]item 1[/li][li]item 2[/li][/list]').html(),
-      u'<ul><li>item 1</li><li>item 2</li></ul>'
-    )
-
-    self.assertEqual (
-      self.bbcode.parse ('[list][*]item 1[*]item 2[/list]').html(),
-      u'<ul><li>item 1</li><li>item 2</li></ul>'
-    )
-
-    self.assertEqual (
-      self.bbcode.parse ('[list=1][li]item 1[/li][li]item 2[/li][/list]').html(),
-      u'<ol type="1"><li>item 1</li><li>item 2</li></ol>'
-    )
-    return
-
-  def testInvalidCode (self):
-    self.assertEqual (self.bbcode.parse ('[invalid]valid text[/invalid]').html(), 'valid text')
-    self.assertEqual (
-      self.bbcode.parse ('[b]bold and [i]italics[/b]').html(),
-      '<b>bold and <i>italics</i></b>'
-    )
-    self.assertEqual (
-      self.bbcode.parse ('[/b]invalid[/b][/p]').html(),
-      'invalid'
-    )
-    self.assertEqual (
-      self.bbcode.parse ('[p][b]bold').html(),
-      '<p><b>bold</b></p>'
-    )
-    self.assertEqual (
-      self.bbcode.parse ('[p][b]a <b>').html(),
-      '<p><b>a &lt;b&gt;</b></p>'
-    )
-
-    self.assertEqual (
-      self.bbcode.parse ('[ol][li]item 1[li]item 2[/li][/ol]').html(),
-      u'<ol><li>item 1</li><li>item 2</li></ol>'
-    )
-
-    self.assertEqual (
-      self.bbcode.parse ('[b]\[b\] stands for [b]bold[/b]').html(),
-      u'<b>[b] stands for </b><b>bold</b>'
-    )
-    return
-
-  def testEscapedBrackets (self):
-    self.assertEqual (
-      self.bbcode.parse ('\[b\]not bold\[/b\]').html(),
-      u'[b]not bold[/b]'
-    )
-
-    self.assertEqual (
-      self.bbcode.parse ('[b]\[b\] stands for bold[/b]').html(),
-      u'<b>[b] stands for bold</b>'
-    )
-
-    self.assertEqual (
-      self.bbcode.parse ('\[b\][b]stands for bold[/b]').html(),
-      u'[b]<b>stands for bold</b>'
-    )
-
-    self.assertEqual (
-      self.bbcode.parse ('\[b\][b]stands for bold[/b] just like <b> in HTML').html(),
-      u'[b]<b>stands for bold</b> just like &lt;b&gt; in HTML'
-    )
-
-  def testBigExample (self):
-    inputText = """check this out
-
-    [h1 class=circle]heading[/h1]
-
-    [p size=14 color=blue font="verdana, Times New Roman"]This is [b] bold [/b] and this [i]italic[/i] and this is [color=red]red[/color] and this is [color="red"]also red[/color].
-    [/p]
-
-    fix [b][i]bold [font=verdana][size=12]and[/size][/font] italic[/b]
-    [img]http://www.codigomanso.com/b.jpg[/img]
-    [url]http://www.codigomanso.com/[/url]
-    [url=http://www.codigomanso.com/]Codigo Manso[/url]
-    [uRl link=http://www.codigomanso.com title="Codigo Manso Blog"]Codigo Manso[/url]
-
-    [ul]
-     [Li]item 1[/Li]
-     [li]item 2[/LI]
-    [/UL]
-
-    [list=1 ]
-     [*]item 1
-     [*]item 2
-    [/list]
-
-    [table class="big"]
-      [tr]
-        [th]big[/th]
-      [/tr]
-    [/table]
-    [invalid class="extra"]whatever[/invalid]"""
-
-    out = self.bbcode.parse (inputText).html(allowClassAttr = True)
-    self.assertEquals (out, '''check this out
-
-    <h1 class="circle">heading</h1>
-
-    <p style="color: blue; font-size: 14pt; font-family: verdana, Times New Roman;">This is <b> bold </b> and this <i>italic</i> and this is <span style="color: red;">red</span> and this is <span style="color: red;">also red</span>.
-    </p>
-
-    fix <b><i>bold <span style="font-family: verdana;"><span style="font-size: 12pt;">and</span></span> italic</i></b>
-    <img src="http://www.codigomanso.com/b.jpg" />
-    <a href="http://www.codigomanso.com/">http://www.codigomanso.com/</a>
-    <a href="http://www.codigomanso.com/">Codigo Manso</a>
-    <a href="http://www.codigomanso.com" title="Codigo Manso Blog">Codigo Manso</a>
-
-    <ul>
-     <li>item 1</li>
-     <li>item 2</li>
-    </ul>
-
-    <ol type="1">
-     <li>item 1
-     </li><li>item 2
-    </li></ol>
-
-    <table class="big">
-      <tr>
-        <th>big</th>
-      </tr>
-    </table>
-    whatever''')
-
-
-  def testBBCodeDumper (self):
-    self.assertEquals (
-      self.bbcode.parse ('[b]bold[/b]').bbcode(),
-      '[b]bold[/b]'
-    )
-
-    self.assertEquals (
-      self.bbcode.parse ('[color=red]text in red[/color]').bbcode(),
-      '[color=red]text in red[/color]'
-    )
-    self.assertEquals (
-      self.bbcode.parse ('[p][color=red]text in red').bbcode(),
-      '[p][color=red]text in red[/color][/p]'
-    )
-
-    self.assertEquals (
-      self.bbcode.parse ('This [b][i]code[/b] will be fixed[/invalid]').bbcode(),
-      'This [b][i]code[/i][/b] will be fixed'
-    )
-
-    self.assertEquals (
-      self.bbcode.parse ('\[[url]http://www.codigomanso.com/en[/url]\]').bbcode(),
-      "\[[url]http://www.codigomanso.com/en[/url]\]"
-    )
-
-  def performanceTest(self):
-    '''
-    This test checks the performance of parse and html operations
-
-    To run this test type:
-      > python test.py BBCodeTests.performanceTest
-    '''
-    inputText = """check this out
-
-    [h1 class=circle]heading[/h1]
-
-    [p size=14 color=blue font="verdana, Times New Roman"]This is [b] bold [/b] and this [i]italic[/i] and this is [color=red]red[/color] and this is [color="red"]also red[/color].
-    [/p]
-
-    fix [b][i]bold [font=verdana][size=12]and[/size][/font] italic[/b]
-    [img]http://www.codigomanso.com/b.jpg[/img]
-    [url]http://www.codigomanso.com/[/url]
-    [url=http://www.codigomanso.com/]Codigo Manso[/url]
-    [uRl link=http://www.codigomanso.com title="Codigo Manso Blog"]Codigo Manso[/url]
-
-    [ul]
-     [Li]item 1[/Li]
-     [li]item 2[/LI]
-    [/UL]
-
-    [list=1 ]
-     [*]item 1
-     [*]item 2
-    [/list]
-
-    [table class="big"]
-      [tr]
-        [th]big[/th]
-      [/tr]
-    [/table]
-    [invalid class="extra"]whatever[/invalid]"""
-
-    import time
-    start = time.time()
-
-    for i in range(0, 12):
-      inputText += inputText
-
-    print "len(inputText) = %.2f MB  (took %.2f seconds)" % (len(inputText)/(1024.0*1024.0), time.time() - start)
-    
-    bbcode = bbcodeparser()
-    start = time.time()
-    bbcode.parse (inputText)
-    total = (time.time() - start)
-    print "time (bbcode.parse()) = %f" % total
-    print "  >> %.2f chars/second" % (len(inputText) / total)
-
-    start = time.time()
-    bbcode.html(doDeepCopy = False)
-    total = (time.time() - start)
-    print "time (bbcode.html()) = %f" % total
-    print "  >> %.2f chars/second" % (len(inputText) / total)
-    return
-    
-  def testCodeBuilder (self):
-    bbcode = bbcodebuilder ()
-    self.assertEquals (bbcode.b ('bold'), u'[b]bold[/b]')
-    self.assertEquals (bbcode.color ('this goes in red', 'red'), u'[color=red]this goes in red[/color]')
-    self.assertEquals (bbcode.url ('Google', 'http://www.google.com'), u'[url=http://www.google.com]Google[/url]')
-    self.assertEquals (bbcode.alist('item 1', 'item 2'), u"[list=a]\n  [*]item 1\n  [*]item 2\n[/list]")
-    return
-
-if __name__ == '__main__':
-    unittest.main()
-
-
-

fanficdownloader/configurable.py

@@ -15,7 +15,7 @@
 # limitations under the License.
 #
 
-import ConfigParser
+import ConfigParser, re
 
 # All of the writers(epub,html,txt) and adapters(ffnet,twlt,etc)
 # inherit from Configurable.  The config file(s) uses ini format:
@@ -37,10 +37,20 @@ class Configuration(ConfigParser.SafeConfigParser):
     def __init__(self, site, fileform):
         ConfigParser.SafeConfigParser.__init__(self)
         self.sectionslist = ['defaults']
-        self.addConfigSection(site)
+
+        if site.startswith("www."):
+            sitewith = site
+            sitewithout = site.replace("www.","")
+        else:
+            sitewith = "www."+site
+            sitewithout = site
+        
+        self.addConfigSection(sitewith)
+        self.addConfigSection(sitewithout)
         if fileform:
             self.addConfigSection(fileform)
-            self.addConfigSection(site+":"+fileform)
+            self.addConfigSection(sitewith+":"+fileform)
+            self.addConfigSection(sitewithout+":"+fileform)
         self.addConfigSection("overrides")
         
         self.validEntries = [
@@ -50,6 +60,7 @@ class Configuration(ConfigParser.SafeConfigParser):
             'characters',
             'ships',
             'series',
+            'seriesUrl',
             'status',
             'datePublished',
             'dateUpdated',
@@ -75,6 +86,7 @@ class Configuration(ConfigParser.SafeConfigParser):
             'langcode',
             'output_css',
             'authorHTML',
+            'seriesHTML',
             'lastupdate'
             ]
 
@@ -87,37 +99,62 @@ class Configuration(ConfigParser.SafeConfigParser):
     def getValidMetaList(self):
         return self.validEntries + self.getConfigList("extra_valid_entries")
 
+    # used by adapters & writers, non-convention naming style
     def hasConfig(self, key):
-        for section in self.sectionslist:
+        return self.has_config(self.sectionslist, key)
+
+    def has_config(self, sections, key):
+        for section in sections:
             try:
                 self.get(section,key)
                 #print("found %s in section [%s]"%(key,section))
                 return True
             except:
-                pass
+                try:
+                    self.get(section,"add_to_"+key)
+                    #print("found add_to_%s in section [%s]"%(key,section))
+                    return True
+                except:
+                    pass
 
         return False
-        
+
+    # used by adapters & writers, non-convention naming style
     def getConfig(self, key, default=""):
+        return self.get_config(self.sectionslist,key,default)
+    
+    def get_config(self, sections, key, default=""):        
         val = default
-        for section in self.sectionslist:
+        for section in sections:
             try:
                 val = self.get(section,key)
                 if val and val.lower() == "false":
                     val = False
                 #print "getConfig(%s)=[%s]%s" % (key,section,val)
-                return val
+                break
             except (ConfigParser.NoOptionError, ConfigParser.NoSectionError), e:
                 pass
 
+        for section in sections[::-1]:
+            # 'martian smiley' [::-1] reverses list by slicing whole list with -1 step.
+            try:
+                val = val + self.get(section,"add_to_"+key)
+                #print "getConfig(add_to_%s)=[%s]%s" % (key,section,val)
+            except (ConfigParser.NoOptionError, ConfigParser.NoSectionError), e:
+                pass
+            
         return val
 
     # split and strip each.
-    def getConfigList(self, key):
-        vlist = self.getConfig(key).split(',')
-        vlist = [ v.strip() for v in vlist ]
+    def get_config_list(self, sections, key):
+        vlist = re.split(r'(?<!\\),',self.get_config(sections,key)) # don't split on \,
+        vlist = filter( lambda x : x !='', [ v.strip().replace('\,',',') for v in vlist ])
         #print "vlist("+key+"):"+str(vlist)
         return vlist        
+    
+    # used by adapters & writers, non-convention naming style
+    def getConfigList(self, key):
+        return self.get_config_list(self.sectionslist, key)
 
 # extended by adapter, writer and story for ease of calling configuration.
 class Configurable(object):
@@ -132,11 +169,19 @@ class Configurable(object):
         return self.configuration.getValidMetaList()
     
     def hasConfig(self, key):
-        return self.configuration.hasConfig(key)
+        return self.configuration.hasConfig(key)        
         
+    def has_config(self, sections, key):
+        return self.configuration.has_config(sections, key)
+
     def getConfig(self, key, default=""):
         return self.configuration.getConfig(key,default)
 
+    def get_config(self, sections, key, default=""):
+        return self.configuration.get_config(sections,key,default)
+
     def getConfigList(self, key):
         return self.configuration.getConfigList(key)
-        
+
+    def get_config_list(self, sections, key):
+        return self.configuration.get_config_list(sections,key)

fanficdownloader/epubutils.py

@@ -130,6 +130,9 @@ def get_update_data(inputio,
                         h2 = soup.find('h2')
                         if h2:
                             h2.extract()
+
+                        for skip in soup.findAll(attrs={'class':'skip_on_ffdl_update'}):
+                            skip.extract()
                             
                         soups.append(soup)
                         

fanficdownloader/exceptions.py

@@ -68,3 +68,10 @@ class UnknownSite(Exception):
     def __str__(self):
         return "Unknown Site(%s).  Supported sites: (%s)" % (self.url, ", ".join(self.supported_sites_list))
 
+class FailedToWriteOutput(Exception):
+    def __init__(self,error):
+        self.error=error
+
+    def __str__(self):
+        return self.error
+

fanficdownloader/geturls.py

@@ -25,10 +25,8 @@ from gziphttp import GZipProcessor
 import adapters
 from configurable import Configuration
 
-def get_urls_from_page(url,configuration=None):
+def get_urls_from_page(url,configuration=None,normalize=False):
 
-    normalized = set() # normalized url
-    retlist = [] # orig urls.
     if not configuration:
         configuration = Configuration("test1.com","EPUB")
 
@@ -56,7 +54,25 @@ def get_urls_from_page(url,configuration=None):
         opener = u2.build_opener(u2.HTTPCookieProcessor(),GZipProcessor())
         data = opener.open(url).read()
 
+    # kludge because I don't see it on enough sites to be worth generalizing yet.
+    restrictsearch=None
+    if 'scarvesandcoffee.net' in url:
+        restrictsearch=('div',{'id':'mainpage'})
+
+    return get_urls_from_html(data,url,configuration,normalize,restrictsearch)
+
+def get_urls_from_html(data,url=None,configuration=None,normalize=False,restrictsearch=None):
+
+    normalized = [] # normalized url
+    retlist = [] # orig urls.
+    
+    if not configuration:
+        configuration = Configuration("test1.com","EPUB")
+
     soup = BeautifulSoup(data)
+    if restrictsearch:
+        soup = soup.find(*restrictsearch)
+        print("restrict search:%s"%soup)
     
     for a in soup.findAll('a'):
         if a.has_key('href'):
@@ -74,12 +90,45 @@ def get_urls_from_page(url,configuration=None):
                 href = href.replace('&index=1','')
                 adapter = adapters.getAdapter(configuration,href)
                 if adapter.story.getMetadata('storyUrl') not in normalized:
-                    normalized.add(adapter.story.getMetadata('storyUrl'))
+                    normalized.append(adapter.story.getMetadata('storyUrl'))
                     retlist.append(href)
             except:
                 pass
 
-    return retlist
+    if normalize:
+        return normalized
+    else:
+        return retlist
+
+def get_urls_from_text(data,configuration=None,normalize=False):
+
+    normalized = [] # normalized url
+    retlist = [] # orig urls.
+    
+    if not configuration:
+        configuration = Configuration("test1.com","EPUB")
+    
+    for href in re.findall('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+', data):
+        # this (should) catch normal story links, some javascript
+        # 'are you old enough' links, and 'Report This' links.
+        # The 'normalized' set prevents duplicates.
+        if 'story.php' in href:
+            m = re.search(r"(?P<sid>(view)?story\.php\?(sid|psid|no|story|stid)=\d+)",href)
+            if m != None:
+                href = form_url(href,m.group('sid'))
+        try:
+            href = href.replace('&index=1','')
+            adapter = adapters.getAdapter(configuration,href)
+            if adapter.story.getMetadata('storyUrl') not in normalized:
+                normalized.append(adapter.story.getMetadata('storyUrl'))
+                retlist.append(href)
+        except:
+            pass
+
+    if normalize:
+        return normalized
+    else:
+        return retlist
 
 def form_url(parenturl,url):
      url = url.strip() # ran across an image with a space in the

fanficdownloader/html.py

@@ -17,7 +17,7 @@ class HtmlProcessor:
     self.unfill = unfill
     html = self._ProcessRawHtml(html)
     self._soup = BeautifulSoup(html)
-    if self._soup.title:
+    if self._soup.title.contents:
       self.title = self._soup.title.contents[0]
     else:
       self.title = None

fanficdownloader/mobi.py

@@ -8,6 +8,8 @@ import time
 import random
 import logging
 
+logger = logging.getLogger(__name__)
+
 from html import HtmlProcessor
 
 # http://wiki.mobileread.com/wiki/MOBI
@@ -124,8 +126,8 @@ class Converter:
       tmp = self.MakeOneHTML(html_strs)
       self._ConvertStringToFile(tmp, out_file)
     except Exception, e:
-      logging.error('Error %s', e)
-      logging.debug('Details: %s' % html_strs)
+      logger.error('Error %s', e)
+      #logger.debug('Details: %s' % html_strs)
 
   def _ConvertStringToFile(self, html_data, out):
     html = HtmlProcessor(html_data)

fanficdownloader/story.py

@@ -280,10 +280,10 @@ class Story(Configurable):
         for line in replace.splitlines():
             (metakeys,regexp,replacement,condkey,condregexp)=(None,None,None,None,None)
             if "&&" in line:
-                (line,conditional) = map( lambda x: x.strip(), line.split("&&") )
-                (condkey,condregexp) = map( lambda x: x.strip(), conditional.split("=>") )
+                (line,conditional) = line.split("&&")
+                (condkey,condregexp) = conditional.split("=>")
             if "=>" in line:
-                parts = map( lambda x: x.strip(), line.split("=>") )
+                parts = line.split("=>")
                 if len(parts) > 2:
                     metakeys = map( lambda x: x.strip(), parts[0].split(",") )
                     (regexp,replacement)=parts[1:]
@@ -294,9 +294,13 @@ class Story(Configurable):
                 regexp = re.compile(regexp)
                 if condregexp:
                     condregexp = re.compile(condregexp)
+                # A way to explicitly include spaces in the
+                # replacement string.  The .ini parser eats any
+                # trailing spaces.
+                replacement=replacement.replace('\s',' ') 
                 self.replacements.append([metakeys,regexp,replacement,condkey,condregexp])
     
-    def doReplacments(self,value,key):
+    def doReplacements(self,value,key):
         for (metakeys,regexp,replacement,condkey,condregexp) in self.replacements:
             if (metakeys == None or key in metakeys) \
                     and isinstance(value,basestring) \
@@ -336,7 +340,7 @@ class Story(Configurable):
                     value = value.strftime(self.getConfig(key+"_format","%Y-%m-%d"))
 
             if doreplacements:
-                value=self.doReplacments(value,key)
+                value=self.doReplacements(value,key)
             if removeallentities and value != None:
                 return removeAllEntities(value)
             else:
@@ -352,7 +356,7 @@ class Story(Configurable):
         allmetadata = {}
         
         # special handling for authors/authorUrls
-        authlinkhtml="<a class='authorlink' href='%s'>%s</a>"
+        linkhtml="<a class='%slink' href='%s'>%s</a>"
         if self.isList('author'): # more than one author, assume multiple authorUrl too.
             htmllist=[]
             for i, v in enumerate(self.getList('author')):
@@ -360,18 +364,24 @@ class Story(Configurable):
                 auth = v
                 # make sure doreplacements & removeallentities are honored.
                 if doreplacements:
-                    aurl=self.doReplacments(aurl,'authorUrl')
-                    auth=self.doReplacments(auth,'author')
+                    aurl=self.doReplacements(aurl,'authorUrl')
+                    auth=self.doReplacements(auth,'author')
                 if removeallentities:
                     aurl=removeAllEntities(aurl)
                     auth=removeAllEntities(auth)
                 
-                htmllist.append(authlinkhtml%(aurl,auth))
+                htmllist.append(linkhtml%('author',aurl,auth))
             self.setMetadata('authorHTML',', '.join(htmllist))
         else:
-           self.setMetadata('authorHTML',authlinkhtml%(self.getMetadata('authorUrl', removeallentities, doreplacements),
-                                                       self.getMetadata('author', removeallentities, doreplacements)))
+            self.setMetadata('authorHTML',linkhtml%('author',self.getMetadata('authorUrl', removeallentities, doreplacements),
+                                                    self.getMetadata('author', removeallentities, doreplacements)))
 
+        if self.getMetadataRaw('seriesUrl') != None:
+            self.setMetadata('seriesHTML',linkhtml%('series',self.getMetadata('seriesUrl', removeallentities, doreplacements),
+                                                    self.getMetadata('series', removeallentities, doreplacements)))
+        elif self.getMetadataRaw('series') != None:
+            self.setMetadata('seriesHTML',self.getMetadataRaw('series'))
+            
         for k in self.getValidMetaList():
             if self.isList(k) and keeplists:
                 allmetadata[k] = self.getList(k, removeallentities, doreplacements)
@@ -415,18 +425,20 @@ class Story(Configurable):
         # includelist prevents infinite recursion of include_in_'s
         if self.hasConfig("include_in_"+listname) and listname not in includelist:
             for k in self.getConfigList("include_in_"+listname):
-                retlist.extend(self.getList(k,removeallentities,doreplacements,includelist=includelist+[listname]))
+                retlist.extend(self.getList(k,removeallentities=False,
+                                            doreplacements=doreplacements,includelist=includelist+[listname]))
         else:
         
             if not self.isList(listname):
-                retlist = [self.getMetadata(listname,removeallentities, doreplacements)]
+                retlist = [self.getMetadata(listname,removeallentities=False,
+                                            doreplacements=doreplacements)]
             else:
                 retlist = self.getMetadataRaw(listname)
 
         if retlist:
             if doreplacements:
                 retlist = filter( lambda x : x!=None and x!='' ,
-                                  map(partial(self.doReplacments,key=listname),retlist) )
+                                  map(partial(self.doReplacements,key=listname),retlist) )
             if removeallentities:
                 retlist = filter( lambda x : x!=None and x!='' ,
                                   map(removeAllEntities,retlist) )
@@ -462,11 +474,11 @@ class Story(Configurable):
 
         return list(subjectset | set(self.getConfigList("extratags")))
             
-    def addChapter(self, title, html):
+    def addChapter(self, url, title, html):
         if self.getConfig('strip_chapter_numbers') and \
                 self.getConfig('chapter_title_strip_pattern'):
             title = re.sub(self.getConfig('chapter_title_strip_pattern'),"",title)
-        self.chapters.append( (title,html) )
+        self.chapters.append( (url,title,html) )
 
     def getChapters(self,fortoc=False):
         "Chapters will be tuples of (title,html)"
@@ -476,8 +488,10 @@ class Story(Configurable):
                 (self.getConfig('add_chapter_numbers') == "true" \
                      or (self.getConfig('add_chapter_numbers') == "toconly" and fortoc)) \
                      and self.getConfig('chapter_title_add_pattern'):
-            for index, (title,html) in enumerate(self.chapters):
-                retval.append( (string.Template(self.getConfig('chapter_title_add_pattern')).substitute({'index':index+1,'title':title}),html) ) 
+            for index, (url,title,html) in enumerate(self.chapters):
+                retval.append( (url,
+                                string.Template(self.getConfig('chapter_title_add_pattern')).substitute({'index':index+1,'title':title}),
+                                html) )
         else:
             retval = self.chapters
             

fanficdownloader/writers/base_writer.py

@@ -177,11 +177,12 @@ class BaseStoryWriter(Configurable):
             
             self._write(out,START.substitute(self.story.getAllMetadata()))
 
-            for index, (title,html) in enumerate(self.story.getChapters(fortoc=True)):
+            for index, (url,title,html) in enumerate(self.story.getChapters(fortoc=True)):
                 if html:
                     self._write(out,ENTRY.substitute({'chapter':title,
                                                       'number':index+1,
-                                                      'index':"%04d"%(index+1)}))
+                                                      'index':"%04d"%(index+1),
+                                                      'url':url}))
 
             self._write(out,END.substitute(self.story.getAllMetadata()))
 

fanficdownloader/writers/writer_epub.py

@@ -501,7 +501,7 @@ div { margin: 0pt; padding: 0pt; }
             items.append(("log_page","OEBPS/log_page.xhtml","application/xhtml+xml","Update Log"))
             itemrefs.append("log_page")
             
-        for index, (title,html) in enumerate(self.story.getChapters(fortoc=True)):
+        for index, (url,title,html) in enumerate(self.story.getChapters(fortoc=True)):
             if html:
                 i=index+1
                 items.append(("file%04d"%i,
@@ -649,10 +649,10 @@ div { margin: 0pt; padding: 0pt; }
         else:
             CHAPTER_END = self.EPUB_CHAPTER_END
         
-        for index, (title,html) in enumerate(self.story.getChapters()):
+        for index, (url,title,html) in enumerate(self.story.getChapters()):
             if html:
                 logger.debug('Writing chapter text for: %s' % title)
-                vals={'chapter':title, 'index':"%04d"%(index+1), 'number':index+1}
+                vals={'url':url, 'chapter':title, 'index':"%04d"%(index+1), 'number':index+1}
                 fullhtml = CHAPTER_START.substitute(vals) + html + CHAPTER_END.substitute(vals)
                 # ffnet(& maybe others) gives the whole chapter text
                 # as one line.  This causes problems for nook(at

fanficdownloader/writers/writer_html.py

@@ -128,10 +128,10 @@ ${output_css}
         else:
             CHAPTER_END = self.HTML_CHAPTER_END
         
-        for index, (title,html) in enumerate(self.story.getChapters()):
+        for index, (url,title,html) in enumerate(self.story.getChapters()):
             if html:
                 logging.debug('Writing chapter text for: %s' % title)
-                vals={'chapter':title, 'index':"%04d"%(index+1), 'number':index+1}
+                vals={'url':url, 'chapter':title, 'index':"%04d"%(index+1), 'number':index+1}
                 self._write(out,CHAPTER_START.substitute(vals))
                 self._write(out,html)
                 self._write(out,CHAPTER_END.substitute(vals))

fanficdownloader/writers/writer_mobi.py

@@ -22,6 +22,9 @@ import StringIO
 from base_writer import *
 from ..htmlcleanup import stripHTML
 from ..mobi import Converter
+from ..exceptions import FailedToWriteOutput
+
+logger = logging.getLogger(__name__)
 
 class MobiWriter(BaseStoryWriter):
 
@@ -158,10 +161,10 @@ ${value}<br />
         else:
             CHAPTER_END = self.MOBI_CHAPTER_END
         
-        for index, (title,html) in enumerate(self.story.getChapters()):
+        for index, (url,title,html) in enumerate(self.story.getChapters()):
             if html:
-                logging.debug('Writing chapter text for: %s' % title)
-                vals={'chapter':title, 'index':"%04d"%(index+1), 'number':index+1}
+                logger.debug('Writing chapter text for: %s' % title)
+                vals={'url':url, 'chapter':title, 'index':"%04d"%(index+1), 'number':index+1}
                 fullhtml = CHAPTER_START.substitute(vals) + html + CHAPTER_END.substitute(vals)
                 # ffnet(& maybe others) gives the whole chapter text
                 # as one line.  This causes problems for nook(at
@@ -175,6 +178,8 @@ ${value}<br />
                       author=self.getMetadata('author'),
                       publisher=self.getMetadata('site'))
         mobidata = c.ConvertStrings(files)
+        if len(mobidata) < 1:
+            raise FailedToWriteOutput("Zero length mobi output")
         out.write(mobidata)
         
         del files

fanficdownloader/writers/writer_txt.py

@@ -154,10 +154,10 @@ End file.
         else:
             CHAPTER_END = self.TEXT_CHAPTER_END
         
-        for index, (title,html) in enumerate(self.story.getChapters()):
+        for index, (url, title,html) in enumerate(self.story.getChapters()):
             if html:
                 logging.debug('Writing chapter text for: %s' % title)
-                vals={'chapter':title, 'index':"%04d"%(index+1), 'number':index+1}
+                vals={'url':url, 'chapter':title, 'index':"%04d"%(index+1), 'number':index+1}
                 self._write(out,self.lineends(self.wraplines(removeAllEntities(CHAPTER_START.substitute(vals)))))
                 self._write(out,self.lineends(html2text(html,wrap_width=self.wrap_width)))
                 self._write(out,self.lineends(self.wraplines(removeAllEntities(CHAPTER_END.substitute(vals)))))