Bump versions, update index.html.

AO3 doesn't reliably report chapter author.

app.yaml

@@ -1,6 +1,6 @@
 # ffd-retief-hrd fanfictiondownloader
 application: fanfictiondownloader
-version: 4-4-46
+version: 4-4-59
 runtime: python27
 api_version: 1
 threadsafe: true

calibre-plugin/__init__.py

@@ -1,11 +1,10 @@
 #!/usr/bin/env python
-# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
 # -*- coding: utf-8 -*-
 from __future__ import (unicode_literals, division, absolute_import,
                         print_function)
 
 __license__   = 'GPL v3'
-__copyright__ = '2011, Jim Miller'
+__copyright__ = '2013, Jim Miller'
 __docformat__ = 'restructuredtext en'
 
 # The class that all Interface Action plugin wrappers must inherit from
@@ -27,7 +26,7 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
     description         = 'UI plugin to download FanFiction stories from various sites.'
     supported_platforms = ['windows', 'osx', 'linux']
     author              = 'Jim Miller'
-    version             = (1, 7, 13)
+    version             = (1, 7, 26)
     minimum_calibre_version = (0, 8, 57)
 
     #: This field defines the GUI plugin class that contains all the code

calibre-plugin/common_utils.py

@@ -239,8 +239,8 @@ class SizePersistedDialog(QDialog):
             self.restoreGeometry(self.geom)
 
     def dialog_closing(self, result):
-        geom = bytearray(self.saveGeometry())
-        gprefs[self.unique_pref_name] = geom
+        self.geom = bytearray(self.saveGeometry())
+        gprefs[self.unique_pref_name] = self.geom
 
 
 class ReadOnlyTableWidgetItem(QTableWidgetItem):

calibre-plugin/config.py

@@ -18,10 +18,10 @@ from PyQt4.Qt import (QDialog, QWidget, QVBoxLayout, QHBoxLayout, QLabel,
 from calibre.utils.config import JSONConfig
 from calibre.gui2.ui import get_gui
 
-from calibre_plugins.fanfictiondownloader_plugin.prefs import prefs
+from calibre_plugins.fanfictiondownloader_plugin.prefs import prefs, PREFS_NAMESPACE
 from calibre_plugins.fanfictiondownloader_plugin.dialogs \
     import (UPDATE, UPDATEALWAYS, OVERWRITE, collision_order, RejectListDialog,
-            EditTextDialog)
+            EditTextDialog, RejectUrlEntry)
     
 from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.adapters \
     import (getConfigSections, getNormalStoryURL)
@@ -37,22 +37,16 @@ class RejectURLList:
         self.sync_lock = threading.RLock()
         self.listcache = None
 
-    def _read_list_from_text(self,text,addreasontext=None):
-        cache = {}
+    def _read_list_from_text(self,text,addreasontext=''):
+        cache = OrderedDict()
+
+        #print("_read_list_from_text")
         for line in text.splitlines():
-            if ',' in line:
-                (rejurl,note) = line.split(',',1)
-            else:
-                (rejurl,note) = (line,'')
-            rejurl = getNormalStoryURL(rejurl)
-            if rejurl:
-                if addreasontext and note:
-                    note = note +" - "+addreasontext
-                elif addreasontext:
-                    note = addreasontext
-                cache[rejurl] = note
-        return cache
-        
+            rue = RejectUrlEntry(line,addreasontext=addreasontext,fromline=True)
+            #print("rue.url:%s"%rue.url)
+            if rue.valid:
+                cache[rue.url] = rue
+        return cache        
 
     def _get_listcache(self):
         if self.listcache == None:
@@ -60,26 +54,35 @@ class RejectURLList:
         return self.listcache
 
     def _save_list(self,listcache):
-        rejectlist = []
-        for url in listcache:
-            rejectlist.append("%s,%s"%(url,listcache[url]))
-            
-        self.prefs['rejecturls'] = '\n'.join(rejectlist)
+        #print("_save_list")
+        self.prefs['rejecturls'] = '\n'.join([x.to_line() for x in listcache.values()])
         self.prefs.save_to_db()
         self.listcache = None
         
     def clear_cache(self):
         self.listcache = None
 
+    # true if url is in list.
     def check(self,url):
+        with self.sync_lock:
+            listcache = self._get_listcache()
+            return url in listcache
+        
+    def get_note(self,url):
         with self.sync_lock:
             listcache = self._get_listcache()
             if url in listcache:
-                note = listcache[url]
-                return note
-            
+                return listcache[url].note
             # not found
-            return None
+            return ''
+
+    def get_full_note(self,url):
+        with self.sync_lock:
+            listcache = self._get_listcache()
+            if url in listcache:
+                return listcache[url].fullnote()
+            # not found
+            return ''
 
     def remove(self,url):
         with self.sync_lock:
@@ -89,27 +92,26 @@ class RejectURLList:
                 self._save_list(listcache)
 
     def add_text(self,rejecttext,addreasontext):
-        self.add(self._read_list_from_text(rejecttext,addreasontext).items())
+        self.add(self._read_list_from_text(rejecttext,addreasontext).values())
             
     def add(self,rejectlist,clear=False):
-        # rejectlist=list of (url,note) tuples.
         with self.sync_lock:
             if clear:
-                listcache={}
+                listcache=OrderedDict()
             else:
                 listcache = self._get_listcache()
-            for (url,note) in rejectlist:
-                listcache[url]=note
+            for l in rejectlist:
+                listcache[l.url]=l
             self._save_list(listcache)
 
     def get_list(self):
-        return copy.deepcopy(self._get_listcache())
+        return self._get_listcache().values()
             
     def get_reject_reasons(self):
         return self.prefs['rejectreasons'].splitlines()
 
 rejecturllist = RejectURLList(prefs)
-    
+
 class ConfigWidget(QWidget):
 
     def __init__(self, plugin_action):
@@ -173,6 +175,7 @@ class ConfigWidget(QWidget):
         prefs['adddialogstaysontop'] = self.basic_tab.adddialogstaysontop.isChecked()
         prefs['includeimages'] = self.basic_tab.includeimages.isChecked()
         prefs['lookforurlinhtml'] = self.basic_tab.lookforurlinhtml.isChecked()
+        prefs['checkforseriesurlid'] = self.basic_tab.checkforseriesurlid.isChecked()
         prefs['injectseries'] = self.basic_tab.injectseries.isChecked()
 
         if self.readinglist_tab:
@@ -369,6 +372,11 @@ class BasicTab(QWidget):
         self.lookforurlinhtml.setChecked(prefs['lookforurlinhtml'])
         self.l.addWidget(self.lookforurlinhtml)
 
+        self.checkforseriesurlid = QCheckBox("Check for existing Series Anthology books?",self)
+        self.checkforseriesurlid.setToolTip("Check for existings Series Anthology books using each new story's series URL before downloading.\nOffer to skip downloading if a Series Anthology is found.")
+        self.checkforseriesurlid.setChecked(prefs['checkforseriesurlid'])
+        self.l.addWidget(self.checkforseriesurlid)
+
         self.injectseries = QCheckBox("Inject calibre Series when none found?",self)
         self.injectseries.setToolTip("If no series is found, inject the calibre series (if there is one) so it appears on the FFDL title page(not cover).")
         self.injectseries.setChecked(prefs['injectseries'])
@@ -412,12 +420,8 @@ class BasicTab(QWidget):
         ShowDefaultsIniDialog(self.windowIcon(),text,self).exec_()
 
     def show_rejectlist(self):
-        rejectlist = []
-        for (url,note) in rejecturllist.get_list().items():
-            rejectlist.append((None,url,note,note))
-            
         d = RejectListDialog(self,
-                             rejectlist,
+                             rejecturllist.get_list(),
                              rejectreasons=rejecturllist.get_reject_reasons(),
                              header="Edit Reject URLs List",
                              show_delete=False,
@@ -427,11 +431,7 @@ class BasicTab(QWidget):
         if d.result() != d.Accepted:
             return
         
-        rejectlist=[]
-        for (bookid,url,note) in d.get_reject_list():
-            rejectlist.append((url,note))
-
-        rejecturllist.add(rejectlist,clear=True)
+        rejecturllist.add(d.get_reject_list(),clear=True)
         
     def show_reject_reasons(self):
         d = EditTextDialog(self,
@@ -446,11 +446,11 @@ class BasicTab(QWidget):
                 
     def add_reject_urls(self):
         d = EditTextDialog(self,
-                           "http://example.com?story.php?sid=5,Reason why I rejected it",
+                           "http://example.com/story.php?sid=5,Reason why I rejected it\nhttp://example.com/story.php?sid=6,Title by Author - Reason why I rejected it",
                            icon=self.windowIcon(),
                            title="Add Reject URLs",
-                           label="Add Reject URLs. Use: <b>http://...,note</b><br>Invalid story URLs will be ignored.",
-                           tooltip="One URL per line, everything after <b>,</b> will be put in the note.",
+                           label="Add Reject URLs. Use: <b>http://...,note</b> or <b>http://...,title by author - note</b><br>Invalid story URLs will be ignored.",
+                           tooltip="One URL per line:\n<b>http://...,note</b>\n<b>http://...,title by author - note</b>",
                            rejectreasons=rejecturllist.get_reject_reasons(),
                            reasonslabel='Add this reason to all URLs added:')
         d.exec_()

calibre-plugin/dialogs.py

@@ -7,33 +7,27 @@ __license__   = 'GPL v3'
 __copyright__ = '2011, Jim Miller'
 __docformat__ = 'restructuredtext en'
 
-import traceback
+import traceback, re
 from functools import partial
 
 import urllib
 import email
 
 from PyQt4 import QtGui
-from PyQt4.Qt import (QDialog, QTableWidget, QMessageBox, QVBoxLayout, QHBoxLayout,
-                      QGridLayout, QPushButton, QProgressDialog, QString, QLabel,
-                      QCheckBox, QIcon, QTextCursor, QTextEdit, QLineEdit, QInputDialog,
-                      QComboBox, QClipboard, QVariant, QProgressDialog, QTimer,
-                      QDialogButtonBox, QPixmap, Qt, QAbstractItemView, SIGNAL,
-                      QTableWidgetItem )
+from PyQt4.Qt import (QDialog, QTableWidget, QVBoxLayout, QHBoxLayout, QGridLayout,
+                      QPushButton, QString, QLabel, QCheckBox, QIcon, QLineEdit,
+                      QComboBox, QVariant, QProgressDialog, QTimer, QDialogButtonBox,
+                      QPixmap, Qt, QAbstractItemView, SIGNAL, QTextEdit, pyqtSignal)
 
-from calibre.gui2 import error_dialog, warning_dialog, question_dialog, info_dialog
 from calibre.gui2.dialogs.confirm_delete import confirm
 from calibre.gui2.complete2 import EditWithComplete
 
-from calibre import confirm_config_name
-from calibre.gui2 import dynamic
-
-from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapters,writers,exceptions
 from calibre_plugins.fanfictiondownloader_plugin.common_utils \
     import (ReadOnlyTableWidgetItem, ReadOnlyTextIconWidgetItem, SizePersistedDialog,
             ImageTitleLayout, get_icon)
 
 from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.geturls import get_urls_from_html, get_urls_from_text
+from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.adapters import getNormalStoryURL
 
 SKIP=u'Skip'
 ADDNEW=u'Add New Book'
@@ -53,7 +47,63 @@ collision_order=[SKIP,
 anthology_collision_order=[UPDATE,
                            UPDATEALWAYS,
                            OVERWRITEALWAYS]
+
+class RejectUrlEntry:
+
+    matchpat=re.compile(r"^(?P<url>[^,]+)(,(?P<fullnote>(((?P<title>.+) by (?P<auth>.+?)( - (?P<note>.+))?)|.*)))?$")
+    
+    def __init__(self,url_or_line,note=None,title=None,auth=None,
+                 addreasontext=None,fromline=False,book_id=None):
+
+        self.url=url_or_line
+        self.note=note
+        self.title=title
+        self.auth=auth
+        self.valid=False
+        self.book_id=book_id
+
+        if fromline:
+            mc = re.match(self.matchpat,url_or_line)
+            if mc:
+                #print("mc:%s"%mc.groupdict())
+                (url,title,auth,note) = mc.group('url','title','auth','note')
+                if not mc.group('title'):
+                    title=''
+                    auth=''
+                    note=mc.group('fullnote')
+                self.url=url
+                self.note=note
+                self.title=title
+                self.auth=auth
         
+        if not self.note:
+            if addreasontext:
+                self.note = addreasontext
+            else:
+                self.note = ''
+        else:
+            if addreasontext:
+                self.note = self.note + ' - ' + addreasontext
+                
+        self.url = getNormalStoryURL(self.url)
+        self.valid = self.url != None
+                
+    def to_line(self):
+        # always 'url,'
+        return self.url+","+self.fullnote()
+        
+    def fullnote(self):
+        retval = ""
+        if self.title and self.auth:
+            retval = retval + "%s by %s"%(self.title,self.auth)
+            if self.note:
+                retval = retval + " - "
+                
+        if self.note:
+            retval = retval + self.note
+            
+        return retval
+
 # This is a more than slightly kludgey way to get
 # EditWithComplete to *not* alpha-order the reasons, but leave
 # them in the order entered.  If
@@ -81,18 +131,8 @@ class DroppableQTextEdit(QTextEdit):
 
     def dropEvent(self,event):
         # print("event:%s"%event)
-        # print("event.mimeData():%s"%event.mimeData())
-        # print("event.mimeData().text():%s"%str(event.mimeData().text()))
-        # print("event.mimeData().data():%s"%str(event.mimeData().data()))
-        # print("event.mimeData().formats():%s"%[str(f) for f in event.mimeData().formats()])
-        # for f in event.mimeData().formats():
-        #     try:
-        #         print("event.mimeData().data('%s'):%s"%(f,event.mimeData().data(f)))
-        #     except:
-        #         print("failed %s"%f)
 
         mimetype='text/uri-list'
-        # print("event.mimeData().data('%s'):%s"%(mimetype,event.mimeData().data(mimetype)))
 
         urllist=[]
         filelist="%s"%event.mimeData().data(mimetype)
@@ -133,25 +173,11 @@ class DroppableQTextEdit(QTextEdit):
                             
 class AddNewDialog(SizePersistedDialog):
 
-    def __init__(self, gui, prefs, icon, url_list_text, merge=False, newmerge=False):
-        SizePersistedDialog.__init__(self, gui, 'FanFictionDownLoader plugin:add new dialog')
-        self.gui = gui
-        self.merge = merge
-        self.newmerge = newmerge
+    go_signal = pyqtSignal(object, object, object, object)
 
-        if merge:
-            labeltext = 'Story URL(s) for anthology, one per line:'
-            tooltiptext = 'URLs for stories to include in the anthology, one per line.\nWill take URLs from clipboard, but only valid URLs.'
-            collisiontext = 'If Story Already Exists in Anthology?'
-            collisiontooltip = "What to do if there's already an existing story with the same URL in the anthology."
-        else:
-            labeltext = 'Story URL(s), one per line:'
-            tooltiptext = 'URLs for stories, one per line.\nWill take URLs from clipboard, but only valid URLs.\nAdd [1,5] after the URL to limit the download to chapters 1-5.'
-            collisiontext = 'If Story Already Exists?'
-            collisiontooltip = "What to do if there's already an existing story with the same URL or title and author."
-            
-        if prefs['adddialogstaysontop']:
-            QDialog.setWindowFlags ( self, Qt.Dialog|Qt.WindowStaysOnTopHint )
+    def __init__(self, gui, prefs, icon):
+        SizePersistedDialog.__init__(self, gui, 'FanFictionDownLoader plugin:add new dialog')
+        self.prefs = prefs
         
         self.setMinimumWidth(300)
         self.l = QVBoxLayout()
@@ -160,69 +186,158 @@ class AddNewDialog(SizePersistedDialog):
         self.setWindowTitle('FanFictionDownLoader')
         self.setWindowIcon(icon)
 
-        self.l.addWidget(QLabel(labeltext))
+        self.toplabel=QLabel("Toplabel")
+        self.l.addWidget(self.toplabel)
         self.url = DroppableQTextEdit(self)
-        self.url.setToolTip(tooltiptext)
+        self.url.setToolTip("UrlTooltip")
         self.url.setLineWrapMode(QTextEdit.NoWrap)
-        self.url.setText(url_list_text)
         self.l.addWidget(self.url)
 
-        if not merge:
-            horz = QHBoxLayout()
-            label = QLabel('Output &Format:')
-            horz.addWidget(label)
-            self.fileform = QComboBox(self)
-            self.fileform.addItem('epub')
-            self.fileform.addItem('mobi')
-            self.fileform.addItem('html')
-            self.fileform.addItem('txt')
-            self.fileform.setCurrentIndex(self.fileform.findText(prefs['fileform']))
-            self.fileform.setToolTip('Choose output format to create.  May set default from plugin configuration.')
-            self.fileform.activated.connect(self.set_collisions)
-            
-            label.setBuddy(self.fileform)
-            horz.addWidget(self.fileform)
-            self.l.addLayout(horz)
-
-        if not newmerge:
-            horz = QHBoxLayout()
-            label = QLabel(collisiontext)
-            horz.addWidget(label)
-            self.collision = QComboBox(self)
-            self.collision.setToolTip(collisiontooltip)
-            # add collision options
-            self.set_collisions()
-            i = self.collision.findText(prefs['collision'])
-            if i > -1:
-                self.collision.setCurrentIndex(i)
-            label.setBuddy(self.collision)
-            horz.addWidget(self.collision)
-            self.l.addLayout(horz)
-    
-            horz = QHBoxLayout()
-            self.updatemeta = QCheckBox('Update Calibre &Metadata?',self)
-            self.updatemeta.setToolTip("Update metadata for existing stories in Calibre from web site?\n(Columns set to 'New Only' in the column tabs will only be set for new books.)")
-            self.updatemeta.setChecked(prefs['updatemeta'])
-            horz.addWidget(self.updatemeta)
-    
-            if not merge: # hide if anthology merge.
-                self.updateepubcover = QCheckBox('Update EPUB Cover?',self)
-                self.updateepubcover.setToolTip('Update book cover image from site or defaults (if found) <i>inside</i> the EPUB when EPUB is updated.')
-                self.updateepubcover.setChecked(prefs['updateepubcover'])
-                horz.addWidget(self.updateepubcover)
-            
-            self.l.addLayout(horz)
-
-        button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
-        button_box.accepted.connect(self.accept)
-        button_box.rejected.connect(self.reject)
-        self.l.addWidget(button_box)
+        self.merge = self.newmerge = False
         
-        if url_list_text:
-            button_box.button(QDialogButtonBox.Ok).setFocus()
+        # elements to hide when doing merge.
+        self.mergehide = []
+        # elements to show again when doing *update* merge
+        self.mergeupdateshow = []
 
+        horz = QHBoxLayout()
+        label = QLabel('Output &Format:')
+        self.mergehide.append(label)
+        
+        self.fileform = QComboBox(self)
+        self.fileform.addItem('epub')
+        self.fileform.addItem('mobi')
+        self.fileform.addItem('html')
+        self.fileform.addItem('txt')
+        self.fileform.setToolTip('Choose output format to create.  May set default from plugin configuration.')
+        self.fileform.activated.connect(self.set_collisions)
+        
+        horz.addWidget(label)
+        label.setBuddy(self.fileform)
+        horz.addWidget(self.fileform)
+        self.l.addLayout(horz)
+        self.mergehide.append(self.fileform)
+
+        horz = QHBoxLayout()
+        self.collisionlabel = QLabel("CollisionLabel")
+        horz.addWidget(self.collisionlabel)
+        self.collision = QComboBox(self)
+        self.collision.setToolTip("CollisionToolTip")
+        # add collision options
+        self.set_collisions()
+        i = self.collision.findText(prefs['collision'])
+        if i > -1:
+            self.collision.setCurrentIndex(i)
+        self.collisionlabel.setBuddy(self.collision)
+        horz.addWidget(self.collision)
+        self.l.addLayout(horz)
+        self.mergehide.append(self.collisionlabel)
+        self.mergehide.append(self.collision)
+        self.mergeupdateshow.append(self.collisionlabel)
+        self.mergeupdateshow.append(self.collision)
+
+        horz = QHBoxLayout()
+        self.updatemeta = QCheckBox('Update Calibre &Metadata?',self)
+        self.updatemeta.setToolTip("Update metadata for existing stories in Calibre from web site?\n(Columns set to 'New Only' in the column tabs will only be set for new books.)")
+        self.updatemeta.setChecked(prefs['updatemeta'])
+        horz.addWidget(self.updatemeta)
+        self.mergehide.append(self.updatemeta)
+        self.mergeupdateshow.append(self.updatemeta)
+
+        self.updateepubcover = QCheckBox('Update EPUB Cover?',self)
+        self.updateepubcover.setToolTip('Update book cover image from site or defaults (if found) <i>inside</i> the EPUB when EPUB is updated.')
+        self.updateepubcover.setChecked(prefs['updateepubcover'])
+        horz.addWidget(self.updateepubcover)
+        self.mergehide.append(self.updateepubcover)
+        
+        self.l.addLayout(horz)
+
+        self.button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
+        self.button_box.accepted.connect(self.ok_clicked)
+        self.button_box.rejected.connect(self.reject)
+        self.l.addWidget(self.button_box)
+
+    # invoke the 
+    def ok_clicked(self):
+        self.dialog_closing(None) # save persistent size.
+        self.hide()
+        self.go_signal.emit( self.get_ffdl_options(),
+                             self.get_urlstext(),
+                             self.merge,
+                             self.extrapayload )
+
+    def show_dialog(self,
+                    url_list_text,
+                    callback,
+                    show=True,
+                    merge=False,
+                    newmerge=True,
+                    extraoptions={},
+                    extrapayload=None):
+        # rather than mutex in ffdl_plugin, just bail here if it's
+        # already in use.
+        if self.isVisible(): return
+
+        try:
+            self.go_signal.disconnect()
+        except:
+            pass # if not already connected.
+        self.go_signal.connect(callback)
+
+        self.merge = merge
+        self.newmerge = newmerge
+        self.extraoptions = extraoptions
+        self.extrapayload = extrapayload
+        
+        if self.merge:
+            self.toplabel.setText('Story URL(s) for anthology, one per line:')
+            self.url.setToolTip('URLs for stories to include in the anthology, one per line.\nWill take URLs from clipboard, but only valid URLs.')
+            self.collisionlabel.setText('If Story Already Exists in Anthology?')
+            self.collision.setToolTip("What to do if there's already an existing story with the same URL in the anthology.")
+            for widget in self.mergehide:
+                widget.setVisible(False)
+            if not self.newmerge:
+                for widget in self.mergeupdateshow:
+                    widget.setVisible(True)
+        else:
+            for widget in self.mergehide:
+                widget.setVisible(True)
+            self.toplabel.setText('Story URL(s), one per line:')
+            self.url.setToolTip('URLs for stories, one per line.\nWill take URLs from clipboard, but only valid URLs.\nAdd [1,5] after the URL to limit the download to chapters 1-5.')
+            self.collisionlabel.setText('If Story Already Exists?')
+            self.collision.setToolTip("What to do if there's already an existing story with the same URL or title and author.")
+
+        # Need to re-able after hiding/showing
+        self.setAcceptDrops(True)
+        self.url.setFocus()
+            
+        if self.prefs['adddialogstaysontop']:
+            QDialog.setWindowFlags ( self, Qt.Dialog | Qt.WindowStaysOnTopHint )
+        else:
+            QDialog.setWindowFlags ( self, Qt.Dialog )
+
+        if not self.merge:
+            self.fileform.setCurrentIndex(self.fileform.findText(self.prefs['fileform']))
+
+        # add collision options
+        self.set_collisions()
+        
+        i = self.collision.findText(self.prefs['collision'])
+        if i > -1:
+            self.collision.setCurrentIndex(i)
+        self.updatemeta.setChecked(self.prefs['updatemeta'])
+            
+        if not self.merge:
+            self.updateepubcover.setChecked(self.prefs['updateepubcover'])
+                
+        self.url.setText(url_list_text)
+        if url_list_text:
+            self.button_box.button(QDialogButtonBox.Ok).setFocus()
         # restore saved size.
         self.resize_dialog()
+        
+        if show: # so anthology update can be modal still.
+            self.show()
         #self.resize(self.sizeHint())
 
     def set_collisions(self):
@@ -240,27 +355,21 @@ class AddNewDialog(SizePersistedDialog):
             self.collision.setCurrentIndex(i)
         
     def get_ffdl_options(self):
+        retval =  {
+            'fileform': unicode(self.fileform.currentText()),
+            'collision': unicode(self.collision.currentText()),
+            'updatemeta': self.updatemeta.isChecked(),
+            'updateepubcover': self.updateepubcover.isChecked(),
+                }
+        
         if self.merge:
+            retval['fileform']=='epub'
+            retval['updateepubcover']=True
             if self.newmerge:
-                updatemeta=True
-                collision=ADDNEW
-            else:
-                updatemeta=self.updatemeta.isChecked()
-                collision=unicode(self.collision.currentText())
-                
-            return {
-                'fileform': 'epub',
-                'collision': collision,
-                'updatemeta': updatemeta,
-                'updateepubcover': True,
-                }
-        else:
-            return {
-                'fileform': unicode(self.fileform.currentText()),
-                'collision': unicode(self.collision.currentText()),
-                'updatemeta': self.updatemeta.isChecked(),
-                'updateepubcover': self.updateepubcover.isChecked(),
-                }
+                retval['updatemeta']=True
+                retval['collision']=ADDNEW
+            
+        return dict(retval.items() + self.extraoptions.items() )
 
     def get_urlstext(self):
         return unicode(self.url.toPlainText())
@@ -279,7 +388,6 @@ class CollectURLDialog(SizePersistedDialog):
     '''
     def __init__(self, gui, title, url_text, epubmerge_plugin=None): 
         SizePersistedDialog.__init__(self, gui, 'FanFictionDownLoader plugin:get story urls')
-        self.gui = gui
         self.status=False
         self.anthology=False
 
@@ -333,7 +441,6 @@ class UserPassDialog(QDialog):
     '''
     def __init__(self, gui, site, exception=None):
         QDialog.__init__(self, gui)
-        self.gui = gui
         self.status=False
 
         self.l = QGridLayout()
@@ -392,7 +499,6 @@ class LoopProgressDialog(QProgressDialog):
                                  QString(), 0, len(book_list), gui)
         self.setWindowTitle(win_title)
         self.setMinimumWidth(500)
-        self.gui = gui
         self.book_list = book_list
         self.foreach_function = foreach_function
         self.finish_function = finish_function
@@ -441,7 +547,6 @@ class LoopProgressDialog(QProgressDialog):
 
     def do_when_finished(self):
         self.hide()
-        self.gui = None        
         # Queues a job to process these books in the background.
         self.finish_function(self.book_list)
 
@@ -490,7 +595,6 @@ class UpdateExistingDialog(SizePersistedDialog):
     def __init__(self, gui, header, prefs, icon, books,
                  save_size_name='fanfictiondownloader_plugin:update list dialog'):
         SizePersistedDialog.__init__(self, gui, save_size_name)
-        self.gui = gui
         
         self.setWindowTitle(header)
         self.setWindowIcon(icon)
@@ -508,11 +612,7 @@ class UpdateExistingDialog(SizePersistedDialog):
 
         button_layout = QVBoxLayout()
         books_layout.addLayout(button_layout)
-        # self.move_up_button = QtGui.QToolButton(self)
-        # self.move_up_button.setToolTip('Move selected books up the list')
-        # self.move_up_button.setIcon(QIcon(I('arrow-up.png')))
-        # self.move_up_button.clicked.connect(self.books_table.move_rows_up)
-        # button_layout.addWidget(self.move_up_button)
+
         spacerItem = QtGui.QSpacerItem(20, 40, QtGui.QSizePolicy.Minimum, QtGui.QSizePolicy.Expanding)
         button_layout.addItem(spacerItem)
         self.remove_button = QtGui.QToolButton(self)
@@ -522,11 +622,6 @@ class UpdateExistingDialog(SizePersistedDialog):
         button_layout.addWidget(self.remove_button)
         spacerItem1 = QtGui.QSpacerItem(20, 40, QtGui.QSizePolicy.Minimum, QtGui.QSizePolicy.Expanding)
         button_layout.addItem(spacerItem1)
-        # self.move_down_button = QtGui.QToolButton(self)
-        # self.move_down_button.setToolTip('Move selected books down the list')
-        # self.move_down_button.setIcon(QIcon(I('arrow-down.png')))
-        # self.move_down_button.clicked.connect(self.books_table.move_rows_down)
-        # button_layout.addWidget(self.move_down_button)
 
         options_layout = QHBoxLayout()
 
@@ -552,7 +647,6 @@ class UpdateExistingDialog(SizePersistedDialog):
         i = self.collision.findText(prefs['collision'])
         if i > -1:
             self.collision.setCurrentIndex(i)
-        # self.collision.setToolTip('Overwrite will replace the existing story.  Add New will create a new story with the same title and author.')
         label.setBuddy(self.collision)
         options_layout.addWidget(self.collision)
 
@@ -661,11 +755,9 @@ class StoryListTableWidget(QTableWidget):
         self.setItem(row, 2, AuthorTableWidgetItem(", ".join(book['author']), ", ".join(book['author_sort'])))
         
         url_cell = ReadOnlyTableWidgetItem(book['url'])
-        #url_cell.setData(Qt.UserRole, QVariant(book['url']))
         self.setItem(row, 3, url_cell)
         
         comment_cell = ReadOnlyTableWidgetItem(book['comment'])
-        #comment_cell.setData(Qt.UserRole, QVariant(book))
         self.setItem(row, 4, comment_cell)
 
     def get_books(self):
@@ -699,56 +791,6 @@ class StoryListTableWidget(QTableWidget):
         self.selectRow(row)
         self.scrollToItem(self.currentItem())
 
-    def move_rows_up(self):
-        self.setFocus()
-        rows = self.selectionModel().selectedRows()
-        if len(rows) == 0:
-            return
-        first_sel_row = rows[0].row()
-        if first_sel_row <= 0:
-            return
-        # Workaround for strange selection bug in Qt which "alters" the selection
-        # in certain circumstances which meant move down only worked properly "once"
-        selrows = []
-        for row in rows:
-            selrows.append(row.row())
-        selrows.sort()
-        for selrow in selrows:
-            self.swap_row_widgets(selrow - 1, selrow + 1)
-        scroll_to_row = first_sel_row - 1
-        if scroll_to_row > 0:
-            scroll_to_row = scroll_to_row - 1
-        self.scrollToItem(self.item(scroll_to_row, 0))
-
-    def move_rows_down(self):
-        self.setFocus()
-        rows = self.selectionModel().selectedRows()
-        if len(rows) == 0:
-            return
-        last_sel_row = rows[-1].row()
-        if last_sel_row == self.rowCount() - 1:
-            return
-        # Workaround for strange selection bug in Qt which "alters" the selection
-        # in certain circumstances which meant move down only worked properly "once"
-        selrows = []
-        for row in rows:
-            selrows.append(row.row())
-        selrows.sort()
-        for selrow in reversed(selrows):
-            self.swap_row_widgets(selrow + 2, selrow)
-        scroll_to_row = last_sel_row + 1
-        if scroll_to_row < self.rowCount() - 1:
-            scroll_to_row = scroll_to_row + 1
-        self.scrollToItem(self.item(scroll_to_row, 0))
-
-    def swap_row_widgets(self, src_row, dest_row):
-        self.blockSignals(True)
-        self.insertRow(dest_row)
-        for col in range(0, self.columnCount()):
-            self.setItem(dest_row, col, self.takeItem(src_row, col))
-        self.removeRow(src_row)
-        self.blockSignals(False)
-
 class RejectListTableWidget(QTableWidget):
 
     def __init__(self, parent,rejectreasons=[]):
@@ -756,85 +798,55 @@ class RejectListTableWidget(QTableWidget):
         self.setSelectionBehavior(QAbstractItemView.SelectRows)
         self.rejectreasons = rejectreasons
 
-    def on_headersection_clicked(self):
-        self.setSortingEnabled(True)
-        
     def populate_table(self, reject_list):
         self.clear()
         self.setAlternatingRowColors(True)
         self.setRowCount(len(reject_list))
-        header_labels = ['URL', 'Note']
+        header_labels = ['URL', 'Title', 'Author', 'Note']
         self.setColumnCount(len(header_labels))
         self.setHorizontalHeaderLabels(header_labels)
         self.horizontalHeader().setStretchLastSection(True)
         #self.verticalHeader().setDefaultSectionSize(24)
         self.verticalHeader().hide()
 
-        # need sortingEnbled to sort, but off to up & down.
-        self.connect(self.horizontalHeader(),
-                     SIGNAL('sectionClicked(int)'),
-                     self.on_headersection_clicked)
-
+        # it's generally recommended to enable sort after pop, not
+        # before.  But then it needs to be sorted on a column and I'd
+        # rather keep the order given.
+        self.setSortingEnabled(True)
         # row is just row number.
         for row, rejectrow in enumerate(reject_list):
+            #print("populating table:%s"%rejectrow.to_line())
             self.populate_table_row(row,rejectrow)
 
         self.resizeColumnsToContents()
-        self.setMinimumColumnWidth(1, 100)
-        self.setMinimumColumnWidth(2, 100)
+        self.setMinimumColumnWidth(0, 100)
+        self.setMinimumColumnWidth(3, 100)
         self.setMinimumSize(300, 0)
 
     def setMinimumColumnWidth(self, col, minimum):
         if self.columnWidth(col) < minimum:
             self.setColumnWidth(col, minimum)
 
-    def populate_table_row(self, row, rejectrow):
+    def populate_table_row(self, row, rej):
 
-        (bookid,url,titleauth,oldrejnote) = rejectrow
-        if oldrejnote:
-            noteprefix = note = oldrejnote
-            # incase the existing note ends with one of the known reasons.
-            for reason in self.rejectreasons:
-                if noteprefix.endswith(' - '+reason):
-                    noteprefix = noteprefix[:-len(' - '+reason)]
-                    break
-        else:
-            noteprefix = note = titleauth
-            
-        if len(noteprefix) > 0:
-            noteprefix = noteprefix+' - '
-            
-        url_cell = ReadOnlyTableWidgetItem(url)
-        url_cell.setData(Qt.UserRole, QVariant(bookid))
-        url_cell.setToolTip('URL to add to the Reject List.')
+        url_cell = ReadOnlyTableWidgetItem(rej.url)
+        url_cell.setData(Qt.UserRole, QVariant(rej.book_id))
         self.setItem(row, 0, url_cell)
-
+        self.setItem(row, 1, ReadOnlyTableWidgetItem(rej.title))
+        self.setItem(row, 2, ReadOnlyTableWidgetItem(rej.auth))
+        
         note_cell = EditWithComplete(self)
-
         note_cell.lineEdit().mcompleter.model().set_items = \
             partial(complete_model_set_items_kludge,
                     note_cell.lineEdit().mcompleter.model())
         
-        items = [note]+[ noteprefix+x for x in self.rejectreasons ]
+        items = [rej.note]+self.rejectreasons
         note_cell.update_items_cache(items)
-        note_cell.show_initial_value(note)
+        note_cell.show_initial_value(rej.note)
         note_cell.set_separator(None)
         note_cell.setToolTip('Select or Edit Reject Note.')
-        self.setCellWidget(row, 1, note_cell)
+        self.setCellWidget(row, 3, note_cell)
         
-        # note_cell = QTableWidgetItem(note)
-        # note_cell.setToolTip('Double-click to edit note.')
-        # self.setItem(row, 1, note_cell)
-      
-    def get_reject_list(self):
-        rejectrows = []
-        for row in range(self.rowCount()):
-            bookid = self.item(row, 0).data(Qt.UserRole).toPyObject()
-            url = unicode(self.item(row, 0).text())
-            note = unicode(self.cellWidget(row, 1).currentText()).strip()
-            rejectrows.append((bookid,url,note))
-        return rejectrows
-
     def remove_selected_rows(self):
         self.setFocus()
         rows = self.selectionModel().selectedRows()
@@ -857,57 +869,6 @@ class RejectListTableWidget(QTableWidget):
         self.selectRow(row)
         self.scrollToItem(self.currentItem())
 
-    def move_rows_up(self):
-        self.setFocus()
-        rows = self.selectionModel().selectedRows()
-        if len(rows) == 0:
-            return
-        first_sel_row = rows[0].row()
-        if first_sel_row <= 0:
-            return
-        # Workaround for strange selection bug in Qt which "alters" the selection
-        # in certain circumstances which meant move down only worked properly "once"
-        selrows = []
-        for row in rows:
-            selrows.append(row.row())
-        selrows.sort()
-        for selrow in selrows:
-            self.swap_row_widgets(selrow - 1, selrow + 1)
-        scroll_to_row = first_sel_row - 1
-        if scroll_to_row > 0:
-            scroll_to_row = scroll_to_row - 1
-        self.scrollToItem(self.item(scroll_to_row, 0))
-
-    def move_rows_down(self):
-        self.setFocus()
-        rows = self.selectionModel().selectedRows()
-        if len(rows) == 0:
-            return
-        last_sel_row = rows[-1].row()
-        if last_sel_row == self.rowCount() - 1:
-            return
-        # Workaround for strange selection bug in Qt which "alters" the selection
-        # in certain circumstances which meant move down only worked properly "once"
-        selrows = []
-        for row in rows:
-            selrows.append(row.row())
-        selrows.sort()
-        for selrow in reversed(selrows):
-            self.swap_row_widgets(selrow + 2, selrow)
-        scroll_to_row = last_sel_row + 1
-        if scroll_to_row < self.rowCount() - 1:
-            scroll_to_row = scroll_to_row + 1
-        self.scrollToItem(self.item(scroll_to_row, 0))
-
-    def swap_row_widgets(self, src_row, dest_row):
-        self.blockSignals(True)
-        self.setSortingEnabled(False)
-        self.insertRow(dest_row)
-        for col in range(0, self.columnCount()):
-            self.setItem(dest_row, col, self.takeItem(src_row, col))
-        self.removeRow(src_row)
-        self.blockSignals(False)
-
 class RejectListDialog(SizePersistedDialog):
     def __init__(self, gui, reject_list,
                  rejectreasons=[],
@@ -917,7 +878,6 @@ class RejectListDialog(SizePersistedDialog):
                  show_all_reasons=True,
                  save_size_name='ffdl:reject list dialog'):
         SizePersistedDialog.__init__(self, gui, save_size_name)
-        self.gui = gui
       
         self.setWindowTitle(header)
         self.setWindowIcon(get_icon(icon))
@@ -937,21 +897,13 @@ class RejectListDialog(SizePersistedDialog):
         rejects_layout.addLayout(button_layout)
         spacerItem = QtGui.QSpacerItem(20, 40, QtGui.QSizePolicy.Minimum, QtGui.QSizePolicy.Expanding)
         button_layout.addItem(spacerItem)
-        # self.move_up_button = QtGui.QToolButton(self)
-        # self.move_up_button.setToolTip('Move selected books up the list')
-        # self.move_up_button.setIcon(QIcon(I('arrow-up.png')))
-        # self.move_up_button.clicked.connect(self.books_table.move_rows_up)
-        # button_layout.addWidget(self.move_up_button)
+        
         self.remove_button = QtGui.QToolButton(self)
         self.remove_button.setToolTip('Remove selected URL(s) from the list')
         self.remove_button.setIcon(get_icon('list_remove.png'))
         self.remove_button.clicked.connect(self.remove_from_list)
         button_layout.addWidget(self.remove_button)
-        # self.move_down_button = QtGui.QToolButton(self)
-        # self.move_down_button.setToolTip('Move selected books down the list')
-        # self.move_down_button.setIcon(QIcon(I('arrow-down.png')))
-        # self.move_down_button.clicked.connect(self.books_table.move_rows_down)
-        # button_layout.addWidget(self.move_down_button)
+
         spacerItem1 = QtGui.QSpacerItem(20, 40, QtGui.QSizePolicy.Minimum, QtGui.QSizePolicy.Expanding)
         button_layout.addItem(spacerItem1)
 
@@ -998,10 +950,30 @@ class RejectListDialog(SizePersistedDialog):
         self.rejects_table.remove_selected_rows()
 
     def get_reject_list(self):
-        return self.rejects_table.get_reject_list()
+        rejectrows = []
+        for row in range(self.rejects_table.rowCount()):
+            url = unicode(self.rejects_table.item(row, 0).text()).strip()
+            book_id = self.rejects_table.item(row, 0).data(Qt.UserRole).toPyObject()
+            title = unicode(self.rejects_table.item(row, 1).text()).strip()
+            auth = unicode(self.rejects_table.item(row, 2).text()).strip()
+            note = unicode(self.rejects_table.cellWidget(row, 3).currentText()).strip()
+            rejectrows.append(RejectUrlEntry(url,note,title,auth,self.get_reason_text(),book_id=book_id))
+        return rejectrows
+
+    def get_reject_list_ids(self):
+        rejectrows = []
+        for row in range(self.rejects_table.rowCount()):
+            book_id = self.rejects_table.item(row, 0).data(Qt.UserRole).toPyObject()
+            if book_id:
+                rejectrows.append(book_id)
+        return rejectrows
 
     def get_reason_text(self):
-        return unicode(self.reason_edit.currentText()).strip()
+        try:
+            return unicode(self.reason_edit.currentText()).strip()
+        except:
+            # doesn't have self.reason_edit when editing existing list.
+            return None
     
     def get_deletebooks(self):
         return self.deletebooks.isChecked()

calibre-plugin/ffdl_plugin.py

@@ -47,7 +47,7 @@ from calibre_plugins.fanfictiondownloader_plugin.dialogs import (
     AddNewDialog, UpdateExistingDialog,
     LoopProgressDialog, UserPassDialog, AboutDialog, CollectURLDialog, RejectListDialog,
     OVERWRITE, OVERWRITEALWAYS, UPDATE, UPDATEALWAYS, ADDNEW, SKIP, CALIBREONLY,
-    NotGoingToDownload )
+    NotGoingToDownload, RejectUrlEntry )
 
 # because calibre immediately transforms html into zip and don't want
 # to have an 'if html'.  db.has_format is cool with the case mismatch,
@@ -102,7 +102,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         # are not found in the zip file will result in null QIcons.
         icon = get_icon('images/icon.png')
 
-        #self.qaction.setText('FFDL')
+        self.qaction.setText('FanFictionDL')
         
         # The qaction is automatically created from the action_spec defined
         # above
@@ -126,6 +126,10 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         # been displayed once.
         self.rebuild_menus()
 
+        self.add_new_dialog = AddNewDialog(self.gui,
+                                           prefs,
+                                           self.qaction.icon())
+
     ## Kludgey, yes, but with the real configuration inside the
     ## library now, how else would a user be able to change this
     ## setting if it's crashing calibre?
@@ -136,7 +140,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
             file_path = os.path.join(calibre_config_dir,
                                      *("plugins/fanfictiondownloader_macmenuhack.txt".split('/')))
             file_path = os.path.abspath(file_path)
-            print("macmenuhack file_path:%s"%file_path)
+            print("Plugin %s macmenuhack file_path:%s"%(self.name,file_path))
             self.macmenuhack = os.access(file_path, os.F_OK)
             return self.macmenuhack
 
@@ -399,17 +403,17 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
             # want title/author, too, for rejects.
             self.populate_book_from_calibre_id(book,db)
         if book['url']:
-            # get existing note, if there is one.
-            book['oldrejnote']=rejecturllist.check(book['url'])
+            # get existing note, if on rejected list.
+            book['oldrejnote']=rejecturllist.get_note(book['url'])
             
     def reject_list_urls_finish(self, book_list):
 
-        # construct reject list of tuples:
-        # (calibre_id, url, "title, authors", old reject note).
-        reject_list = [ ( x['calibre_id'],x['url'],
-                          "%s by %s"%(x['title'],
-                                      ', '.join(x['author'])),
-                          x['oldrejnote'])
+        # construct reject list of objects
+        reject_list = [ RejectUrlEntry(x['url'],
+                                       x['oldrejnote'],
+                                       x['title'],
+                                       ', '.join(x['author']),
+                                       book_id=x['calibre_id'])
                         for x in book_list if x['good'] ]
         if reject_list:
             d = RejectListDialog(self.gui,reject_list,
@@ -419,62 +423,31 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
             if d.result() != d.Accepted:
                 return
 
-            bookids=[]
-            rejectlist=[]
-            addreasontext=d.get_reason_text()
-            for (bookid,url,note) in d.get_reject_list():
-                bookids.append(bookid)
-                if addreasontext and note:
-                    note = note +" - "+addreasontext
-                elif addreasontext:
-                    note = addreasontext
-                rejectlist.append((url,note))
-                print("Adding (%s) to Reject List: %s"%(url,note))
-                
-            rejecturllist.add(rejectlist)
+            rejecturllist.add(d.get_reject_list())
 
             if d.get_deletebooks():
-                self.gui.iactions['Remove Books'].delete_books()
+                self.gui.iactions['Remove Books'].do_library_delete(d.get_reject_list_ids())
             
         else:
             message="<p>Rejecting FFDL URLs: None of the books selected have FanFiction URLs.</p><p>Proceed to Remove?</p>"
             if confirm(message,'fanfictiondownloader_reject_non_fanfiction', self.gui):
                 self.gui.iactions['Remove Books'].delete_books()
-                
-    def add_dialog(self,url_list_text=None,merge=False,anthology_url=None):
 
-        #print("add_dialog()")
+    def add_dialog(self,url_list_text=None,merge=False,anthology_url=None):
+        'Both new individual stories and new anthologies are created here.'
 
         if not url_list_text:
             url_list = self.get_urls_clip()
             url_list_text = "\n".join(url_list)
         
-        # self.gui is the main calibre GUI. It acts as the gateway to access
-        # all the elements of the calibre user interface, it should also be the
-        # parent of the dialog
-        # AddNewDialog just collects URLs, format and presents buttons.
-        d = AddNewDialog(self.gui,
-                         prefs,
-                         self.qaction.icon(),
-                         url_list_text,
-                         merge=merge,
-                         newmerge=merge # if here, it's a new anthology.
-                         )
-        d.exec_()
-        if d.result() != d.Accepted:
-            return
-        
-        url_list = split_text_to_urls(d.get_urlstext())
-        add_books = self.convert_urls_to_books(url_list)
-        #print("add_books:%s"%add_books)
-        #print("options:%s"%d.get_ffdl_options())
-
-        options = d.get_ffdl_options()
-        options['version'] = self.version
-        options['anthology_url']=anthology_url
-        print(self.version)
-        
-        self.prep_downloads( options, add_books, merge=merge )
+        # AddNewDialog collects URLs, format and presents buttons.
+        # add_new_dialog is modeless and reused, both for new stories
+        # and anthologies, and for updating existing anthologies.
+        self.add_new_dialog.show_dialog(url_list_text,
+                                        self.prep_downloads,
+                                        merge=merge,
+                                        newmerge=True,
+                                        extraoptions={'anthology_url':anthology_url})
 
     def update_anthology(self):
         if not self.get_epubmerge_plugin():
@@ -488,7 +461,6 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         if len(self.gui.library_view.get_selected_ids()) != 1:
             self.gui.status_bar.show_message(_('Can only update 1 anthology at a time'), 3000)
             return
-        #print("update_existing()")        
         
         db = self.gui.current_db
         book_id = self.gui.library_view.get_selected_ids()[0]
@@ -530,25 +502,30 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         url_list_text = "\n".join(url_list)
         
         #print("urlmapfile:%s"%urlmapfile)
+        
+        # AddNewDialog collects URLs, format and presents buttons.
+        # add_new_dialog is modeless and reused, both for new stories
+        # and anthologies, and for updating existing anthologies.
+        self.add_new_dialog.show_dialog(url_list_text,
+                                        self.prep_anthology_downloads,
+                                        show=False,
+                                        merge=True,
+                                        newmerge=False,
+                                        extrapayload=urlmapfile,
+                                        extraoptions={'tdir':tdir,
+                                                      'mergebook':mergebook})
+        # Need to use AddNewDialog modal here because it's an update
+        # of an existing book.  Don't want the user deleting it or
+        # switching libraries on us.
+        self.add_new_dialog.exec_()
 
-        # self.gui is the main calibre GUI. It acts as the gateway to access
-        # all the elements of the calibre user interface, it should also be the
-        # parent of the dialog
-        # AddNewDialog just collects URLs, format and presents buttons.
-        d = AddNewDialog(self.gui,
-                         prefs,
-                         self.qaction.icon(),
-                         url_list_text,
-                         merge=True,
-                         newmerge=False
-                         )
-        d.exec_()
-        if d.result() != d.Accepted:
-            return
 
-        url_list = split_text_to_urls(d.get_urlstext())
-            
-        update_books = self.convert_urls_to_books(url_list)
+    def prep_anthology_downloads(self, options, update_books,
+                                 merge=False, urlmapfile=None):
+        
+        if isinstance(update_books,basestring):
+            url_list = split_text_to_urls(update_books)
+            update_books = self.convert_urls_to_books(url_list)
         
         for j, book in enumerate(update_books):
             url = book['url']
@@ -574,13 +551,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                 print("Canceling anthology update due to removed stories.")
                 return
 
-        options = d.get_ffdl_options()
-        options['version'] = self.version
-        options['tdir'] = tdir
-        #options['collision'] = UPDATEALWAYS
-        print(self.version)
-
-        options['mergebook'] = mergebook
+        # Now that we've 
         self.prep_downloads( options, update_books, merge=True )
         
     def update_dialog(self):
@@ -631,8 +602,6 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         # only if there's some good ones.
         if 0 < len(filter(lambda x : x['good'], update_books)):
             options = d.get_ffdl_options()
-            options['version'] = self.version
-            print(self.version)
             self.prep_downloads( options, update_books )
             
     def get_urls_clip(self,storyurls=True):
@@ -648,9 +617,16 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         # No need to do anything with perfs here, but we could.
         prefs
 
-    def prep_downloads(self, options, books, merge=False):
+    def prep_downloads(self, options, books, merge=False, extrapayload=None):
         '''Fetch metadata for stories from servers, launch BG job when done.'''
 
+        if isinstance(books,basestring):
+            url_list = split_text_to_urls(books)
+            books = self.convert_urls_to_books(url_list)
+
+        options['version'] = self.version
+        print(self.version)
+        
         #print("prep_downloads:%s"%books)
 
         if 'tdir' not in options: # if merging an anthology, there's alread a tdir.
@@ -685,14 +661,16 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
 
         url = book['url']
         print("url:%s"%url)
+        mi = None
 
         if not merge: # skip reject list when merging.
-            rejnote = rejecturllist.check(url)
-            if rejnote:
+            if rejecturllist.check(url):
+                rejnote = rejecturllist.get_full_note(url)
                 if question_dialog(self.gui, 'Reject URL?',
-                                   '<p>Reject URL?</p>'+
-                                   '<p>%s is on the Reject URL list:<br />"%s"</p>'%(url,rejnote)+
-                                   "<p>Click 'No' to download anyway.</p>",
+                                   '<h3>Reject URL?</h3>'+
+                                   '<p><b>%s</b> is on your Reject URL list:</p><p>"<b>%s</b>"</p>'%(url,rejnote)+
+                                   "<p>Click '<b>Yes</b>' to Reject.</p>"+
+                                   "<p>Click '<b>No</b>' to download anyway.</p>",
                                    show_copy_button=False):
                     book['comment'] = "Story on Reject URLs list (%s)."%rejnote
                     book['good']=False
@@ -701,9 +679,10 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                     return
                 else:
                     if question_dialog(self.gui, 'Remove Reject URL?',
-                                       "<p>Remove URL from Reject List?</p>"+
-                                       '<p>%s is on the Reject URL list:<br />"%s"</p>'%(url,rejnote)+
-                                       "<p>Click 'Yes' to remove it from the list and download,<br /> 'No' to download, but leave it on the Reject list.</p>",
+                                       "<h3>Remove URL from Reject List?</h3>"+
+                                       '<p><b>%s</b> is on your Reject URL list:</p><p>"<b>%s</b>"</p>'%(url,rejnote)+
+                                       "<p>Click '<b>Yes</b>' to remove it from the list,</p>"+
+                                       "<p>Click '<b>No</b>' to leave it on the list.</p>",
                                        show_copy_button=False):
                         rejecturllist.remove(url)
             
@@ -753,6 +732,31 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         # let other exceptions percolate up.
         story = adapter.getStoryMetadataOnly()
 
+        series = story.getMetadata('series')
+        if not merge and series and prefs['checkforseriesurlid']:
+            # try to find *series anthology* by *seriesUrl* identifier url or uri first.
+            searchstr = 'identifiers:"~ur(i|l):=%s"'%story.getMetadata('seriesUrl').replace(":","|")
+            identicalbooks = db.search_getting_ids(searchstr, None)
+            # print("searchstr:%s"%searchstr)
+            # print("identicalbooks:%s"%identicalbooks)
+            if len(identicalbooks) > 0 and question_dialog(self.gui, 'Skip Story?',
+                                                           '<h3>Skip Anthology Story?</h3>'+
+                                                           '<p>"<b>%s</b>" is in series "<b><a href="%s">%s</a></b>" that you have an anthology book for.</p>'%
+                                                           (story.getMetadata('title'),story.getMetadata('seriesUrl'),series[:series.index(' [')])+
+                                                           "<p>Click '<b>Yes</b>' to Skip.</p>"+
+                                                           "<p>Click '<b>No</b>' to download anyway.</p>",
+                                                           show_copy_button=False):
+                book['comment'] = "Story in Series Anthology(%s)."%series
+                book['title'] = story.getMetadata('title')
+                book['author'] = [story.getMetadata('author')]
+                book['good']=False
+                book['icon']='rotate-right.png'
+                book['status'] = 'Skipped'
+                return
+                
+
+        ################################################################################################################################################33
+
         # set PI version instead of default.
         if 'version' in options:
             story.setMetadata('version',options['version'])
@@ -842,6 +846,28 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                     book['icon'] = 'edit-redo.png'
                     book['status'] = 'Update'
     
+                if book_id and mi: # book_id and mi only set if matched by title/author.
+                    liburl = self.get_story_url(db,book_id)
+                    if book['url'] != liburl:
+                        if collision in (OVERWRITE,OVERWRITEALWAYS):
+                            updat="overwrit"
+                        else:
+                            updat="updat"
+                        if not question_dialog(self.gui, 'Change Story URL?',
+                                               '<h3>Change Story URL?</h3>'+
+                                               '<p><b>%s</b> by <b>%s</b> is already in your library with a different source URL:</p>'%
+                                                   (mi.title,', '.join(mi.author))+
+                                               '<p>In library: <a href="%(liburl)s">%(liburl)s</a></p><p>New URL: <a href="%(newurl)s">%(newurl)s</a></p>'%
+                                                   {'liburl':liburl,'newurl':book['url']}+
+                                               "<p>Click '<b>Yes</b>' to %se book with new URL.</p>"%updat+
+                                               "<p>Click '<b>No</b>' to skip %sing this book.</p>"%updat,
+                                               show_copy_button=False):
+                            book['comment'] = "Update declined by user due to differing story URL(%s)"%liburl
+                            book['good']=False
+                            book['icon']='rotate-right.png'
+                            book['status'] = 'Different URL'
+                            return
+        
             if book_id != None and collision != ADDNEW:
                 if collision in (CALIBREONLY):
                     book['comment'] = 'Metadata collected.'
@@ -886,7 +912,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                     db.copy_format_to(book_id,fileform,tmp,index_is_id=True)
                     print("existing epub tmp:"+tmp.name)
                     book['epub_for_update'] = tmp.name
-    
+
             if book_id != None and prefs['injectseries']:
                 mi = db.get_metadata(book_id,index_is_id=True)
                 if not book['series'] and mi.series != None:
@@ -1156,7 +1182,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         if 'mergebook' in options:
             existingbook = options['mergebook']
         #print("existingbook:\n%s"%existingbook)
-        mergebook = self.merge_meta_books(existingbook,good_list)
+        mergebook = self.merge_meta_books(existingbook,good_list,options['fileform'])
 
         if 'mergebook' in options:
             mergebook['calibre_id'] = options['mergebook']['calibre_id']
@@ -1278,7 +1304,10 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
             existingepub = db.format(book_id,'EPUB',index_is_id=True, as_file=True)
             epubmi = get_metadata(existingepub,'EPUB')
             if epubmi.cover_data[1] is not None:
-                db.set_cover(book_id, epubmi.cover_data[1])
+                try:
+                    db.set_cover(book_id, epubmi.cover_data[1])
+                except:
+                    print("Failed to set_cover, skipping")
 
         # set author link if found.  All current adapters have authorUrl, except anonymous on AO3.
         if 'authorUrl' in book['all_metadata']:
@@ -1363,9 +1392,17 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                     else:
                         coldef = custom_columns[custcol]
                         label = coldef['label']
-                    
+
                     if flag == 'r' or book['added']: # flag 'n' isn't actually needed--*always* set if configured and new book.
-                        db.set_custom(book_id, book['all_metadata'][meta], label=label, commit=False)
+                        if coldef['datatype'] in ('int','float'): # for favs, etc--site specific metadata.
+                            if 'anthology_meta_list' in book and meta in book['anthology_meta_list']:
+                                # re-split list, strip commas, convert to floats, sum up.
+                                val = sum([ float(x.replace(",","")) for x in book['all_metadata'][meta].split(", ") ])
+                            else:
+                                val = unicode(book['all_metadata'][meta]).replace(",","")
+                        else:
+                            val = book['all_metadata'][meta]
+                        db.set_custom(book_id, val, label=label, commit=False)
 
                     if flag == 'a':
                         vallist = []
@@ -1526,7 +1563,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
     def convert_url_to_book(self, url):
         book = self.make_book()
         # look here for [\d,\d] at end of url, and remove?
-        mc = re.match(r"^(?P<url>.*?)(?:\[(?P<begin>\d+)?(?P<comma>,)?(?P<end>\d+)?\])?$",url)
+        mc = re.match(r"^(?P<url>.*?)(?:\[(?P<begin>\d+)?(?P<comma>[,-])?(?P<end>\d+)?\])?$",url)
         #print("url:(%s) begin:(%s) end:(%s)"%(mc.group('url'),mc.group('begin'),mc.group('end')))
         url = mc.group('url')
         book['begin'] = mc.group('begin')
@@ -1635,12 +1672,13 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
     def is_good_downloader_url(self,url):
         return adapters.getNormalStoryURL(url)
 
-    def merge_meta_books(self,existingbook,book_list):
+    def merge_meta_books(self,existingbook,book_list,fileform):
         book = self.make_book()
         book['author'] = []
         book['tags'] = []
         book['url'] = ''
         book['all_metadata'] = {}
+        book['anthology_meta_list'] = {}
         book['comment'] = ''
         book['added'] = True
         book['good'] = True
@@ -1707,12 +1745,16 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                         book['all_metadata'][k]=book['all_metadata'][k]+"\n\n"+v
                     else:
                         book['all_metadata'][k]=book['all_metadata'][k]+", "+v
+                        # flag psuedo list element.  Used so numeric
+                        # cust cols can convert back to numbers and
+                        # add.
+                        book['anthology_meta_list'][k]=True
     
         if existingbook:
             book['title'] = deftitle = existingbook['title']
             book['comments'] = existingbook['comments']
         else:
-            book['title'] = deftitle = book_list[0]['title']+" Anthology"
+            book['title'] = deftitle = book_list[0]['title']
             book['comments'] = "Anthology containing:\n" + \
                 "\n".join([ "%s by %s"%(b['title'],', '.join(b['author'])) for b in book_list ])
             # book['all_metadata']['description']
@@ -1720,12 +1762,22 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
             # if all same series, use series for name.  But only if all and not previous named
             if len(serieslist) == len(book_list):
                 series = serieslist[0]
-                book['title'] = series+" Anthology"
+                book['title'] = series
                 for sr in serieslist:
                     if series != sr:
                         book['title'] = deftitle;
                         break
             
+            configuration = get_ffdl_config(book['url'],fileform)
+            print("anthology_title_pattern:%s"%configuration.getConfig('anthology_title_pattern'))
+            if configuration.getConfig('anthology_title_pattern'):
+                tmplt = Template(configuration.getConfig('anthology_title_pattern'))
+                book['title'] = tmplt.safe_substitute({'title':book['title']}).encode('utf8')
+            else:
+                # No setting, do fall back default.  Shouldn't happen,
+                # should always have a version in defaults.
+                book['title'] = book['title']+" Anthology"
+        
         book['all_metadata']['title'] = book['title'] # because custom columns are set from all_metadata
         book['all_metadata']['author'] = ", ".join(book['author'])
         book['author_sort']=book['author']
@@ -1734,6 +1786,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                 book['tags'].remove(v)
         book['tags'].append('Anthology')
         book['all_metadata']['anthology'] = "true"
+        
         return book
 
 def split_text_to_urls(urls):

calibre-plugin/prefs.py

@@ -39,6 +39,7 @@ default_prefs['deleteotherforms'] = False
 default_prefs['adddialogstaysontop'] = False
 default_prefs['includeimages'] = False
 default_prefs['lookforurlinhtml'] = False
+default_prefs['checkforseriesurlid'] = True
 default_prefs['injectseries'] = False
 
 default_prefs['send_lists'] = ''
@@ -60,6 +61,13 @@ default_prefs['allow_custcol_from_ini'] = True
 
 default_prefs['std_cols_newonly'] = {}
 
+# This is where all preferences for this plugin *were* stored
+# Remember that this name (i.e. plugins/fanfictiondownloader_plugin) is also
+# in a global namespace, so make it as unique as possible.
+# You should always prefix your config file name with plugins/,
+# so as to ensure you dont accidentally clobber a calibre config file
+old_prefs = JSONConfig('plugins/fanfictiondownloader_plugin')
+
 def set_library_config(library_config,db):
     db.prefs.set_namespaced(PREFS_NAMESPACE,
                             PREFS_KEY_SETTINGS,
@@ -73,7 +81,7 @@ def get_library_config(db):
     if library_id in old_prefs:
         #print("get prefs from old_prefs")
         library_config = old_prefs[library_id]
-        set_library_config(library_config)
+        set_library_config(library_config,db)
         old_prefs["migrated to library db %s"%library_id] = old_prefs[library_id]
         del old_prefs[library_id]
 
@@ -83,13 +91,6 @@ def get_library_config(db):
                                                  copy.deepcopy(default_prefs))
     return library_config
 
-# This is where all preferences for this plugin *were* stored
-# Remember that this name (i.e. plugins/fanfictiondownloader_plugin) is also
-# in a global namespace, so make it as unique as possible.
-# You should always prefix your config file name with plugins/,
-# so as to ensure you dont accidentally clobber a calibre config file
-old_prefs = JSONConfig('plugins/fanfictiondownloader_plugin')
-
 # fake out so I don't have to change the prefs calls anywhere.  The
 # Java programmer in me is offended by op-overloading, but it's very
 # tidy.

defaults.ini

@@ -1,4 +1,4 @@
-# Copyright 2012 Fanficdownloader team
+# Copyright 2013 Fanficdownloader team
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -44,6 +44,9 @@ language_label:Language
 characters_label:Characters
 ships_label:Relationships
 series_label:Series
+seriesUrl_label:Series URL
+## seriesHTML is series as a link to seriesUrl.
+seriesHTML_label:Series
 ## Completed/In-Progress
 status_label:Status
 ## Dates story first published, last updated, and downloaded(last with time).
@@ -81,7 +84,7 @@ dateUpdated_format:%%Y-%%m-%%d
 ## You can include extra text or HTML that will be included as-is in
 ## the title page. Eg: titlepage_entries: ...,<br />,summary,<br />,...
 ## All current formats already include title and author.
-titlepage_entries: series,category,genre,language,characters,ships,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,description
+titlepage_entries: seriesHTML,category,genre,language,characters,ships,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,description
 
 ## Try to collect series name and number of this story in series.
 ## Some sites (ab)use 'series' for reading lists and personal
@@ -166,6 +169,8 @@ extratags: FanFiction
 ## *Five* part lines.  Effect only when trailing conditional key=>regexp matches
 ## metakey[,metakey]=>pattern=>replacement[&&conditionalkey=>regexp]
 ## Note that if metakey == conditionalkey the conditional is ignored.
+## You can use \s in the replacement to add explicit spaces.  (The config parser
+## tends to discard trailing spaces.)
 #replace_metadata:
 # genre,category=>Sci-Fi=>SF
 # Puella Magi Madoka Magica.* => Madoka
@@ -186,6 +191,10 @@ extratags: FanFiction
 ## summary.
 keep_summary_html:true
 
+## If set true, any style attributes on tags in the story HTML will be
+## kept.  Useful for keeping extra colors & formatting from original.
+#keep_style_attr: false
+
 ## Don't like the numbers at the start of chapter titles on some
 ## sites?  You can use strip_chapter_numbers to strip them off.  Just
 ## want to make them all look the same?  Strip them off, then add them
@@ -270,7 +279,7 @@ output_css:
 
 [txt]
 ## Add URLs since there aren't links.
-titlepage_entries: series,category,genre,language,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,storyUrl, authorUrl, description
+titlepage_entries: series,seriesUrl,category,genre,language,characters,ships,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,storyUrl, authorUrl, description
 
 ## Width to word wrap text output.  0 indicates no wrapping.
 wrap_width: 78
@@ -304,13 +313,13 @@ include_logpage: false
 ## if in the list.  You can include extra text or HTML that will be
 ## included as-is in each log entry. Eg: logpage_entries: ...,<br />,
 ## summary,<br />,...
-logpage_entries: dateCreated,datePublished,dateUpdated,numChapters,numWords,status,title,author,description,category,genre,rating,warnings
+logpage_entries: dateCreated,datePublished,dateUpdated,numChapters,numWords,status,series,title,author,description,category,genre,rating,warnings
 
 ## epub->mobi conversions typically don't like tables.
 titlepage_use_table: false
 
 ## When using tables, make these span both columns.
-wide_titlepage_entries: description, storyUrl, author URL
+wide_titlepage_entries: description, storyUrl, authorUrl, seriesUrl
 
 ## output background color--only used by html and epub (and ignored in
 ## epub by many readers).  Included below in output_css--will be
@@ -504,6 +513,15 @@ extraships:Severus Snape/Hermione Granger
 #username:YourName
 #password:yourpassword
 
+[asr3.slashzone.org]
+## Site dedicated to these categories/characters/ships
+extracategories:The Sentinel
+
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
 [bloodties-fans.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:Blood Ties
@@ -568,6 +586,11 @@ extraships:Spike/Buffy
 #username:YourName
 #password:yourpassword
 
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
 [dramione.org]
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter
@@ -599,7 +622,7 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 ## composite metadata entries.  dramione.org, for example, adds
 ## 'cliches' and then defines as the composite of hermiones,dracos in
 ## include_in_cliches.
-extra_valid_entries:themes,hermiones,dracos,timeline,cliches
+extra_valid_entries:themes,hermiones,dracos,timeline,cliches,read,reviews
 include_in_cliches:hermiones,dracos
 
 ## For another example, you could, by uncommenting this line, include
@@ -699,6 +722,10 @@ extracharacters:Hermione Granger
 ## personal.ini, not defaults.ini.
 #is_adult:true
 
+## Extra metadata that this adapter knows about.  See [dramione.org]
+## for examples of how to use them.
+extra_valid_entries:read,reviews
+
 [hlfiction.net]
 ## Site dedicated to these categories/characters/ships
 extracategories:Highlander
@@ -759,6 +786,8 @@ extracategories:West Wing
 ## Site dedicated to these categories/characters/ships
 extracategories:NCIS
 
+[netraptor.org]
+
 [nfacommunity.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:NCIS
@@ -915,6 +944,15 @@ extracategories:Harry Potter
 ## personal.ini, not defaults.ini.
 #is_adult:true
 
+[tokra.fandomnet.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Stargate: SG-1
+
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
 [www.adastrafanfic.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:Star Trek
@@ -969,6 +1007,13 @@ extracategories:InuYasha
 extracharacters:Sesshoumaru,Kagome
 extraships:Sesshoumaru/Kagome
 
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
 [www.dotmoon.net]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -1138,6 +1183,11 @@ cover_exclusion_regexp:/stories/999/images/.*?_trophy.png
 ## Site dedicated to these categories/characters/ships
 extracategories:NCIS
 
+[www.nickandgreg.net]
+## Site dedicated to these categories/characters/ships
+extracategories:CSI
+extraships:Nick Stokes/Greg Sanders
+
 [www.phoenixsong.net]
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter
@@ -1219,12 +1269,12 @@ extraships:Harry Potter/Ginny Weasley
 # www.squidge.org/peja calls it Fandom <shrug>
 category_label:Fandom
 # Remove numWords -- www.squidge.org/peja word counts are inaccurate
-titlepage_entries: series,category,genre,language,characters,ships,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,site,description
+titlepage_entries: seriesHTML,category,genre,language,characters,ships,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,site,description
 
 [www.squidge.org/peja:txt]
 ## Add URLs since there aren't links.
 # Remove numWords -- www.squidge.org/peja word counts are inaccurate
-titlepage_entries: series,category,genre,language,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,site,storyUrl, authorUrl, description
+titlepage_entries: series,seriesUrl,category,genre,language,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,site,storyUrl, authorUrl, description
 
 [www.storiesofarda.com]
 ## Site dedicated to these categories/characters/ships
@@ -1346,3 +1396,47 @@ extracategories:Stargate: Atlantis
 ## extratags params in all other sections.  Only commandline options
 ## beat overrides.
 #extratags:fanficdownloader
+
+
+[teststory:defaults]
+valid_entries:title,author_list,authorId_list,authorUrl_list,storyUrl,
+ datePublished,dateUpdated,numWords,status,language,series,seriesUrl,
+ rating,category_list,genre_list,warnings_list,characters_list
+
+# {{storyId}} is a special case--it's the only one that works.
+title:Test Story Title {{storyId}}
+author_list:Test Author aa
+authorId_list:1
+authorUrl_list:http://test1.com?authid=1
+storyUrl:http://test1.com?sid={{storyId}}
+datePublished:1975-03-15
+dateUpdated:1975-04-15
+numWords:123,456
+status:In-Progress
+language:English
+
+chaptertitles:Prologue
+
+## Add additional sections with different numbers to get different
+## parameters for different story urls.
+## test1.com?sid=1000
+[teststory:1000]
+# note the leading commas when doing add_to_ with valid_entries and *_list
+add_to_valid_entries:,favs
+title:Testing New Feature {{storyId}}
+author_list:Bob Smith
+authorId_list:45
+authorUrl_list:http://test1.com?authid=45
+datePublished:2013-03-15
+dateUpdated:2013-04-15
+numWords:1456
+favs:56
+series:The Great Test [4]
+seriesUrl:http://test1.com?seriesid=1
+rating:Tweenie
+category_list:Harry Potter,Furbie,Crossover,Puella Magi Madoka Magica/魔法少女まどか★マギカ,Magical Girl Lyrical Nanoha
+genre_list:Fantasy,Comedy,Sci-Fi,Noir
+warnings_list:Swearing,Violence
+characters_list:Bob Smith,George Johnson,Fred Smythe
+
+chaptertitles:Prologue,Chapter 1\, Xenos on Cinnabar,Chapter 2\, Sinmay on Kintikin,3. Chapter 3

downloader.py

@@ -33,16 +33,24 @@ if sys.version_info >= (2, 7):
     rootlogger.addHandler(loghandler)
 
 try:
-    from fanficdownloader import adapters,writers,exceptions
-    from fanficdownloader.configurable import Configuration
-    from fanficdownloader.epubutils import get_dcsource_chaptercount, get_update_data
-    from fanficdownloader.geturls import get_urls_from_page
+    from calibre.constants import numeric_version as calibre_version
+    is_calibre = True
 except:
+    is_calibre = False
+
+# using try/except directly was masking errors during development.
+if is_calibre:
     # running under calibre
     from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapters,writers,exceptions
     from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.configurable import Configuration
     from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.epubutils import get_dcsource_chaptercount, get_update_data
     from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.geturls import get_urls_from_page
+else:
+    from fanficdownloader import adapters,writers,exceptions
+    from fanficdownloader.configurable import Configuration
+    from fanficdownloader.epubutils import get_dcsource_chaptercount, get_update_data
+    from fanficdownloader.geturls import get_urls_from_page
+    
 
 if sys.version_info < (2, 5):
     print "This program requires Python 2.5 or newer."
@@ -94,6 +102,9 @@ def main(argv,
    parser.add_option("-l", "--list",
                      action="store_true", dest="list",
                      help="Get list of valid story URLs from page given.",)
+   parser.add_option("-n", "--normalize-list",
+                     action="store_true", dest="normalize",default=False,
+                     help="Get list of valid story URLs from page given, but normalized to standard forms.",)
    parser.add_option("-d", "--debug",
                      action="store_true", dest="debug",
                      help="Show debug output while downloading.",)
@@ -169,8 +180,8 @@ def main(argv,
            (var,val) = opt.split('=')
            configuration.set("overrides",var,val)
 
-   if options.list:
-       retlist = get_urls_from_page(args[0], configuration)
+   if options.list or options.normalize:
+       retlist = get_urls_from_page(args[0], configuration, normalize=options.normalize)
        print "\n".join(retlist)
                
        return

fanficdownloader/adapters/__init__.py

@@ -71,7 +71,6 @@ import adapter_thehexfilesnet
 import adapter_dokugacom
 import adapter_iketernalnet
 import adapter_onedirectionfanfictioncom
-import adapter_prisonbreakficnet
 import adapter_storiesofardacom
 import adapter_samdeanarchivenu
 import adapter_destinysgatewaycom
@@ -88,7 +87,6 @@ import adapter_pretendercentrecom
 import adapter_darksolaceorg
 import adapter_finestoriescom
 import adapter_hpfanficarchivecom
-import adapter_svufictioncom
 import adapter_twilightarchivescom
 import adapter_wizardtalesnet
 import adapter_nhamagicalworldsus
@@ -103,7 +101,6 @@ import adapter_merlinficdtwinscouk
 import adapter_thehookupzonenet
 import adapter_bloodtiesfancom
 import adapter_indeathnet
-import adapter_jlaunlimitedcom
 import adapter_qafficcom
 import adapter_efpfanficnet
 import adapter_potterficscom
@@ -115,6 +112,10 @@ import adapter_imagineeficcom
 import adapter_buffynfaithnet
 import adapter_psychficcom
 import adapter_hennethannunnet
+import adapter_tokrafandomnetcom
+import adapter_netraptororg
+import adapter_asr3slashzoneorg
+import adapter_nickandgregnet
 
 ## This bit of complexity allows adapters to be added by just adding
 ## importing.  It eliminates the long if/else clauses we used to need
@@ -153,7 +154,7 @@ def getAdapter(config,url):
 
     #logger.debug("trying url:"+url)
     (cls,fixedurl) = getClassFor(url)
-    logger.debug("fixedurl:"+fixedurl)
+    #logger.debug("fixedurl:"+fixedurl)
     if cls:
         adapter = cls(config,fixedurl) # raises InvalidStoryURL
         return adapter

fanficdownloader/adapters/adapter_adastrafanficcom.py

@@ -195,6 +195,7 @@ class AdAstraFanficComSiteAdapter(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             

fanficdownloader/adapters/adapter_archiveofourownorg.py

@@ -1,4 +1,4 @@
-# -*- coding: utf-8 -*-
+#  -*- coding: utf-8 -*-
 
 # Copyright 2011 Fanficdownloader team
 #
@@ -292,9 +292,10 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
             a = metasoup.find('dd',{'class':"series"})
             b = a.find('a', href=re.compile(r"/series/\d+"))
             series_name = b.string
-            series_url = 'http://'+self.host+'/fanfic/'+b['href']
+            series_url = 'http://'+self.host+b['href']
             series_index = int(a.text.split(' ')[1])
             self.setSeries(series_name, series_index)
+            self.story.setMetadata('seriesUrl',series_url)
             
         except:
             # I find it hard to care if the series parsing fails

fanficdownloader/adapters/adapter_asr3slashzoneorg.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-# Copyright 2011 Fanficdownloader team
+# Copyright 2013 Fanficdownloader team
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -27,12 +27,10 @@ from .. import exceptions as exceptions
 
 from base_adapter import BaseSiteAdapter,  makeDate
 
-
 def getClass():
-    return JLAUnlimitedComAdapter
+    return Asr3SlashzoneOrgAdapter
 
-
-class JLAUnlimitedComAdapter(BaseSiteAdapter):
+class Asr3SlashzoneOrgAdapter(BaseSiteAdapter):
 
     def __init__(self, config, url):
         BaseSiteAdapter.__init__(self, config, url)
@@ -42,36 +40,34 @@ class JLAUnlimitedComAdapter(BaseSiteAdapter):
                                # Most sites that claim to be
                                # iso-8859-1 (and some that claim to be
                                # utf8) are really windows-1252.
-        self.username = "" # if left empty, site doesn't return any message at all.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
         self.password = ""
         self.is_adult=False
-
+        
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
         logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-
-        self._setURL('http://' + self.getSiteDomain() + '/eFiction1.1/viewstory.php?sid='+self.story.getMetadata('storyId'))
-
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/archive/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
         # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','jla')
+        self.story.setMetadata('siteabbrev','asr3')
 
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%m/%d/%y"
-
+        self.dateformat = "%d/%m/%y"
+            
     @staticmethod # must be @staticmethod, don't remove it.
     def getSiteDomain():
         # The site domain.  Does have www here, if it uses it.
-        return 'www.jlaunlimited.com'
+        return 'asr3.slashzone.org'
 
     def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/eFiction1.1/viewstory.php?sid=1234"
+        return "http://"+self.getSiteDomain()+"/archive/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/eFiction1.1/viewstory.php?sid=")+r"\d+$"
-
-
-
+        return re.escape("http://"+self.getSiteDomain()+"/archive/viewstory.php?sid=")+r"\d+$"
 
     ## Getting the chapter list and the meta data, plus 'is adult' checking.
     def extractChapterUrlsAndMetadata(self):
@@ -81,7 +77,7 @@ class JLAUnlimitedComAdapter(BaseSiteAdapter):
             # If the title search below fails, there's a good chance
             # you need a different number.  print data at that point
             # and see what the 'click here to continue' url says.
-            addurl = "&ageconsent=ok&warning=4" # XXX
+            addurl = "&ageconsent=ok&warning=3"
         else:
             addurl=""
 
@@ -98,8 +94,7 @@ class JLAUnlimitedComAdapter(BaseSiteAdapter):
             else:
                 raise e
 
-        # Assume that if there is a url with 'warning=#' in the page then it is a 'check' page
-        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)",data)
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
         if m != None:
             if self.is_adult or self.getConfig("is_adult"):
                 # We tried the default and still got a warning, so
@@ -120,140 +115,111 @@ class JLAUnlimitedComAdapter(BaseSiteAdapter):
                         raise e    
             else:
                 raise exceptions.AdultCheckRequired(self.url)
-                
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
         # use BeautifulSoup HTML parser to make everything easier to find.
         soup = bs.BeautifulSoup(data)
-        # print data
+        #print data
 
         # Now go hunting for all the meta data and the chapter list.
-
+        
         ## Title
-        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')))
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
         self.story.setMetadata('title',a.string)
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
         self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/archive/'+a['href'])
         self.story.setMetadata('author',a.string)
 
+        # Rating
+        rate = stripHTML(soup.find('div',{'id':'pagetitle'}))
+        rate = rate[rate.rindex('[')+1:rate.rindex(']')]
+        self.story.setMetadata('rating', rate)
+
         # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+")):
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
             # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/eFiction1.1/'+chapter['href']+addurl))
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/archive/'+chapter['href']+addurl))
 
         self.story.setMetadata('numChapters',len(self.chapterUrls))
 
         # eFiction sites don't help us out a lot with their meta data
         # formating, so it's a little ugly.
 
-        # utility method
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
+        metadiv = soup.find('div',{'class':'content'})
+        smalldiv = metadiv.find('div',{'class':'small'})
 
-        # <span class="label">Rated:</span> NC-17<br /> etc
-        labels = soup.findAll('span',{'class':'label'})
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = labelspan.string
+        categorys = smalldiv.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+        for category in categorys:
+            self.story.addToList('category',category.string)
+        
+        chars = smalldiv.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+        for char in chars:
+            self.story.addToList('characters',char.string)
 
-            if 'Summary' in label:
-                ## Everything until the next span class='label'
-                svalue = ""
-                while not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
-                    value = value.nextSibling
-                self.setDescription(url,svalue)
-                #self.story.setMetadata('description',stripHTML(svalue))
+        ships = smalldiv.parent.findAll('a',href=re.compile(r'browse\.php\?type=class&type_id=2&classid=1'))
+        for ship in ships:
+            self.story.addToList('ships',ship.string)
 
-            if 'Rated' in label:
-                self.story.setMetadata('rating', value)
+        metatext = stripHTML(smalldiv)
 
-            if 'Word count' in label:
-                self.story.setMetadata('numWords', value)
+        if 'Completed: Yes' in metatext:
+            self.story.setMetadata('status', 'Completed')
+        else:
+            self.story.setMetadata('status', 'In-Progress')
 
-            if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-                catstext = [cat.string for cat in cats]
-                for cat in catstext:
-                    self.story.addToList('category',cat.string)
+        wordstart=metatext.rindex('Word count:')+12
+        words = metatext[wordstart:metatext.index(' ',wordstart)]
+        self.story.setMetadata('numWords', words)
 
-            if 'Characters' in label:
-                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-                charstext = [char.string for char in chars]
-                for char in charstext:
-                    self.story.addToList('characters',char.string)
-
-            ## Not all sites use Genre, but there's no harm to
-            ## leaving it in.  Check to make sure the type_id number
-            ## is correct, though--it's site specific.
-            if 'Genre' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) # XXX
-                genrestext = [genre.string for genre in genres]
-                self.genre = ', '.join(genrestext)
-                for genre in genrestext:
-                    self.story.addToList('genre',genre.string)
-
-            ## Not all sites use Warnings, but there's no harm to
-            ## leaving it in.  Check to make sure the type_id number
-            ## is correct, though--it's site specific.
-            if 'Warnings' in label:
-                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
-                warningstext = [warning.string for warning in warnings]
-                self.warning = ', '.join(warningstext)
-                for warning in warningstext:
-                    self.story.addToList('warnings',warning.string)
-
-            if 'Completed' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-            
-            if 'Updated' in label:
-                # there's a stray [ at the end.
-                #value = value[0:-1]
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+        datesdiv = soup.find('div',{'class':'bottom'})
+        dates = stripHTML(datesdiv).split()
+        # Published: 04/26/2011 Updated: 03/06/2013
+        self.story.setMetadata('datePublished', makeDate(dates[1], self.dateformat))
+        self.story.setMetadata('dateUpdated', makeDate(dates[3], self.dateformat))
 
         try:
             # Find Series name from series URL.
             a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
             series_name = a.string
-            series_url = 'http://'+self.host+'/fanfic/'+a['href']
+            series_url = 'http://'+self.host+'/archive/'+a['href']
 
             # use BeautifulSoup HTML parser to make everything easier to find.
             seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
             i=1
             for a in storyas:
-                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                    self.setSeries(series_name, i)
-                    break
-                i+=1
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
+                        break
+                    i+=1
             
         except:
             # I find it hard to care if the series parsing fails
             pass
-
-
+            
+        # remove 'small' leaving only summary.
+        smalldiv.extract()
+        self.setDescription(url,metadiv)
+        
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
         logger.debug('Getting chapter text from: %s' % url)
 
-        soup = bs.BeautifulSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-
-        div = soup.find('div', {'id' : 'story'})
+        soup = bs.BeautifulSoup(self._fetchUrl(url))
         
+        div = soup.find('div', {'id' : 'story'})
+
         if None == div:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-
+    
         return self.utf8FromSoup(url,div)
-

fanficdownloader/adapters/adapter_bloodtiesfancom.py

@@ -211,7 +211,7 @@ class BloodTiesFansComAdapter(BaseSiteAdapter): # XXX
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
         self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/fiction/'+a['href'])
         self.story.setMetadata('author',a.string)
 
         # Find the chapters:
@@ -311,6 +311,7 @@ class BloodTiesFansComAdapter(BaseSiteAdapter): # XXX
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             

fanficdownloader/adapters/adapter_castlefansorg.py

@@ -184,7 +184,7 @@ class CastleFansOrgAdapter(BaseSiteAdapter): # XXX
         # Find authorid and URL from... author url.
         a = pagetitle.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
         self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/fanfic/'+a['href'])
         self.story.setMetadata('author',a.string)
 
         # Find the chapters:
@@ -284,6 +284,7 @@ class CastleFansOrgAdapter(BaseSiteAdapter): # XXX
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             

fanficdownloader/adapters/adapter_chaossycophanthexcom.py

@@ -212,6 +212,7 @@ class ChaosSycophantHexComAdapter(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             

fanficdownloader/adapters/adapter_darksolaceorg.py

@@ -49,7 +49,6 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
         logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-		
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/elysian/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -59,7 +58,7 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
 
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%d %B %Y"
+        self.dateformat = "%B %d, %Y"
             
     @staticmethod # must be @staticmethod, don't remove it.
     def getSiteDomain():
@@ -79,7 +78,8 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
     ## Login seems to be reasonably standard across eFiction sites.
     def needToLoginCheck(self, data):
         if 'This story contains adult content not suitable for children' in data \
-                or "That password doesn't match the one in our database" in data:
+                or "That password doesn't match the one in our database" in data \
+                or "Registered Users Only" in data:
             return True
         else:
             return False
@@ -94,17 +94,16 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
             params['penname'] = self.getConfig("username")
             params['password'] = self.getConfig("password")
         params['rememberme'] = '1'
-        params['sid'] = ''
-        params['intent'] = ''
+        params['action'] = 'login'
         params['submit'] = 'Submit'
     
         loginUrl = 'http://' + self.getSiteDomain() + '/elysian/user.php'
         logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['penname']))
     
-        d = self._fetchUrl(loginUrl, params)
+        d = self._postUrl(loginUrl, params)
     
-        if "User Account Page" not in d : #Member Account
+        if "Member Account" not in d : #User Account Page
             logger.info("Failed to login to URL %s as %s, or have no authorization to access the story" % (loginUrl, params['penname']))
             raise exceptions.FailedToLogin(url,params['penname'])
             return False
@@ -113,9 +112,19 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
 
     ## Getting the chapter list and the meta data, plus 'is adult' checking.
     def extractChapterUrlsAndMetadata(self):
+        
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=5"
+        else:
+            addurl=""
+
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
-        url = self.url
+        url = self.url+'&index=1'+addurl
         logger.debug("URL: "+url)
 
         try:
@@ -128,9 +137,30 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
 
         if self.needToLoginCheck(data):
             # need to log in for this one.
-            addurl="&ageconsent=ok"
             self.performLogin(url)
-            data = self._fetchUrl(url+addurl)
+            data = self._fetchUrl(url)
+            
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid & error in url.
+                addurl = addurl.replace("&","&")
+                url = self.url+'&index=1'+addurl
+                logger.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
             
         if "Access denied. This story has not been validated by the adminstrators of this site." in data:
             raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
@@ -142,33 +172,41 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
         # Now go hunting for all the meta data and the chapter list.
         
         ## Title and author
-        a = soup.find('div', {'id' : 'pagetitle'})
+        div = soup.find('div', {'id' : 'pagetitle'})
         
-        aut = a.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        aut = div.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
         self.story.setMetadata('authorId',aut['href'].split('=')[1])
         self.story.setMetadata('authorUrl','http://'+self.host+'/elysian/'+aut['href'])
         self.story.setMetadata('author',aut.string)
         aut.extract()
-        
-        self.story.setMetadata('title',a.string[:(len(a.string)-3)])
+
+        # first a tag in pagetitle is title
+        self.story.setMetadata('title',stripHTML(div.find('a')))
 
         # Find the chapters:
-        chapters=soup.find('select', {'name' : 'chapter'})
-        if chapters != None:
-            for chapter in chapters.findAll('option'):
-                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/elysian/viewstory.php?sid='+self.story.getMetadata('storyId')+'&chapter='+chapter['value']))
-        else:
-            self.chapterUrls.append((self.story.getMetadata('title'),url))
+        # chapters=soup.find('select', {'name' : 'chapter'})
+        # if chapters != None:
+        #     for chapter in chapters.findAll('option'):
+        #         self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/elysian/viewstory.php?sid='+self.story.getMetadata('storyId')+'&chapter='+chapter['value']))
+        # else:
+        #     self.chapterUrls.append((self.story.getMetadata('title'),url+"&chapter=1"))
 
+        for chapa in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+
+                                                       self.story.getMetadata('storyId')+'&chapter=\d+')):
+            self.chapterUrls.append((stripHTML(chapa),'http://'+self.host+'/elysian/'+chapa['href']))
+        
         self.story.setMetadata('numChapters',len(self.chapterUrls))
         
         asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
         
-        for list in asoup.findAll('div', {'class' : re.compile('listbox\s+')}):
-            a = list.find('a', href=re.compile(r'viewstory.php\?sid='))
-            if a != None:
-                if 'viewstory.php?sid='+self.story.getMetadata('storyId') in a['href']:
-                    break
+        # for metalist in asoup.findAll('div', {'class' : re.compile('listbox\s+')}):
+        #     a = metalist.find('a', href=re.compile(r'viewstory.php\?sid='))
+        #     if a != None:
+        #         if 'viewstory.php?sid='+self.story.getMetadata('storyId') in a['href']:
+        #             break
+
+        metalist = asoup.find('a', href=re.compile(r'viewstory.php\?sid='+
+                                                   self.story.getMetadata('storyId')+'($|[^\d])')).parent.parent
 
         # eFiction sites don't help us out a lot with their meta data
         # formating, so it's a little ugly.
@@ -182,7 +220,7 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
                 
 
         # <span class="label">Rated:</span> NC-17<br /> etc
-        labels = list.findAll('span', {'class' : 'classification'})
+        labels = metalist.findAll('span', {'class' : 'label'})
         for labelspan in labels:
             label = labelspan.text
             value = labelspan.nextSibling
@@ -190,7 +228,7 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
             if 'Summary' in label:
                 ## Everything until the next span class='label'
                 svalue = ""
-                while not (defaultGetattr(value,'class') == 'classification' or "Chapters: " in stripHTML(value)):
+                while not (defaultGetattr(value,'class') == 'label' or "Chapters: " in stripHTML(value)):
                     svalue += str(value)
                     value = value.nextSibling
                 self.setDescription(url,svalue)
@@ -238,7 +276,7 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
 
         try:
             # Find Series name from series URL.
-            a = list.find('a', href=re.compile(r"series.php\?seriesid=\d+"))
+            a = metalist.find('a', href=re.compile(r"series.php\?seriesid=\d+"))
             series_name = a.string
             series_url = 'http://'+self.host+'/elysian/'+a['href']
 
@@ -252,6 +290,7 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if ('viewstory.php?sid='+self.story.getMetadata('storyId')) in a['href']:
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             
@@ -264,8 +303,7 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
 
         logger.debug('Getting chapter text from: %s' % url)
 
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        soup = bs.BeautifulSoup(self._fetchUrl(url))
         
         div = soup.find('div', {'id' : 'story'})
 

fanficdownloader/adapters/adapter_destinysgatewaycom.py

@@ -218,6 +218,7 @@ class DestinysGatewayComAdapter(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             

fanficdownloader/adapters/adapter_dokugacom.py

@@ -81,6 +81,44 @@ class DokugaComAdapter(BaseSiteAdapter):
     def getSiteURLPattern(self):
         return r"http://"+self.getSiteDomain()+"/(fanfiction|spark)?/story/\d+/?\d+?$"
 
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'The author has disabled anonymous viewing for this story.' in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url,soup):
+        params = {}
+
+        if self.password:
+            params['username'] = self.username
+            params['passwd'] = self.password
+        else:
+            params['username'] = self.getConfig("username")
+            params['passwd'] = self.getConfig("password")
+        params['Submit'] = 'Submit'
+
+        # copy all hidden input tags to pick up appropriate tokens.
+        for tag in soup.findAll('input',{'type':'hidden'}):
+            params[tag['name']] = tag['value']
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/fanfiction'
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['username']))
+    
+        d = self._postUrl(loginUrl, params)
+
+        if "Your session has expired. Please log in again." in d:
+            d = self._postUrl(loginUrl, params)
+
+        if "Logout" not in d : #Member Account
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['username']))
+            raise exceptions.FailedToLogin(url,params['username'])
+            return False
+        else:
+            return True
 
     ## Getting the chapter list and the meta data, plus 'is adult' checking.
     def extractChapterUrlsAndMetadata(self):
@@ -97,12 +135,18 @@ class DokugaComAdapter(BaseSiteAdapter):
                 raise exceptions.StoryDoesNotExist(self.url)
             else:
                 raise e
+                
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url,soup)
+            data = self._fetchUrl(url)
+            soup = bs.BeautifulSoup(data)
             
         if "Access denied. This story has not been validated by the adminstrators of this site." in data:
             raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
         # print data
 
         # Now go hunting for all the meta data and the chapter list.
@@ -218,10 +262,6 @@ class DokugaComAdapter(BaseSiteAdapter):
             a=div.text.split('Words ')
             if len(a)==2: self.story.setMetadata('numWords', a[1])
             
-        
-
-
-            
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 

fanficdownloader/adapters/adapter_dracoandginnycom.py

@@ -273,6 +273,7 @@ class DracoAndGinnyComAdapter(BaseSiteAdapter):
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             

fanficdownloader/adapters/adapter_dramioneorg.py

@@ -237,6 +237,9 @@ class DramioneOrgAdapter(BaseSiteAdapter):
             if 'Word count' in label:
                 self.story.setMetadata('numWords', value)
 
+            if 'Read' in label:
+                self.story.setMetadata('read', value)
+
             if 'Categories' in label:
                 cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
                 for cat in cats:
@@ -274,6 +277,7 @@ class DramioneOrgAdapter(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             
@@ -281,6 +285,14 @@ class DramioneOrgAdapter(BaseSiteAdapter):
             # I find it hard to care if the series parsing fails
             pass
             
+        try:
+            self.story.setMetadata('reviews',
+                                   stripHTML(soup.find('h2',{'id':'pagetitle'}).
+                                             findAll('a', href=re.compile(r'^reviews.php'))[1]))
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 

fanficdownloader/adapters/adapter_efictionestelielde.py

@@ -195,6 +195,7 @@ class EfictionEstelielDeAdapter(BaseSiteAdapter):
                             name=seriessoup.find('div', {'id' : 'pagetitle'})
                             name.find('a').extract()
                             self.setSeries(name.text.split(' by[')[0], i)
+                            self.story.setMetadata('seriesUrl',series_url)
                             i=0
                             break
                         i+=1

fanficdownloader/adapters/adapter_efpfanficnet.py

@@ -280,6 +280,7 @@ class EFPFanFicNet(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId'))+'&i=1':
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             

fanficdownloader/adapters/adapter_erosnsapphosycophanthexcom.py

@@ -230,6 +230,7 @@ class ErosnSapphoSycophantHexComAdapter(BaseSiteAdapter):
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             

fanficdownloader/adapters/adapter_fanfictionnet.py

@@ -132,9 +132,10 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         ## For 1, use the second link.
         ## For 2, fetch the crossover page and pull the two categories from there.
 
-        categories = soup.findAll('a',{'class':'xcontrast_txt'})
+        categories = soup.find('div',{'id':'pre_story_links'}).findAll('a',{'class':'xcontrast_txt'})
+        #print("xcontrast_txt a:%s"%categories)
         if len(categories) > 1:
-            self.story.addToList('category',stripHTML(categories[-1]))
+            self.story.addToList('category',stripHTML(categories[1]))
         elif 'Crossover' in categories[0]['href']:
             caturl = "http://%s%s"%(self.getSiteDomain(),categories[0]['href'])
             catsoup = bs.BeautifulSoup(self._fetchUrl(caturl))
@@ -150,7 +151,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
 
         # after Rating, the same bit of text containing id:123456 contains
         # Complete--if completed.
-        gui_table1i = soup.find(id="gui_table1i")
+        gui_table1i = soup.find('table',{'cellpadding':'5'})
 
         self.story.setMetadata('title', stripHTML(gui_table1i.find('b'))) # title appears to be only(or at least first) bold tag in gui_table1i
 
@@ -158,12 +159,17 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         if summarydiv:
             self.setDescription(url,stripHTML(summarydiv))
             
-        
-        metatext = stripHTML(gui_table1i.find('div', {'style':'color:gray;'})).replace('Hurt/Comfort','Hurt-Comfort')
+
+        grayspan = gui_table1i.find('span', {'class':'xgray xcontrast_txt'})
+        # for b in grayspan.findAll('button'):
+        #     b.extract()
+        metatext = stripHTML(grayspan).replace('Hurt/Comfort','Hurt-Comfort')
+        #logger.debug("metatext:(%s)"%metatext)
         metalist = metatext.split(" - ")
-        #logger.debug("metatext:(%s)"%metalist)
+        #logger.debug("metalist:(%s)"%metalist)
 
         # Rated: Fiction K - English - Words: 158,078 - Published: 02-04-11
+        # Rated: Fiction T - English - Adventure/Sci-Fi - Naruto U. - Chapters: 22 - Words: 114,414 - Reviews: 395 - Favs: 779 - Follows: 835 - Updated: 03-21-13 - Published: 04-28-12 - id: 8067258 
 
         # rating is obtained above more robustly.
         if metalist[0].startswith('Rated:'):

fanficdownloader/adapters/adapter_fimfictionnet.py

@@ -179,7 +179,11 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
 
             self.setCoverImage(self.url,coverurl)
 
-        self.setDescription(self.url,soup.find("div", {"class":"description"}))
+        # fimf has started including extra stuff inside the description div.
+        descdivstr = "%s"%soup.find("div", {"class":"description"})
+        hrstr="<hr />"
+        descdivstr = '<div class="description">'+descdivstr[descdivstr.index(hrstr)+len(hrstr):]
+        self.setDescription(self.url,descdivstr)
         
         # Dates are in Unix time
         # Take the publish date from the first chapter posted
@@ -188,7 +192,7 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
         rawDateUpdated = storyMetadata["date_modified"]
         self.story.setMetadata("dateUpdated", datetime.fromtimestamp(rawDateUpdated))
 
-        chars = soup.find("div", {"class":"story"})
+        chars = soup.find("div", {"class":"inner_data"})
         # fimfic stopped putting the char name on or around the char
         # icon now for some reason.  Pull it from the image name with
         # some heuristics.

fanficdownloader/adapters/adapter_finestoriescom.py

@@ -187,6 +187,7 @@ class FineStoriesComAdapter(BaseSiteAdapter):
             a = lc4.find('a', href=re.compile(r"/library/show_series.php\?id=\d+"))
             i = a.parent.text.split('(')[1].split(')')[0]
             self.setSeries(a.text, i)
+            self.story.setMetadata('seriesUrl','http://'+self.host+a['href'])
         except:
             pass
         try:

fanficdownloader/adapters/adapter_grangerenchantedcom.py

@@ -227,6 +227,9 @@ class GrangerEnchantedCom(BaseSiteAdapter):
             if 'Word count' in label:
                 self.story.setMetadata('numWords', value)
 
+            if 'Read' in label:
+                self.story.setMetadata('read', value)
+
             if 'Categories' in label:
                 cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
                 for cat in cats:
@@ -275,9 +278,17 @@ class GrangerEnchantedCom(BaseSiteAdapter):
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
-            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+
+        try:
+            self.story.setMetadata('reviews',
+                                   stripHTML(soup.find('div',{'id':'sort'}).
+                                             findAll('a', href=re.compile(r'^reviews.php'))[1]))
         except:
             # I find it hard to care if the series parsing fails
             pass

fanficdownloader/adapters/adapter_hlfictionnet.py

@@ -206,6 +206,7 @@ class HLFictionNetAdapter(BaseSiteAdapter):
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if ('viewstory.php?sid='+self.story.getMetadata('storyId')) in a['href']:
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             

fanficdownloader/adapters/adapter_hpfanficarchivecom.py

@@ -193,6 +193,7 @@ class HPFanficArchiveComAdapter(BaseSiteAdapter):
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             

fanficdownloader/adapters/adapter_imagineeficcom.py

@@ -264,6 +264,7 @@ class ImagineEFicComAdapter(BaseSiteAdapter):
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             

fanficdownloader/adapters/adapter_ksarchivecom.py

@@ -285,6 +285,7 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             

fanficdownloader/adapters/adapter_libraryofmoriacom.py

@@ -133,7 +133,7 @@ class LibraryOfMoriaComAdapter(BaseSiteAdapter):
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
         self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/a/'+a['href'])
         self.story.setMetadata('author',a.string)
 
         # Find the chapters:
@@ -225,6 +225,7 @@ class LibraryOfMoriaComAdapter(BaseSiteAdapter):
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             

fanficdownloader/adapters/adapter_lumossycophanthexcom.py

@@ -212,6 +212,7 @@ class LumosSycophantHexComAdapter(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             

fanficdownloader/adapters/adapter_merlinficdtwinscouk.py

@@ -268,6 +268,7 @@ class MerlinFicDtwinsCoUk(BaseSiteAdapter):
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             

fanficdownloader/adapters/adapter_midnightwhispersca.py

@@ -260,6 +260,7 @@ class MidnightwhispersCaAdapter(BaseSiteAdapter): # XXX
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             

fanficdownloader/adapters/adapter_mugglenetcom.py

@@ -306,6 +306,7 @@ class MuggleNetComAdapter(BaseSiteAdapter): # XXX
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             

fanficdownloader/adapters/adapter_nationallibrarynet.py

@@ -163,6 +163,7 @@ class NationalLibraryNetAdapter(BaseSiteAdapter):
             
             if 'Series' in label:
                 self.setSeries(stripHTML(value.nextSibling), value.nextSibling.nextSibling.string[2:])
+                self.story.setMetadata('seriesUrl','http://'+self.host+'/'+value.nextSibling['href'])
                 
         asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
         story=asoup.find('a', href=re.compile(r'viewstory.php\?storyid='+self.story.getMetadata('storyId')))

fanficdownloader/adapters/adapter_ncisficcom.py

@@ -164,6 +164,7 @@ class NCISFicComAdapter(BaseSiteAdapter):
             if 'Series' in label:
                 if "No Series" not in value.nextSibling.string:
                     self.setSeries(stripHTML(value.nextSibling), value.nextSibling.nextSibling.string[2:])
+                    self.story.setMetadata('seriesUrl','http://'+self.host+'/'+value.nextSibling['href'])
                 
         asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
         story=asoup.find('a', href=re.compile(r'viewstory.php\?storyid='+self.story.getMetadata('storyId')))

fanficdownloader/adapters/adapter_ncisfictionnet.py

@@ -193,7 +193,7 @@ class NCISFictionNetAdapter(BaseSiteAdapter):
             series_name = a.find('a').string
             i = a.text.split('#')[1]
             self.setSeries(series_name, i)
-            
+            self.story.setMetadata('seriesUrl','http://'+self.host+'/'+a.find('a')['href'])
         except:
             # I find it hard to care if the series parsing fails
             pass

fanficdownloader/adapters/adapter_netraptororg.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-# Copyright 2012 Fanficdownloader team
+# Copyright 2013 Fanficdownloader team
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -28,11 +28,9 @@ from .. import exceptions as exceptions
 from base_adapter import BaseSiteAdapter,  makeDate
 
 def getClass():
-    return PrisonBreakFicNetAdapter
+    return NetRaptorOrgAdapter
 
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class PrisonBreakFicNetAdapter(BaseSiteAdapter):
+class NetRaptorOrgAdapter(BaseSiteAdapter):
 
     def __init__(self, config, url):
         BaseSiteAdapter.__init__(self, config, url)
@@ -51,31 +49,29 @@ class PrisonBreakFicNetAdapter(BaseSiteAdapter):
         logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        self._setURL('http://' + self.getSiteDomain() + '/fanfiction/viewstory.php?sid='+self.story.getMetadata('storyId'))
         
         # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','pbf')
+        self.story.setMetadata('siteabbrev','netrap')
 
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%B %d, %Y"
+        self.dateformat = "%d/%m/%y"
             
     @staticmethod # must be @staticmethod, don't remove it.
     def getSiteDomain():
         # The site domain.  Does have www here, if it uses it.
-        return 'www.prisonbreakfic.net'
+        return 'netraptor.org'
 
     def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+        return "http://"+self.getSiteDomain()+"/fanfiction/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+        return re.escape("http://"+self.getSiteDomain()+"/fanfiction/viewstory.php?sid=")+r"\d+$"
 
     ## Getting the chapter list and the meta data, plus 'is adult' checking.
     def extractChapterUrlsAndMetadata(self):
 
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
         url = self.url+'&index=1'
         logger.debug("URL: "+url)
 
@@ -86,6 +82,10 @@ class PrisonBreakFicNetAdapter(BaseSiteAdapter):
                 raise exceptions.StoryDoesNotExist(self.url)
             else:
                 raise e
+
+
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
             
         # use BeautifulSoup HTML parser to make everything easier to find.
         soup = bs.BeautifulSoup(data)
@@ -94,19 +94,20 @@ class PrisonBreakFicNetAdapter(BaseSiteAdapter):
         # Now go hunting for all the meta data and the chapter list.
         
         ## Title
-        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        pagetitle = soup.find('div',{'id':'pagetitle'})
+        a = pagetitle.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
         self.story.setMetadata('title',a.string)
         
         # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        a = pagetitle.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
         self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/fanfiction/'+a['href'])
         self.story.setMetadata('author',a.string)
 
         # Find the chapters:
         for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
             # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']))
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/fanfiction/'+chapter['href']))
 
         self.story.setMetadata('numChapters',len(self.chapterUrls))
 
@@ -119,10 +120,7 @@ class PrisonBreakFicNetAdapter(BaseSiteAdapter):
                 return d[k]
             except:
                 return ""
-                
-# summary, rated, word count, categories, characters, genre, warnings, completed, published, updated, seires
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
+				
         labels = soup.findAll('span',{'class':'label'})
         for labelspan in labels:
             value = labelspan.nextSibling
@@ -145,21 +143,18 @@ class PrisonBreakFicNetAdapter(BaseSiteAdapter):
 
             if 'Categories' in label:
                 cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-                for cat in cats:
+                catstext = [cat.string for cat in cats]
+                for cat in catstext:
                     self.story.addToList('category',cat.string)
 
             if 'Characters' in label:
                 chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-                for char in chars:
+                charstext = [char.string for char in chars]
+                for char in charstext:
                     self.story.addToList('characters',char.string)
 
-            if 'Pairing' in label:
-                ships = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=3')) # XXX
-                for ship in ships:
-                    self.story.addToList('ships',ship.string.replace(" and ","/"))
-
             if 'Genre' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) # XXX
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
                 for genre in genres:
                     self.story.addToList('genre',genre.string)
 
@@ -184,7 +179,7 @@ class PrisonBreakFicNetAdapter(BaseSiteAdapter):
             # Find Series name from series URL.
             a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
             series_name = a.string
-            series_url = 'http://'+self.host+'/'+a['href']
+            series_url = 'http://'+self.host+'/fanfiction/'+a['href']
 
             # use BeautifulSoup HTML parser to make everything easier to find.
             seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
@@ -193,6 +188,7 @@ class PrisonBreakFicNetAdapter(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             
@@ -205,8 +201,7 @@ class PrisonBreakFicNetAdapter(BaseSiteAdapter):
 
         logger.debug('Getting chapter text from: %s' % url)
 
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        soup = bs.BeautifulSoup(self._fetchUrl(url))
         
         div = soup.find('div', {'id' : 'story'})
 

fanficdownloader/adapters/adapter_nfacommunitycom.py

@@ -264,6 +264,7 @@ class NfaCommunityComAdapter(BaseSiteAdapter): # XXX
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             

fanficdownloader/adapters/adapter_nickandgregnet.py

@@ -0,0 +1,174 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return NickAndGregNetAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class NickAndGregNetAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
+        self._setURL('http://' + self.getSiteDomain() + '/desert_archive/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','nag')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%Y/%m/%d"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.nickandgreg.net'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/desert_archive/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/desert_archive/viewstory.php?sid=")+r"\d+$"
+    
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&i=1'
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/desert_archive/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        chapters = soup.find('select')
+        for chapter in chapters.findAll('option'):
+            if chapter.text != 'Story Index' and chapter.text != 'Chapters':
+                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/desert_archive/'+chapter['value']))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+        
+        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
+        
+        for div in asoup.findAll('td', {'class' : 'tblborder6'}):
+            a = div.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+            if a != None:
+                break
+        
+        self.setDescription(url,div.find('br').nextSibling)
+        
+        a=div.text.split('Rating:')
+        if len(a) == 2: self.story.setMetadata('rating', a[1].split(' -')[0])
+        
+        a=div.text.split('Characters:')
+        if len(a) == 2: 
+        	for char in a[1].split(' -')[0].split(', '):
+        	    self.story.addToList('characters',char)
+        	    
+        a=div.text.split('Genres:')
+        if len(a) == 2: 
+        	for genre in a[1].split(' -')[0].split(', '):
+        	    self.story.addToList('genre',genre)
+        	    
+        a=div.text.split('Warnings:')
+        if len(a) == 2: 
+        	for warn in a[1].split(' -')[0].split(', '):
+        	    if 'none' not in warn:
+        	        self.story.addToList('warnings',warn)
+        	    
+        a=div.text.split('Completed:')
+        if len(a) ==2:
+            if 'Yes' in a[1]:
+                self.story.setMetadata('status', 'Completed')
+            else:
+                self.story.setMetadata('status', 'In-Progress')
+                
+        a=div.text.split('Published:')
+        if len(a) == 2: self.story.setMetadata('datePublished', makeDate(stripHTML(a[1].split(' -')[0]), self.dateformat))
+        
+        a=div.text.split('Updated:')
+        if len(a) == 2: self.story.setMetadata('dateUpdated', makeDate(stripHTML(a[1].split(' -')[0]), self.dateformat))
+
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('table', {'class' : 'tblborder6'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_onedirectionfanfictioncom.py

@@ -244,6 +244,7 @@ class OneDirectionFanfictionComAdapter(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             

fanficdownloader/adapters/adapter_pommedesangcom.py

@@ -275,6 +275,7 @@ class PommeDeSangComAdapter(BaseSiteAdapter):
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if ('viewstory.php?sid='+self.story.getMetadata('storyId')) in a['href']:
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             

fanficdownloader/adapters/adapter_ponyfictionarchivenet.py

@@ -225,6 +225,7 @@ class PonyFictionArchiveNetAdapter(BaseSiteAdapter):
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             

fanficdownloader/adapters/adapter_potionsandsnitchesnet.py

@@ -182,6 +182,7 @@ class PotionsAndSnitchesNetSiteAdapter(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             

fanficdownloader/adapters/adapter_pretendercentrecom.py

@@ -228,6 +228,7 @@ class PretenderCenterComAdapter(BaseSiteAdapter):
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             

fanficdownloader/adapters/adapter_psychficcom.py

@@ -223,6 +223,7 @@ class PsychFicComAdapter(BaseSiteAdapter):
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             

fanficdownloader/adapters/adapter_qafficcom.py

@@ -236,6 +236,7 @@ class QafFicComAdapter(BaseSiteAdapter):
                             name=seriessoup.find('div', {'id' : 'pagetitle'})
                             name.find('a').extract()
                             self.setSeries(name.text.split(' by[')[0], i)
+                            self.story.setMetadata('seriesUrl',series_url)
                             i=0
                             break
                         i+=1

fanficdownloader/adapters/adapter_samdeanarchivenu.py

@@ -207,6 +207,7 @@ class SamDeanArchiveNuAdapter(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             

fanficdownloader/adapters/adapter_scarvesandcoffeenet.py

@@ -222,6 +222,7 @@ class ScarvesAndCoffeeNetAdapter(BaseSiteAdapter):
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             

fanficdownloader/adapters/adapter_sg1heliopoliscom.py

@@ -233,6 +233,7 @@ class SG1HeliopolisComAdapter(BaseSiteAdapter):
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             

fanficdownloader/adapters/adapter_sinfuldesireorg.py

@@ -226,6 +226,7 @@ class SinfulDesireOrgAdapter(BaseSiteAdapter):
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             

fanficdownloader/adapters/adapter_siyecouk.py

@@ -215,6 +215,7 @@ class SiyeCoUkAdapter(BaseSiteAdapter): # XXX
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             

fanficdownloader/adapters/adapter_squidgeorgpeja.py

@@ -212,6 +212,7 @@ class SquidgeOrgPejaAdapter(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
                 

fanficdownloader/adapters/adapter_stargateatlantisorg.py

@@ -204,6 +204,7 @@ class StargateAtlantisOrgAdapter(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             

fanficdownloader/adapters/adapter_tenhawkpresentscom.py

@@ -220,6 +220,7 @@ class TenhawkPresentsComSiteAdapter(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             

fanficdownloader/adapters/adapter_test1.py

@@ -47,31 +47,63 @@ class TestSiteAdapter(BaseSiteAdapter):
         return BaseSiteAdapter.getSiteURLPattern(self)+r'/?\?sid=\d+$'
 
     def extractChapterUrlsAndMetadata(self):
+        idstr = self.story.getMetadata('storyId')
+        idnum = int(idstr)
 
-        if self.story.getMetadata('storyId') == '665' and not (self.is_adult or self.getConfig("is_adult")):
+        if idnum >= 1000:
+            logger.warn("storyId:%s - Custom INI data will be used."%idstr)
+
+            sections = ['teststory:%s'%idstr,'teststory:defaults']
+            #print("self.get_config_list(sections,'valid_entries'):%s"%self.get_config_list(sections,'valid_entries'))
+            for key in self.get_config_list(sections,'valid_entries'):
+                if key.endswith("_list"):
+                    nkey = key[:-len("_list")]
+                    #print("addList:%s"%(nkey))
+                    for val in self.get_config_list(sections,key):
+                        #print("addList:%s->%s"%(nkey,val))
+                        self.story.addToList(nkey,val.decode('utf-8').replace('{{storyId}}',idstr))
+                else:
+                    # Special cases:
+                    if key in ['datePublished','dateUpdated']:
+                        self.story.setMetadata(key,makeDate(self.get_config(sections,key),"%Y-%m-%d"))
+                    else:
+                        self.story.setMetadata(key,self.get_config(sections,key).decode('utf-8').replace('{{storyId}}',idstr))
+                    #print("set:%s->%s"%(key,self.story.getMetadata(key)))
+
+            self.chapterUrls = []
+            for (j,chap) in enumerate(self.get_config_list(sections,'chaptertitles'),start=1):
+                self.chapterUrls.append( (chap,self.url+"&chapter=%d"%j) )
+            # self.chapterUrls = [(u'Prologue '+self.crazystring,self.url+"&chapter=1"),
+            #                 ('Chapter 1, Xenos on Cinnabar',self.url+"&chapter=2"),
+            #                 ]
+            self.story.setMetadata('numChapters',len(self.chapterUrls))
+            
+            return
+
+        if idstr == '665' and not (self.is_adult or self.getConfig("is_adult")):
             logger.warn("self.is_adult:%s"%self.is_adult)
             raise exceptions.AdultCheckRequired(self.url)
 
-        if self.story.getMetadata('storyId') == '666':
+        if idstr == '666':
             raise exceptions.StoryDoesNotExist(self.url)
 
-        if self.story.getMetadata('storyId').startswith('670'):
+        if idstr.startswith('670'):
             time.sleep(1.0)
             
-        if self.story.getMetadata('storyId').startswith('671'):
+        if idstr.startswith('671'):
             time.sleep(1.0)
             
         if self.getConfig("username"):
             self.username = self.getConfig("username")
         
-        if self.story.getMetadata('storyId') == '668' and self.username != "Me" :
+        if idstr == '668' and self.username != "Me" :
             raise exceptions.FailedToLogin(self.url,self.username)
 
-        if self.story.getMetadata('storyId') == '664':
-            self.story.setMetadata(u'title',"Test Story Title "+self.story.getMetadata('storyId')+self.crazystring)
+        if idstr == '664':
+            self.story.setMetadata(u'title',"Test Story Title "+idstr+self.crazystring)
             self.story.setMetadata('author','Test Author aa bare amp(&) quote(') amp(&)')
         else:
-            self.story.setMetadata(u'title',"Test Story Title "+self.story.getMetadata('storyId'))
+            self.story.setMetadata(u'title',"Test Story Title "+idstr)
             self.story.setMetadata('author','Test Author aa')
         self.story.setMetadata('storyUrl',self.url)
         self.setDescription(self.url,u'Description '+self.crazystring+u''' Done
@@ -79,13 +111,12 @@ class TestSiteAdapter(BaseSiteAdapter):
 Some more longer description.  "I suck at summaries!"  "Better than it sounds!"  "My first fic"
 ''')
         self.story.setMetadata('datePublished',makeDate("1975-03-15","%Y-%m-%d"))
-        if self.story.getMetadata('storyId') == '669':
+        if idstr == '669':
             self.story.setMetadata('dateUpdated',datetime.datetime.now())
         else:
             self.story.setMetadata('dateUpdated',makeDate("1975-04-15","%Y-%m-%d"))
         self.story.setMetadata('numWords','123456')
 
-        idnum = int(self.story.getMetadata('storyId'))
         if idnum % 2 == 1:
             self.story.setMetadata('status','In-Progress')
         else:
@@ -101,12 +132,14 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
                 }
             self.story.setMetadata('language',langs[idnum%len(langs)])
             self.setSeries('The Great Test',idnum)
+            self.story.setMetadata('seriesUrl','http://test1.com?seriesid=1')
         if idnum == 0:
             self.setSeries("A Nook Hyphen Test "+self.story.getMetadata('dateCreated'),idnum)
+            self.story.setMetadata('seriesUrl','http://test1.com?seriesid=0')
             
         self.story.setMetadata('rating','Tweenie')
 
-        if self.story.getMetadata('storyId') == '673':
+        if idstr == '673':
             self.story.addToList('author','Author From List')
             self.story.addToList('author','Author From List 2')
         
@@ -126,21 +159,37 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
         self.story.addToList('warnings','Swearing')
         self.story.addToList('warnings','Violence')
 
-        self.story.addToList('category','Harry Potter')
-        self.story.addToList('category','Furbie')
-        self.story.addToList('category','Crossover')
-        self.story.addToList('category',u'Puella Magi Madoka Magica/魔法少女まどか★マギカ')
-        self.story.addToList('category',u'Magical Girl Lyrical Nanoha')
-
+        if idstr == '80':
+            self.story.addToList('category',u'Rizzoli & Isles')
+            self.story.addToList('characters','J. Rizzoli')
+        elif idstr == '81':
+            self.story.addToList('category',u'Pitch Perfect')
+            self.story.addToList('characters','Chloe B.')
+        elif idstr == '83':
+            self.story.addToList('category',u'Rizzoli & Isles')
+            self.story.addToList('characters','J. Rizzoli')
+            self.story.addToList('category',u'Pitch Perfect')
+            self.story.addToList('characters','Chloe B.')
+            self.story.addToList('ships','Chloe B. & J. Rizzoli')
+        elif idstr == '82':
+            self.story.addToList('characters','Henry (Once Upon a Time)')        
+            self.story.addToList('category',u'Once Upon a Time (TV)')
+        else:
+            self.story.addToList('category','Harry Potter')
+            self.story.addToList('category','Furbie')
+            self.story.addToList('category','Crossover')
+            self.story.addToList('category',u'Puella Magi Madoka Magica/魔法少女まどか★マギカ')
+            self.story.addToList('category',u'Magical Girl Lyrical Nanoha')
+            self.story.addToList('category',u'Once Upon a Time (TV)')
+            self.story.addToList('characters','Bob Smith')
+            self.story.addToList('characters','George Johnson')
+            self.story.addToList('characters','Fred Smythe')
+            
         self.story.addToList('genre','Fantasy')
         self.story.addToList('genre','Comedy')
         self.story.addToList('genre','Sci-Fi')
         self.story.addToList('genre','Noir')
-        
-        self.story.addToList('characters','Bob Smith')
-        self.story.addToList('characters','George Johnson')
-        self.story.addToList('characters','Fred Smythe')
-        
+                
         self.story.addToList('listX','xVal1')
         self.story.addToList('listX','xVal2')
         self.story.addToList('listX','xVal3')
@@ -200,6 +249,9 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
 <div>
 <h3>Prologue</h3>
 <p>This is a fake adapter for testing purposes.  Different sid's will give different errors:</p>
+<h4>Config(personal.ini)</h4>
+<p>sid&gt;=1000 will use custom test story data from your configuration(personal.ini)</p>
+<p>Hard coded ids:</p>
 <p>http://test1.com?sid=664 - Crazy string title</p>
 <p>http://test1.com?sid=665 - raises AdultCheckRequired</p>
 <p>http://test1.com?sid=666 - raises StoryDoesNotExist</p>
@@ -207,10 +259,6 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
 <p>http://test1.com?sid=668 - raises FailedToLogin unless username='Me'</p>
 <p>http://test1.com?sid=669 - Succeeds with Updated Date=now</p>
 <p>http://test1.com?sid=670 - Succeeds, but sleeps 2sec on each chapter</p>
-
-
-
-
 <p>http://test1.com?sid=671 - Succeeds, but sleeps 2sec metadata only</p>
 <p>http://test1.com?sid=672 - Succeeds, quick meta, sleeps 2sec chapters only</p>
 <p>http://test1.com?sid=673 - Succeeds, multiple authors, extra categories, genres</p>

fanficdownloader/adapters/adapter_thealphagatecom.py

@@ -189,6 +189,7 @@ class TheAlphaGateComAdapter(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             

fanficdownloader/adapters/adapter_thehookupzonenet.py

@@ -183,7 +183,7 @@ class TheHookupZoneNetAdapter(BaseSiteAdapter): # XXX
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
         self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/CriminalMinds/'+a['href'])
         self.story.setMetadata('author',a.string)
 
         # Find the chapters:
@@ -283,6 +283,7 @@ class TheHookupZoneNetAdapter(BaseSiteAdapter): # XXX
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             

fanficdownloader/adapters/adapter_thequidditchpitchorg.py

@@ -263,6 +263,7 @@ class TheQuidditchPitchOrgAdapter(BaseSiteAdapter): # XXX
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             

fanficdownloader/adapters/adapter_thewriterscoffeeshopcom.py

@@ -227,6 +227,7 @@ class TheWritersCoffeeShopComSiteAdapter(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             
@@ -243,16 +244,23 @@ class TheWritersCoffeeShopComSiteAdapter(BaseSiteAdapter):
         # issues with different SGML parsers in python.  This is a
         # nasty hack, but it works.
         data = data[data.index("<body"):]
-        
-        soup = bs.BeautifulStoneSoup(data,
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        span = soup.find('div', {'id' : 'story'})
 
-        if None == span:
+        chapter=bs.BeautifulSoup('<div class="story"></div>')
+        
+        soup = bs.BeautifulSoup(data)
+
+        found=False
+        for div in soup.findAll('div'):
+            if div.has_key('class') and div['class'] == 'notes':
+                chapter.append(div)
+            if div.has_key('id') and div['id'] == 'story':
+                chapter.append(div)
+                found=True
+
+        if not found:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,span)
+
+        return self.utf8FromSoup(url,chapter)
 
 def getClass():
     return TheWritersCoffeeShopComSiteAdapter

fanficdownloader/adapters/adapter_tokrafandomnetcom.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-# Copyright 2012 Fanficdownloader team
+# Copyright 2013 Fanficdownloader team
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -28,11 +28,11 @@ from .. import exceptions as exceptions
 from base_adapter import BaseSiteAdapter,  makeDate
 
 def getClass():
-    return SVUFictionComAdapter
+    return TokraFandomnetComAdapter
 
 # Class name has to be unique.  Our convention is camel case the
 # sitename with Adapter at the end.  www is skipped.
-class SVUFictionComAdapter(BaseSiteAdapter):
+class TokraFandomnetComAdapter(BaseSiteAdapter): 
 
     def __init__(self, config, url):
         BaseSiteAdapter.__init__(self, config, url)
@@ -54,16 +54,17 @@ class SVUFictionComAdapter(BaseSiteAdapter):
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
         
         # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','svuf')
+        self.story.setMetadata('siteabbrev','tokra')
 
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%b %d, %Y"
+        self.dateformat = "%m/%d/%Y"
             
     @staticmethod # must be @staticmethod, don't remove it.
     def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'svufiction.com'
+        # The site domain.  Does have www here, if it uses it.  But it
+        # doesn't matter too much anymore.
+        return 'tokra.fandomnet.com'
 
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
@@ -71,50 +72,15 @@ class SVUFictionComAdapter(BaseSiteAdapter):
     def getSiteURLPattern(self):
         return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
 
-    ## Login seems to be reasonably standard across eFiction sites.
-    def needToLoginCheck(self, data):
-        if 'Registered Users Only' in data \
-                or 'There is no such account on our website' in data \
-                or "That password doesn't match the one in our database" in data:
-            return True
-        else:
-            return False
-        
-    def performLogin(self, url):
-        params = {}
-
-        if self.password:
-            params['penname'] = self.username
-            params['password'] = self.password
-        else:
-            params['penname'] = self.getConfig("username")
-            params['password'] = self.getConfig("password")
-        params['cookiecheck'] = '1'
-        params['submit'] = 'Submit'
-    
-        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
-        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
-                                                              params['penname']))
-    
-        d = self._fetchUrl(loginUrl, params)
-    
-        if "Member Account" not in d : #Member Account
-            logger.info("Failed to login to URL %s as %s" % (loginUrl,
-                                                              params['penname']))
-            raise exceptions.FailedToLogin(url,params['penname'])
-            return False
-        else:
-            return True
-
     ## Getting the chapter list and the meta data, plus 'is adult' checking.
     def extractChapterUrlsAndMetadata(self):
-    
+
         if self.is_adult or self.getConfig("is_adult"):
             # Weirdly, different sites use different warning numbers.
             # If the title search below fails, there's a good chance
             # you need a different number.  print data at that point
             # and see what the 'click here to continue' url says.
-            addurl = "&ageconsent=ok&warning=6"
+            addurl = "&ageconsent=ok&warning=3"
         else:
             addurl=""
 
@@ -130,12 +96,7 @@ class SVUFictionComAdapter(BaseSiteAdapter):
                 raise exceptions.StoryDoesNotExist(self.url)
             else:
                 raise e
-                
-        if self.needToLoginCheck(data):
-            # need to log in for this one.
-            self.performLogin(url)
-            data = self._fetchUrl(url)
-                
+
         m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
         if m != None:
             if self.is_adult or self.getConfig("is_adult"):
@@ -163,26 +124,24 @@ class SVUFictionComAdapter(BaseSiteAdapter):
             
         # use BeautifulSoup HTML parser to make everything easier to find.
         soup = bs.BeautifulSoup(data)
-        # print data
+        #print data
 
         # Now go hunting for all the meta data and the chapter list.
         
         ## Title
-        pt = soup.find('div', {'class' : 'title'})
-        a = pt.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        if a.text != "":
-            self.story.setMetadata('title',a.text)
-        else:
-            self.story.setMetadata('title','Banner')
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
         
         # Find authorid and URL from... author url.
-        a = pt.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
         self.story.setMetadata('authorId',a['href'].split('=')[1])
         self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
         self.story.setMetadata('author',a.string)
 
-        rating=pt.text.split('[')[1].split(']')[0]
-        self.story.setMetadata('rating', rating)
+        # Rating
+        rate = stripHTML(soup.find('div',{'id':'pagetitle'}))
+        rate = rate[rate.rindex('[')+1:rate.rindex(']')]
+        self.story.setMetadata('rating', rate)
 
         # Find the chapters:
         for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
@@ -194,45 +153,36 @@ class SVUFictionComAdapter(BaseSiteAdapter):
         # eFiction sites don't help us out a lot with their meta data
         # formating, so it's a little ugly.
 
-        # utility method
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
+        metadiv = soup.find('div',{'class':'content'})
+        smalldiv = metadiv.find('div',{'class':'small'})
 
-        bottom=soup.find('div', {'class' : 'bottom'}).text
-        self.story.setMetadata('numWords', bottom.split('Word count: ')[1].split(' Read:')[0])
-        self.story.setMetadata('datePublished', makeDate(bottom.split('Published: ')[1].split(' Updated:')[0], self.dateformat))
-        self.story.setMetadata('dateUpdated', makeDate(bottom.split('Updated: ')[1], self.dateformat))
+        # tokra categories -> genre
+        # categories will be filled from ini.
+        genres = smalldiv.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+        for genre in genres:
+            self.story.addToList('genre',genre.string)
+        
+        chars = smalldiv.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+        for char in chars:
+            self.story.addToList('characters',char.string)
 
-        
-        content=soup.find('div', {'class' : 'content'})
-        value=content.find('h1').nextSibling
-        svalue = ""
-        while not defaultGetattr(value,'class') == 'smaller':
-            svalue += str(value)
-            value = value.nextSibling
-        self.setDescription(url,svalue)
-        
-        status = content.text.split('Completed: ')[1]
-        if 'Yes' in status:
+        metatext = stripHTML(smalldiv)
+
+        if 'Completed: Yes' in metatext:
             self.story.setMetadata('status', 'Completed')
         else:
             self.story.setMetadata('status', 'In-Progress')
-        
-        cats = content.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-        for cat in cats:
-            self.story.addToList('category',cat.string)
-            
-        chars = content.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-        for char in chars:
-            self.story.addToList('characters',char.string)
-            
-        genres = content.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
-        for genre in genres:
-            self.story.addToList('genre',genre.string)
-            
+
+        wordstart=metatext.rindex('Word count:')+12
+        words = metatext[wordstart:metatext.index(' ',wordstart)]
+        self.story.setMetadata('numWords', words)
+
+        datesdiv = soup.find('div',{'class':'bottom'})
+        dates = stripHTML(datesdiv).split()
+        # Published: 04/26/2011 Updated: 03/06/2013
+        self.story.setMetadata('datePublished', makeDate(dates[1], self.dateformat))
+        self.story.setMetadata('dateUpdated', makeDate(dates[3], self.dateformat))
+
         try:
             # Find Series name from series URL.
             a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
@@ -249,6 +199,7 @@ class SVUFictionComAdapter(BaseSiteAdapter):
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             
@@ -256,17 +207,30 @@ class SVUFictionComAdapter(BaseSiteAdapter):
             # I find it hard to care if the series parsing fails
             pass
             
+        # remove 'small' leaving only summary.
+        smalldiv.extract()
+        self.setDescription(url,metadiv)
+        
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
         logger.debug('Getting chapter text from: %s' % url)
 
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        soup = bs.BeautifulSoup(self._fetchUrl(url))
         
-        div = soup.find('div', {'id' : 'story'})
+        div = soup.find('div', {'class' : 'content'})
 
         if None == div:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+
+        # remove some decorations while keeping notes.
+        remove = div.find('div', {'id' : 'pagetitle'})
+        remove.extract()
+    
+        for remove in div.findAll('div', {'class' : 'right'}):
+            remove.extract()
+    
+        for remove in div.findAll('div', {'class' : 'left'}):
+            remove.extract()
     
         return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_tthfanficorg.py

@@ -154,26 +154,6 @@ class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter):
         self.story.setMetadata('author',stripHTML(a))
         authorurl = 'http://'+self.host+a['href']
 
-        ainfo = soup.find('a', href='/StoryInfo-%s-1'%self.story.getMetadata('storyId'))
-        if ainfo != None: # indicates multiple authors/contributors.
-            try:
-                # going to pull part of the meta data from author list page.
-                infourl = 'http://'+self.host+ainfo['href']
-                logger.debug("**StoryInfo** URL: "+infourl)
-                infodata = self._fetchUrl(infourl)
-                infosoup = bs.BeautifulSoup(infodata)
-
-                for a in infosoup.findAll('a',href=re.compile(r"^/Author-\d+")):
-                    self.story.addToList('authorId',a['href'].split('/')[1].split('-')[1])
-                    self.story.addToList('authorUrl','http://'+self.host+a['href'].replace("/Author-","/AuthorStories-"))
-                    self.story.addToList('author',stripHTML(a))                
-                                            
-            except urllib2.HTTPError, e:
-                if e.code == 404:
-                    raise exceptions.StoryDoesNotExist(url)
-                else:
-                    raise e
-    
         try:
             # going to pull part of the meta data from *primary* author list page.
             logger.debug("**AUTHOR** URL: "+authorurl)
@@ -198,6 +178,53 @@ class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter):
         #self.story.setMetadata('description',stripHTML(storydiv.find('div',{'class':'storydesc'})))
         self.story.setMetadata('title',stripHTML(storydiv.find('a',{'class':'storylink'})))
 
+        ainfo = soup.find('a', href='/StoryInfo-%s-1'%self.story.getMetadata('storyId'))
+        if ainfo != None: # indicates multiple authors/contributors.
+            try:
+                # going to pull part of the meta data from author list page.
+                infourl = 'http://'+self.host+ainfo['href']
+                logger.debug("**StoryInfo** URL: "+infourl)
+                infodata = self._fetchUrl(infourl)
+                infosoup = bs.BeautifulSoup(infodata)
+
+                # for a in infosoup.findAll('a',href=re.compile(r"^/Author-\d+")):
+                #     self.story.addToList('authorId',a['href'].split('/')[1].split('-')[1])
+                #     self.story.addToList('authorUrl','http://'+self.host+a['href'].replace("/Author-","/AuthorStories-"))
+                #     self.story.addToList('author',stripHTML(a))
+
+                # second verticaltable is the chapter list.
+                table = infosoup.findAll('table',{'class':'verticaltable'})[1]
+                for a in table.findAll('a',href=re.compile(r"^/Story-"+self.story.getMetadata('storyId'))):
+                    autha = a.findNext('a',href=re.compile(r"^/Author-\d+"))
+                    self.story.addToList('authorId',autha['href'].split('/')[1].split('-')[1])
+                    self.story.addToList('authorUrl','http://'+self.host+autha['href'].replace("/Author-","/AuthorStories-"))
+                    self.story.addToList('author',stripHTML(autha))
+                    # include leading number to match 1. ... 2. ...
+                    self.chapterUrls.append(("%d. %s by %s"%(len(self.chapterUrls)+1,
+                                                             stripHTML(a),
+                                                             stripHTML(autha)),'http://'+self.host+a['href']))
+                                            
+            except urllib2.HTTPError, e:
+                if e.code == 404:
+                    raise exceptions.StoryDoesNotExist(url)
+                else:
+                    raise e
+        else: # single author:
+            # Find the chapter selector 
+            select = soup.find('select', { 'name' : 'chapnav' } )
+        	 
+            if select is None:
+        	   # no selector found, so it's a one-chapter story.
+        	   self.chapterUrls.append((self.story.getMetadata('title'),url))
+            else:
+                allOptions = select.findAll('option')
+                for o in allOptions:
+                    url = "http://"+self.host+o['value']
+                    # just in case there's tags, like <i> in chapter titles.
+                    self.chapterUrls.append((stripHTML(o),url))
+    
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
         verticaltable = soup.find('table', {'class':'verticaltable'})
 
         BtVS = True
@@ -235,29 +262,12 @@ class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter):
         if BtVS:
             self.story.addToList('category','Buffy: The Vampire Slayer')
             
-        # Find the chapter selector 
-        select = soup.find('select', { 'name' : 'chapnav' } )
-    	 
-        if select is None:
-    	   # no selector found, so it's a one-chapter story.
-    	   self.chapterUrls.append((self.story.getMetadata('title'),url))
-        else:
-            allOptions = select.findAll('option')
-            for o in allOptions:
-                url = "http://"+self.host+o['value']
-                # just in case there's tags, like <i> in chapter titles.
-                self.chapterUrls.append((stripHTML(o),url))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
         pseries = soup.find('p', {'style':'margin-top:0px'})
         m = re.match('This story is No\. (?P<num>\d+) in the series &quot;(?P<series>.+)&quot;\.',
                      pseries.text)
         if m:
             self.setSeries(m.group('series'),m.group('num'))
-        
-        return
-
+            self.story.setMetadata('seriesUrl',"http://"+self.host+pseries.find('a')['href'])
 
     def getChapterText(self, url):
         logger.debug('Getting chapter text from: %s' % url)

fanficdownloader/adapters/adapter_twilightarchivescom.py

@@ -163,6 +163,7 @@ class TwilightArchivesComAdapter(BaseSiteAdapter):
                 for a in storyas:
                     if a['href'] == ('/read/'+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
                     

fanficdownloader/adapters/adapter_twilightednet.py

@@ -217,6 +217,7 @@ class TwilightedNetSiteAdapter(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
         

fanficdownloader/adapters/adapter_twiwritenet.py

@@ -246,6 +246,7 @@ class TwiwriteNetSiteAdapter(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             

fanficdownloader/adapters/adapter_walkingtheplankorg.py

@@ -119,7 +119,7 @@ class WalkingThePlankOrgAdapter(BaseSiteAdapter):
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
         self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/archive/'+a['href'])
         self.story.setMetadata('author',a.string)
 
         # Find the chapters:
@@ -206,6 +206,7 @@ class WalkingThePlankOrgAdapter(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             

fanficdownloader/adapters/adapter_whoficcom.py

@@ -204,6 +204,7 @@ class WhoficComSiteAdapter(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             

fanficdownloader/adapters/adapter_wizardtalesnet.py

@@ -277,6 +277,7 @@ class WizardTalesNetAdapter(BaseSiteAdapter):
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             

fanficdownloader/adapters/adapter_wolverineandroguecom.py

@@ -194,6 +194,7 @@ class WolverineAndRogueComAdapter(BaseSiteAdapter):
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                         self.setSeries(series_name, i)
+                        self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
             

fanficdownloader/adapters/adapter_wraithbaitcom.py

@@ -126,8 +126,13 @@ class WraithBaitComAdapter(BaseSiteAdapter):
 
         # Find the chapters:
         for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # include author on chapters if multiple authors.
+            if len(alist) > 1:
+                add = " by %s"%stripHTML(chapter.findNext('a', href=re.compile(r"viewuser.php\?uid=\d+")))
+            else:
+                add = ""
             # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+            self.chapterUrls.append((stripHTML(chapter)+add,'http://'+self.host+'/'+chapter['href']+addurl))
 
         self.story.setMetadata('numChapters',len(self.chapterUrls))
 
@@ -196,6 +201,7 @@ class WraithBaitComAdapter(BaseSiteAdapter):
             for a in storyas:
                 if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
                     self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
                     break
                 i+=1
             

fanficdownloader/adapters/base_adapter.py

@@ -197,7 +197,8 @@ class BaseSiteAdapter(Configurable):
             for index, (title,url) in enumerate(self.chapterUrls):
                 if (self.chapterFirst!=None and index < self.chapterFirst) or \
                         (self.chapterLast!=None and index > self.chapterLast):
-                    self.story.addChapter(removeEntities(title),
+                    self.story.addChapter(url,
+                                          removeEntities(title),
                                           None)
                 else:
                     if self.oldchapters and index < len(self.oldchapters):
@@ -206,7 +207,8 @@ class BaseSiteAdapter(Configurable):
                                                  partial(cachedfetch,self._fetchUrlRaw,self.oldimgs))
                     else:
                         data = self.getChapterText(url)
-                    self.story.addChapter(removeEntities(title),
+                    self.story.addChapter(url,
+                                          removeEntities(title),
                                           removeEntities(data))
             self.storyDone = True
             
@@ -313,6 +315,8 @@ class BaseSiteAdapter(Configurable):
             fetch=self._fetchUrlRaw
 
         acceptable_attributes = ['href','name','class','id']
+        if self.getConfig("keep_style_attr"):
+            acceptable_attributes.append('style')
         #print("include_images:"+self.getConfig('include_images'))
         if self.getConfig('include_images'):
             acceptable_attributes.extend(('src','alt','longdesc'))

fanficdownloader/configurable.py

@@ -15,7 +15,7 @@
 # limitations under the License.
 #
 
-import ConfigParser
+import ConfigParser, re
 
 # All of the writers(epub,html,txt) and adapters(ffnet,twlt,etc)
 # inherit from Configurable.  The config file(s) uses ini format:
@@ -37,10 +37,20 @@ class Configuration(ConfigParser.SafeConfigParser):
     def __init__(self, site, fileform):
         ConfigParser.SafeConfigParser.__init__(self)
         self.sectionslist = ['defaults']
-        self.addConfigSection(site)
+
+        if site.startswith("www."):
+            sitewith = site
+            sitewithout = site.replace("www.","")
+        else:
+            sitewith = "www."+site
+            sitewithout = site
+        
+        self.addConfigSection(sitewith)
+        self.addConfigSection(sitewithout)
         if fileform:
             self.addConfigSection(fileform)
-            self.addConfigSection(site+":"+fileform)
+            self.addConfigSection(sitewith+":"+fileform)
+            self.addConfigSection(sitewithout+":"+fileform)
         self.addConfigSection("overrides")
         
         self.validEntries = [
@@ -50,6 +60,7 @@ class Configuration(ConfigParser.SafeConfigParser):
             'characters',
             'ships',
             'series',
+            'seriesUrl',
             'status',
             'datePublished',
             'dateUpdated',
@@ -75,6 +86,7 @@ class Configuration(ConfigParser.SafeConfigParser):
             'langcode',
             'output_css',
             'authorHTML',
+            'seriesHTML',
             'lastupdate'
             ]
 
@@ -87,8 +99,12 @@ class Configuration(ConfigParser.SafeConfigParser):
     def getValidMetaList(self):
         return self.validEntries + self.getConfigList("extra_valid_entries")
 
+    # used by adapters & writers, non-convention naming style
     def hasConfig(self, key):
-        for section in self.sectionslist:
+        return self.has_config(self.sectionslist, key)
+
+    def has_config(self, sections, key):
+        for section in sections:
             try:
                 self.get(section,key)
                 #print("found %s in section [%s]"%(key,section))
@@ -102,10 +118,14 @@ class Configuration(ConfigParser.SafeConfigParser):
                     pass
 
         return False
-        
+
+    # used by adapters & writers, non-convention naming style
     def getConfig(self, key, default=""):
+        return self.get_config(self.sectionslist,key,default)
+    
+    def get_config(self, sections, key, default=""):        
         val = default
-        for section in self.sectionslist:
+        for section in sections:
             try:
                 val = self.get(section,key)
                 if val and val.lower() == "false":
@@ -115,7 +135,7 @@ class Configuration(ConfigParser.SafeConfigParser):
             except (ConfigParser.NoOptionError, ConfigParser.NoSectionError), e:
                 pass
 
-        for section in self.sectionslist[::-1]:
+        for section in sections[::-1]:
             # 'martian smiley' [::-1] reverses list by slicing whole list with -1 step.
             try:
                 val = val + self.get(section,"add_to_"+key)
@@ -126,11 +146,15 @@ class Configuration(ConfigParser.SafeConfigParser):
         return val
 
     # split and strip each.
-    def getConfigList(self, key):
-        vlist = self.getConfig(key).split(',')
-        vlist = filter( lambda x : x !='', [ v.strip() for v in vlist ])
+    def get_config_list(self, sections, key):
+        vlist = re.split(r'(?<!\\),',self.get_config(sections,key)) # don't split on \,
+        vlist = filter( lambda x : x !='', [ v.strip().replace('\,',',') for v in vlist ])
         #print "vlist("+key+"):"+str(vlist)
         return vlist        
+    
+    # used by adapters & writers, non-convention naming style
+    def getConfigList(self, key):
+        return self.get_config_list(self.sectionslist, key)
 
 # extended by adapter, writer and story for ease of calling configuration.
 class Configurable(object):
@@ -145,11 +169,19 @@ class Configurable(object):
         return self.configuration.getValidMetaList()
     
     def hasConfig(self, key):
-        return self.configuration.hasConfig(key)
+        return self.configuration.hasConfig(key)        
         
+    def has_config(self, sections, key):
+        return self.configuration.has_config(sections, key)
+
     def getConfig(self, key, default=""):
         return self.configuration.getConfig(key,default)
 
+    def get_config(self, sections, key, default=""):
+        return self.configuration.get_config(sections,key,default)
+
     def getConfigList(self, key):
         return self.configuration.getConfigList(key)
-        
+
+    def get_config_list(self, sections, key):
+        return self.configuration.get_config_list(sections,key)

fanficdownloader/epubutils.py

@@ -130,6 +130,9 @@ def get_update_data(inputio,
                         h2 = soup.find('h2')
                         if h2:
                             h2.extract()
+
+                        for skip in soup.findAll(attrs={'class':'skip_on_ffdl_update'}):
+                            skip.extract()
                             
                         soups.append(soup)
                         

fanficdownloader/exceptions.py

@@ -68,3 +68,10 @@ class UnknownSite(Exception):
     def __str__(self):
         return "Unknown Site(%s).  Supported sites: (%s)" % (self.url, ", ".join(self.supported_sites_list))
 
+class FailedToWriteOutput(Exception):
+    def __init__(self,error):
+        self.error=error
+
+    def __str__(self):
+        return self.error
+

fanficdownloader/geturls.py

@@ -25,7 +25,7 @@ from gziphttp import GZipProcessor
 import adapters
 from configurable import Configuration
 
-def get_urls_from_page(url,configuration=None):
+def get_urls_from_page(url,configuration=None,normalize=False):
 
     if not configuration:
         configuration = Configuration("test1.com","EPUB")
@@ -54,11 +54,11 @@ def get_urls_from_page(url,configuration=None):
         opener = u2.build_opener(u2.HTTPCookieProcessor(),GZipProcessor())
         data = opener.open(url).read()
 
-    return get_urls_from_html(data,url)
+    return get_urls_from_html(data,url,configuration,normalize)
 
-def get_urls_from_html(data,url=None,configuration=None):
+def get_urls_from_html(data,url=None,configuration=None,normalize=False):
 
-    normalized = set() # normalized url
+    normalized = [] # normalized url
     retlist = [] # orig urls.
     
     if not configuration:
@@ -82,16 +82,19 @@ def get_urls_from_html(data,url=None,configuration=None):
                 href = href.replace('&index=1','')
                 adapter = adapters.getAdapter(configuration,href)
                 if adapter.story.getMetadata('storyUrl') not in normalized:
-                    normalized.add(adapter.story.getMetadata('storyUrl'))
+                    normalized.append(adapter.story.getMetadata('storyUrl'))
                     retlist.append(href)
             except:
                 pass
 
-    return retlist
+    if normalize:
+        return normalized
+    else:
+        return retlist
 
-def get_urls_from_text(data,configuration=None):
+def get_urls_from_text(data,configuration=None,normalize=False):
 
-    normalized = set() # normalized url
+    normalized = [] # normalized url
     retlist = [] # orig urls.
     
     if not configuration:
@@ -102,19 +105,22 @@ def get_urls_from_text(data,configuration=None):
         # 'are you old enough' links, and 'Report This' links.
         # The 'normalized' set prevents duplicates.
         if 'story.php' in href:
-            m = re.search(r"(?P<sid>(view)?story\.php\?(sid|psid|no|story|stid)=\d+)",a['href'])
+            m = re.search(r"(?P<sid>(view)?story\.php\?(sid|psid|no|story|stid)=\d+)",href)
             if m != None:
-                href = form_url(None,m.group('sid'))
+                href = form_url(href,m.group('sid'))
         try:
             href = href.replace('&index=1','')
             adapter = adapters.getAdapter(configuration,href)
             if adapter.story.getMetadata('storyUrl') not in normalized:
-                normalized.add(adapter.story.getMetadata('storyUrl'))
+                normalized.append(adapter.story.getMetadata('storyUrl'))
                 retlist.append(href)
         except:
             pass
 
-    return retlist
+    if normalize:
+        return normalized
+    else:
+        return retlist
 
 def form_url(parenturl,url):
      url = url.strip() # ran across an image with a space in the

fanficdownloader/html.py

@@ -17,7 +17,7 @@ class HtmlProcessor:
     self.unfill = unfill
     html = self._ProcessRawHtml(html)
     self._soup = BeautifulSoup(html)
-    if self._soup.title:
+    if self._soup.title.contents:
       self.title = self._soup.title.contents[0]
     else:
       self.title = None

fanficdownloader/mobi.py

@@ -8,6 +8,8 @@ import time
 import random
 import logging
 
+logger = logging.getLogger(__name__)
+
 from html import HtmlProcessor
 
 # http://wiki.mobileread.com/wiki/MOBI
@@ -124,8 +126,8 @@ class Converter:
       tmp = self.MakeOneHTML(html_strs)
       self._ConvertStringToFile(tmp, out_file)
     except Exception, e:
-      logging.error('Error %s', e)
-      logging.debug('Details: %s' % html_strs)
+      logger.error('Error %s', e)
+      #logger.debug('Details: %s' % html_strs)
 
   def _ConvertStringToFile(self, html_data, out):
     html = HtmlProcessor(html_data)

fanficdownloader/story.py

@@ -280,10 +280,10 @@ class Story(Configurable):
         for line in replace.splitlines():
             (metakeys,regexp,replacement,condkey,condregexp)=(None,None,None,None,None)
             if "&&" in line:
-                (line,conditional) = map( lambda x: x.strip(), line.split("&&") )
-                (condkey,condregexp) = map( lambda x: x.strip(), conditional.split("=>") )
+                (line,conditional) = line.split("&&")
+                (condkey,condregexp) = conditional.split("=>")
             if "=>" in line:
-                parts = map( lambda x: x.strip(), line.split("=>") )
+                parts = line.split("=>")
                 if len(parts) > 2:
                     metakeys = map( lambda x: x.strip(), parts[0].split(",") )
                     (regexp,replacement)=parts[1:]
@@ -294,9 +294,13 @@ class Story(Configurable):
                 regexp = re.compile(regexp)
                 if condregexp:
                     condregexp = re.compile(condregexp)
+                # A way to explicitly include spaces in the
+                # replacement string.  The .ini parser eats any
+                # trailing spaces.
+                replacement=replacement.replace('\s',' ') 
                 self.replacements.append([metakeys,regexp,replacement,condkey,condregexp])
     
-    def doReplacments(self,value,key):
+    def doReplacements(self,value,key):
         for (metakeys,regexp,replacement,condkey,condregexp) in self.replacements:
             if (metakeys == None or key in metakeys) \
                     and isinstance(value,basestring) \
@@ -336,7 +340,7 @@ class Story(Configurable):
                     value = value.strftime(self.getConfig(key+"_format","%Y-%m-%d"))
 
             if doreplacements:
-                value=self.doReplacments(value,key)
+                value=self.doReplacements(value,key)
             if removeallentities and value != None:
                 return removeAllEntities(value)
             else:
@@ -352,7 +356,7 @@ class Story(Configurable):
         allmetadata = {}
         
         # special handling for authors/authorUrls
-        authlinkhtml="<a class='authorlink' href='%s'>%s</a>"
+        linkhtml="<a class='%slink' href='%s'>%s</a>"
         if self.isList('author'): # more than one author, assume multiple authorUrl too.
             htmllist=[]
             for i, v in enumerate(self.getList('author')):
@@ -360,18 +364,24 @@ class Story(Configurable):
                 auth = v
                 # make sure doreplacements & removeallentities are honored.
                 if doreplacements:
-                    aurl=self.doReplacments(aurl,'authorUrl')
-                    auth=self.doReplacments(auth,'author')
+                    aurl=self.doReplacements(aurl,'authorUrl')
+                    auth=self.doReplacements(auth,'author')
                 if removeallentities:
                     aurl=removeAllEntities(aurl)
                     auth=removeAllEntities(auth)
                 
-                htmllist.append(authlinkhtml%(aurl,auth))
+                htmllist.append(linkhtml%('author',aurl,auth))
             self.setMetadata('authorHTML',', '.join(htmllist))
         else:
-           self.setMetadata('authorHTML',authlinkhtml%(self.getMetadata('authorUrl', removeallentities, doreplacements),
-                                                       self.getMetadata('author', removeallentities, doreplacements)))
+            self.setMetadata('authorHTML',linkhtml%('author',self.getMetadata('authorUrl', removeallentities, doreplacements),
+                                                    self.getMetadata('author', removeallentities, doreplacements)))
 
+        if self.getMetadataRaw('seriesUrl') != None:
+            self.setMetadata('seriesHTML',linkhtml%('series',self.getMetadata('seriesUrl', removeallentities, doreplacements),
+                                                    self.getMetadata('series', removeallentities, doreplacements)))
+        elif self.getMetadataRaw('series') != None:
+            self.setMetadata('seriesHTML',self.getMetadataRaw('series'))
+            
         for k in self.getValidMetaList():
             if self.isList(k) and keeplists:
                 allmetadata[k] = self.getList(k, removeallentities, doreplacements)
@@ -415,18 +425,20 @@ class Story(Configurable):
         # includelist prevents infinite recursion of include_in_'s
         if self.hasConfig("include_in_"+listname) and listname not in includelist:
             for k in self.getConfigList("include_in_"+listname):
-                retlist.extend(self.getList(k,removeallentities,doreplacements,includelist=includelist+[listname]))
+                retlist.extend(self.getList(k,removeallentities=False,
+                                            doreplacements=doreplacements,includelist=includelist+[listname]))
         else:
         
             if not self.isList(listname):
-                retlist = [self.getMetadata(listname,removeallentities, doreplacements)]
+                retlist = [self.getMetadata(listname,removeallentities=False,
+                                            doreplacements=doreplacements)]
             else:
                 retlist = self.getMetadataRaw(listname)
 
         if retlist:
             if doreplacements:
                 retlist = filter( lambda x : x!=None and x!='' ,
-                                  map(partial(self.doReplacments,key=listname),retlist) )
+                                  map(partial(self.doReplacements,key=listname),retlist) )
             if removeallentities:
                 retlist = filter( lambda x : x!=None and x!='' ,
                                   map(removeAllEntities,retlist) )
@@ -462,11 +474,11 @@ class Story(Configurable):
 
         return list(subjectset | set(self.getConfigList("extratags")))
             
-    def addChapter(self, title, html):
+    def addChapter(self, url, title, html):
         if self.getConfig('strip_chapter_numbers') and \
                 self.getConfig('chapter_title_strip_pattern'):
             title = re.sub(self.getConfig('chapter_title_strip_pattern'),"",title)
-        self.chapters.append( (title,html) )
+        self.chapters.append( (url,title,html) )
 
     def getChapters(self,fortoc=False):
         "Chapters will be tuples of (title,html)"
@@ -476,8 +488,10 @@ class Story(Configurable):
                 (self.getConfig('add_chapter_numbers') == "true" \
                      or (self.getConfig('add_chapter_numbers') == "toconly" and fortoc)) \
                      and self.getConfig('chapter_title_add_pattern'):
-            for index, (title,html) in enumerate(self.chapters):
-                retval.append( (string.Template(self.getConfig('chapter_title_add_pattern')).substitute({'index':index+1,'title':title}),html) ) 
+            for index, (url,title,html) in enumerate(self.chapters):
+                retval.append( (url,
+                                string.Template(self.getConfig('chapter_title_add_pattern')).substitute({'index':index+1,'title':title}),
+                                html) )
         else:
             retval = self.chapters
             

fanficdownloader/writers/base_writer.py

@@ -177,11 +177,12 @@ class BaseStoryWriter(Configurable):
             
             self._write(out,START.substitute(self.story.getAllMetadata()))
 
-            for index, (title,html) in enumerate(self.story.getChapters(fortoc=True)):
+            for index, (url,title,html) in enumerate(self.story.getChapters(fortoc=True)):
                 if html:
                     self._write(out,ENTRY.substitute({'chapter':title,
                                                       'number':index+1,
-                                                      'index':"%04d"%(index+1)}))
+                                                      'index':"%04d"%(index+1),
+                                                      'url':url}))
 
             self._write(out,END.substitute(self.story.getAllMetadata()))
 

fanficdownloader/writers/writer_epub.py

@@ -501,7 +501,7 @@ div { margin: 0pt; padding: 0pt; }
             items.append(("log_page","OEBPS/log_page.xhtml","application/xhtml+xml","Update Log"))
             itemrefs.append("log_page")
             
-        for index, (title,html) in enumerate(self.story.getChapters(fortoc=True)):
+        for index, (url,title,html) in enumerate(self.story.getChapters(fortoc=True)):
             if html:
                 i=index+1
                 items.append(("file%04d"%i,
@@ -649,10 +649,10 @@ div { margin: 0pt; padding: 0pt; }
         else:
             CHAPTER_END = self.EPUB_CHAPTER_END
         
-        for index, (title,html) in enumerate(self.story.getChapters()):
+        for index, (url,title,html) in enumerate(self.story.getChapters()):
             if html:
                 logger.debug('Writing chapter text for: %s' % title)
-                vals={'chapter':title, 'index':"%04d"%(index+1), 'number':index+1}
+                vals={'url':url, 'chapter':title, 'index':"%04d"%(index+1), 'number':index+1}
                 fullhtml = CHAPTER_START.substitute(vals) + html + CHAPTER_END.substitute(vals)
                 # ffnet(& maybe others) gives the whole chapter text
                 # as one line.  This causes problems for nook(at

fanficdownloader/writers/writer_html.py

@@ -128,10 +128,10 @@ ${output_css}
         else:
             CHAPTER_END = self.HTML_CHAPTER_END
         
-        for index, (title,html) in enumerate(self.story.getChapters()):
+        for index, (url,title,html) in enumerate(self.story.getChapters()):
             if html:
                 logging.debug('Writing chapter text for: %s' % title)
-                vals={'chapter':title, 'index':"%04d"%(index+1), 'number':index+1}
+                vals={'url':url, 'chapter':title, 'index':"%04d"%(index+1), 'number':index+1}
                 self._write(out,CHAPTER_START.substitute(vals))
                 self._write(out,html)
                 self._write(out,CHAPTER_END.substitute(vals))

fanficdownloader/writers/writer_mobi.py

@@ -22,6 +22,9 @@ import StringIO
 from base_writer import *
 from ..htmlcleanup import stripHTML
 from ..mobi import Converter
+from ..exceptions import FailedToWriteOutput
+
+logger = logging.getLogger(__name__)
 
 class MobiWriter(BaseStoryWriter):
 
@@ -158,10 +161,10 @@ ${value}<br />
         else:
             CHAPTER_END = self.MOBI_CHAPTER_END
         
-        for index, (title,html) in enumerate(self.story.getChapters()):
+        for index, (url,title,html) in enumerate(self.story.getChapters()):
             if html:
-                logging.debug('Writing chapter text for: %s' % title)
-                vals={'chapter':title, 'index':"%04d"%(index+1), 'number':index+1}
+                logger.debug('Writing chapter text for: %s' % title)
+                vals={'url':url, 'chapter':title, 'index':"%04d"%(index+1), 'number':index+1}
                 fullhtml = CHAPTER_START.substitute(vals) + html + CHAPTER_END.substitute(vals)
                 # ffnet(& maybe others) gives the whole chapter text
                 # as one line.  This causes problems for nook(at
@@ -175,6 +178,8 @@ ${value}<br />
                       author=self.getMetadata('author'),
                       publisher=self.getMetadata('site'))
         mobidata = c.ConvertStrings(files)
+        if len(mobidata) < 1:
+            raise FailedToWriteOutput("Zero length mobi output")
         out.write(mobidata)
         
         del files

fanficdownloader/writers/writer_txt.py

@@ -154,10 +154,10 @@ End file.
         else:
             CHAPTER_END = self.TEXT_CHAPTER_END
         
-        for index, (title,html) in enumerate(self.story.getChapters()):
+        for index, (url, title,html) in enumerate(self.story.getChapters()):
             if html:
                 logging.debug('Writing chapter text for: %s' % title)
-                vals={'chapter':title, 'index':"%04d"%(index+1), 'number':index+1}
+                vals={'url':url, 'chapter':title, 'index':"%04d"%(index+1), 'number':index+1}
                 self._write(out,self.lineends(self.wraplines(removeAllEntities(CHAPTER_START.substitute(vals)))))
                 self._write(out,self.lineends(html2text(html,wrap_width=self.wrap_width)))
                 self._write(out,self.lineends(self.wraplines(removeAllEntities(CHAPTER_END.substitute(vals)))))

index.html

@@ -57,7 +57,8 @@
 	  <h3>Changes:</h3>
 	  <p>
 	    <ul>
-	      <li>Change how fimfiction.net story descriptions are collected.</li>
+	      <li>New Site: nickandgreg.net - Thanks, Ida.</li>
+	      <li>Add Read & Review site specific counts to dramione.org and grangerenchanted.com</li>
 	    </ul>
 	  </p>
 	  <p>
@@ -68,7 +69,7 @@
 	    If you have any problems with this application, please
 	    report them in
 	    the <a href="http://groups.google.com/group/fanfic-downloader">FanFictionDownLoader Google Group</a>.  The
-	    <a href="http://4-4-45.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
+	    <a href="http://4-4-58.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
 	  </p>
 	  <div id='error'>
 	    {{ error_message }}
@@ -378,11 +379,6 @@
 	    Use the URL of the story's first chapter, such as
 	    <br /><a href="http://onedirectionfanfiction.com/viewstory.php?sid=1234">http://onedirectionfanfiction.com/viewstory.php?sid=1234</a>
 	  </dd>
-	  <dt>www.prisonbreakfic.net</dt>
-	  <dd>
-	    Use the URL of the story's first chapter, such as
-	    <br /><a href="http://www.prisonbreakfic.net/viewstory.php?sid=1234">http://www.prisonbreakfic.net/viewstory.php?sid=1234</a>
-	  </dd>
 	  <dt>www.storiesofarda.com</dt>
 	  <dd>
 	    Use the URL of the story's chapter list, such as
@@ -471,11 +467,6 @@
 	    Use the URL of the story's first chapter, such as
 	    <br /><a href="http://www.hpfanficarchive.com/stories/viewstory.php?sid=1234">http://www.hpfanficarchive.com/stories/viewstory.php?sid=1234</a>
 	  </dd>
-	  <dt>svufiction.com</dt>
-	  <dd>
-	    Use the URL of the story's first chapter, such as
-	    <br /><a href="http://svufiction.com/viewstory.php?sid=1234">http://svufiction.com/viewstory.php?sid=1234</a>
-	  </dd>
 	  <dt>www.twilightarchives.com</dt>
 	  <dd>
 	    Use the URL of the story's first chapter, such as
@@ -552,11 +543,6 @@
 	    Use the URL of the story's chapter list, such as
 	    <br /><a href="http://www.indeath.net/blog/25-nightmare-in-death/">http://www.indeath.net/blog/25-nightmare-in-death/</a>
 	  </dd>
-	  <dt>www.jlaunlimited.com</dt>
-	  <dd>
-	    Use the URL of the story's first chapter, such as
-	    <br /><a href="http://www.jlaunlimited.com/eFiction1.1/viewstory.php?sid=1234">http://www.jlaunlimited.com/eFiction1.1/viewstory.php?sid=1234</a>
-	  </dd>
 	  <dt>www.efpfanfic.net</dt>
 	  <dd>
 	    Use the URL of any story chapter, such as
@@ -607,11 +593,31 @@
 	    Use the URL of the story's chapter list, such as
 	    <br /><a href="http://www.henneth-annun.net/stories/chapter.cfm?stid=1234">http://www.henneth-annun.net/stories/chapter.cfm?stid=1234</a>
 	  </dd>
-	  <dt>http://www.psychfic.com</dt>
+	  <dt>www.psychfic.com</dt>
 	  <dd>
 	    Use the URL of the story's chapter list, such as
 	    <br /><a href="http://www.psychfic.com/viewstory.php?sid=1234">http://www.psychfic.com/viewstory.php?sid=1234</a>
 	  </dd>
+	  <dt>tokra.fandomnet.com</dt>
+	  <dd>
+	    Use the URL of the story's chapter list, such as
+	    <br /><a href="http://tokra.fandomnet.com/viewstory.php?sid=1234">http://tokra.fandomnet.com/viewstory.php?sid=1234</a>
+	  </dd>
+	  <dt>asr3.slashzone.org</dt>
+	  <dd>
+	    Use the URL of the story's chapter list, such as
+	    <br /><a href="http://asr3.slashzone.org/archive/viewstory.php?sid=1234">http://asr3.slashzone.org/archive/viewstory.php?sid=1234</a>
+	  </dd>
+	  <dt>netraptor.org</dt>
+	  <dd>
+	    Use the URL of the story's chapter list, such as
+	    <br /><a href="http://netraptor.org/archive/viewstory.php?sid=1234">http://netraptor.org/archive/viewstory.php?sid=1234</a>
+	  </dd>
+	  <dt>nickandgreg.net</dt>
+	  <dd>
+	    Use the URL of the story's chapter list, such as
+	    <br /><a href="http://www.nickandgreg.net/desert_archive/viewstory.php?sid=1234">http://www.nickandgreg.net/desert_archive/viewstory.php?sid=1234</a>
+	  </dd>
 	</dl>
 
 	<p>