Bump versions, add more test cases to test1.com adapter.

For favs, reviews, etc from site specific metadata.

app.yaml

@@ -1,6 +1,6 @@
 # ffd-retief-hrd fanfictiondownloader
 application: fanfictiondownloader
-version: 4-4-34
+version: 4-4-47
 runtime: python27
 api_version: 1
 threadsafe: true

calibre-plugin/__init__.py

@@ -27,7 +27,7 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
     description         = 'UI plugin to download FanFiction stories from various sites.'
     supported_platforms = ['windows', 'osx', 'linux']
     author              = 'Jim Miller'
-    version             = (1, 6, 19)
+    version             = (1, 7, 14)
     minimum_calibre_version = (0, 8, 57)
 
     #: This field defines the GUI plugin class that contains all the code
@@ -80,11 +80,29 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
         if ac is not None:
             ac.apply_settings()
 
-# For testing, run from command line with this:
-# calibre-debug -e __init__.py
-# 
-if __name__ == '__main__':
-    from PyQt4.Qt import QApplication
-    from calibre.gui2.preferences import test_widget
-    app = QApplication([])
-    test_widget('Advanced', 'Plugins')
+    def cli_main(self,argv):
+        # I believe there's no performance hit loading these here when
+        # CLI--it would load everytime anyway.
+        from StringIO import StringIO
+        from calibre.library import db
+        from calibre_plugins.fanfictiondownloader_plugin.downloader import main as ffdl_main
+        from calibre_plugins.fanfictiondownloader_plugin.prefs import PrefsFacade
+        from calibre.utils.config import prefs as calibre_prefs
+        from optparse import OptionParser      
+
+        parser = OptionParser('%prog --run-plugin '+self.name+' -- [options] <storyurl>')
+        parser.add_option('--library-path', '--with-library', default=None, help=_('Path to the calibre library. Default is to use the path stored in the settings.'))
+        # parser.add_option('--dont-notify-gui', default=False, action='store_true',
+        #               help=_('Do not notify the running calibre GUI (if any) that the database has'
+        #                      ' changed. Use with care, as it can lead to database corruption!'))
+
+        pargs = [x for x in argv if x.startswith('--with-library') or x.startswith('--library-path')
+                 or not x.startswith('-')]
+        opts, args = parser.parse_args(pargs)
+
+        ffdl_prefs = PrefsFacade(db(path=opts.library_path,
+                                    read_only=True))
+        ffdl_main(argv[1:],
+                  parser=parser,
+                  passed_defaultsini=StringIO(get_resources("defaults.ini")),
+                  passed_personalini=StringIO(ffdl_prefs["personal.ini"]))

calibre-plugin/about.txt

@@ -4,7 +4,7 @@
 '<a href="http://www.mobileread.com/forums/showthread.php?t=134856">Reading List</a>',
 '<a href="http://www.mobileread.com/forums/showthread.php?t=126727">Extract ISBN</a>' and
 '<a href="http://www.mobileread.com/forums/showthread.php?t=134000">Count Pages</a>'
-plugins.  bbcodeutils code contributed by Pau Sanchez.</p>
+plugins.</p>
 
 <p>
 Calibre officially distributes plugins from the mobileread.com forum site.

calibre-plugin/common_utils.py

@@ -196,7 +196,7 @@ class ImageTitleLayout(QHBoxLayout):
     '''
     A reusable layout widget displaying an image followed by a title
     '''
-    def __init__(self, parent, icon_name, title):
+    def __init__(self, parent, icon_name, title, tooltip=None):
         QHBoxLayout.__init__(self)
         title_image_label = QLabel(parent)
         pixmap = get_pixmap(icon_name)
@@ -217,6 +217,9 @@ class ImageTitleLayout(QHBoxLayout):
         self.addWidget(shelf_label)
         self.insertStretch(-1)
 
+        if tooltip:
+            title_image_label.setToolTip(tooltip)
+            shelf_label.setToolTip(tooltip)
 
 class SizePersistedDialog(QDialog):
     '''

calibre-plugin/config.py

@@ -7,138 +7,108 @@ __license__   = 'GPL v3'
 __copyright__ = '2012, Jim Miller'
 __docformat__ = 'restructuredtext en'
 
-import traceback, copy
+import traceback, copy, threading
 from collections import OrderedDict
 
-from PyQt4.Qt import (QDialog, QWidget, QVBoxLayout, QHBoxLayout, QLabel, QLineEdit, QFont, QWidget,
-                      QTextEdit, QComboBox, QCheckBox, QPushButton, QTabWidget, QVariant, QScrollArea)
+from PyQt4.Qt import (QDialog, QWidget, QVBoxLayout, QHBoxLayout, QLabel,
+                      QLineEdit, QFont, QWidget, QTextEdit, QComboBox,
+                      QCheckBox, QPushButton, QTabWidget, QVariant, QScrollArea,
+                      QDialogButtonBox )
 
-from calibre.gui2 import dynamic, info_dialog
 from calibre.utils.config import JSONConfig
 from calibre.gui2.ui import get_gui
 
+from calibre_plugins.fanfictiondownloader_plugin.prefs import prefs
 from calibre_plugins.fanfictiondownloader_plugin.dialogs \
-    import (UPDATE, UPDATEALWAYS, OVERWRITE, collision_order)
-
-from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.adapters import getConfigSections
+    import (UPDATE, UPDATEALWAYS, OVERWRITE, collision_order, RejectListDialog,
+            EditTextDialog)
+    
+from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.adapters \
+    import (getConfigSections, getNormalStoryURL)
 
 from calibre_plugins.fanfictiondownloader_plugin.common_utils \
-    import ( get_library_uuid, KeyboardConfigDialog, PrefsViewerDialog )
+    import ( KeyboardConfigDialog, PrefsViewerDialog )
 
 from calibre.gui2.complete import MultiCompleteLineEdit
 
-PREFS_NAMESPACE = 'FanFictionDownLoaderPlugin'
-PREFS_KEY_SETTINGS = 'settings'
+class RejectURLList:
+    def __init__(self,prefs):
+        self.prefs = prefs
+        self.sync_lock = threading.RLock()
+        self.listcache = None
 
-# Set defaults used by all.  Library specific settings continue to
-# take from here.
-default_prefs = {}
-default_prefs['personal.ini'] = get_resources('plugin-example.ini')
-
-default_prefs['updatemeta'] = True
-default_prefs['updatecover'] = False
-default_prefs['updateepubcover'] = False
-default_prefs['keeptags'] = False
-default_prefs['urlsfromclip'] = True
-default_prefs['updatedefault'] = True
-default_prefs['fileform'] = 'epub'
-default_prefs['collision'] = OVERWRITE
-default_prefs['deleteotherforms'] = False
-default_prefs['adddialogstaysontop'] = False
-default_prefs['includeimages'] = False
-default_prefs['lookforurlinhtml'] = False
-default_prefs['injectseries'] = False
-
-default_prefs['send_lists'] = ''
-default_prefs['read_lists'] = ''
-default_prefs['addtolists'] = False
-default_prefs['addtoreadlists'] = False
-default_prefs['addtolistsonread'] = False
-
-default_prefs['gcnewonly'] = False
-default_prefs['gc_site_settings'] = {}
-default_prefs['allow_gc_from_ini'] = True
-
-default_prefs['countpagesstats'] = []
-
-default_prefs['errorcol'] = ''
-default_prefs['custom_cols'] = {}
-default_prefs['custom_cols_newonly'] = {}
-default_prefs['allow_custcol_from_ini'] = True
-
-default_prefs['std_cols_newonly'] = {}
-
-def set_library_config(library_config):
-    get_gui().current_db.prefs.set_namespaced(PREFS_NAMESPACE,
-                                              PREFS_KEY_SETTINGS,
-                                              library_config)
-    
-def get_library_config():
-    db = get_gui().current_db
-    library_id = get_library_uuid(db)
-    library_config = None
-    # Check whether this is a configuration needing to be migrated
-    # from json into database.  If so: get it, set it, rename it in json.
-    if library_id in old_prefs:
-        #print("get prefs from old_prefs")
-        library_config = old_prefs[library_id]
-        set_library_config(library_config)
-        old_prefs["migrated to library db %s"%library_id] = old_prefs[library_id]
-        del old_prefs[library_id]
-
-    if library_config is None:
-        #print("get prefs from db")
-        library_config = db.prefs.get_namespaced(PREFS_NAMESPACE, PREFS_KEY_SETTINGS,
-                                                 copy.deepcopy(default_prefs))
-    return library_config
-
-# This is where all preferences for this plugin *were* stored
-# Remember that this name (i.e. plugins/fanfictiondownloader_plugin) is also
-# in a global namespace, so make it as unique as possible.
-# You should always prefix your config file name with plugins/,
-# so as to ensure you dont accidentally clobber a calibre config file
-old_prefs = JSONConfig('plugins/fanfictiondownloader_plugin')
-
-# fake out so I don't have to change the prefs calls anywhere.  The
-# Java programmer in me is offended by op-overloading, but it's very
-# tidy.
-class PrefsFacade():
-    def __init__(self,default_prefs):
-        self.default_prefs = default_prefs
-        self.libraryid = None
-        self.current_prefs = None
-        
-    def _get_prefs(self):
-        libraryid = get_library_uuid(get_gui().current_db)
-        if self.current_prefs == None or self.libraryid != libraryid:
-            #print("self.current_prefs == None(%s) or self.libraryid != libraryid(%s)"%(self.current_prefs == None,self.libraryid != libraryid))
-            self.libraryid = libraryid
-            self.current_prefs = get_library_config()
-        return self.current_prefs
-        
-    def __getitem__(self,k):            
-        prefs = self._get_prefs()
-        if k not in prefs:
-            # pulls from default_prefs.defaults automatically if not set
-            # in default_prefs
-            return self.default_prefs[k]
-        return prefs[k]
-
-    def __setitem__(self,k,v):
-        prefs = self._get_prefs()
-        prefs[k]=v
-        # self._save_prefs(prefs)
-
-    def __delitem__(self,k):
-        prefs = self._get_prefs()
-        if k in prefs:
-            del prefs[k]
-
-    def save_to_db(self):
-        set_library_config(self._get_prefs())
+    def _read_list_from_text(self,text,addreasontext=None):
+        cache = {}
+        for line in text.splitlines():
+            if ',' in line:
+                (rejurl,note) = line.split(',',1)
+            else:
+                (rejurl,note) = (line,'')
+            rejurl = getNormalStoryURL(rejurl)
+            if rejurl:
+                if addreasontext and note:
+                    note = note +" - "+addreasontext
+                elif addreasontext:
+                    note = addreasontext
+                cache[rejurl] = note
+        return cache
         
 
-prefs = PrefsFacade(default_prefs)
+    def _get_listcache(self):
+        if self.listcache == None:
+            self.listcache = self._read_list_from_text(prefs['rejecturls'])
+        return self.listcache
+
+    def _save_list(self,listcache):
+        rejectlist = []
+        for url in listcache:
+            rejectlist.append("%s,%s"%(url,listcache[url]))
+            
+        self.prefs['rejecturls'] = '\n'.join(rejectlist)
+        self.prefs.save_to_db()
+        self.listcache = None
+        
+    def clear_cache(self):
+        self.listcache = None
+
+    def check(self,url):
+        with self.sync_lock:
+            listcache = self._get_listcache()
+            if url in listcache:
+                note = listcache[url]
+                return note
+            
+            # not found
+            return None
+
+    def remove(self,url):
+        with self.sync_lock:
+            listcache = self._get_listcache()
+            if url in listcache:
+                del listcache[url]
+                self._save_list(listcache)
+
+    def add_text(self,rejecttext,addreasontext):
+        self.add(self._read_list_from_text(rejecttext,addreasontext).items())
+            
+    def add(self,rejectlist,clear=False):
+        # rejectlist=list of (url,note) tuples.
+        with self.sync_lock:
+            if clear:
+                listcache={}
+            else:
+                listcache = self._get_listcache()
+            for (url,note) in rejectlist:
+                listcache[url]=note
+            self._save_list(listcache)
+
+    def get_list(self):
+        return copy.deepcopy(self._get_listcache())
+            
+    def get_reject_reasons(self):
+        return self.prefs['rejectreasons'].splitlines()
+
+rejecturllist = RejectURLList(prefs)
     
 class ConfigWidget(QWidget):
 
@@ -196,6 +166,7 @@ class ConfigWidget(QWidget):
         prefs['updatecover'] = self.basic_tab.updatecover.isChecked()
         prefs['updateepubcover'] = self.basic_tab.updateepubcover.isChecked()
         prefs['keeptags'] = self.basic_tab.keeptags.isChecked()
+        prefs['showmarked'] = self.basic_tab.showmarked.isChecked()
         prefs['urlsfromclip'] = self.basic_tab.urlsfromclip.isChecked()
         prefs['updatedefault'] = self.basic_tab.updatedefault.isChecked()
         prefs['deleteotherforms'] = self.basic_tab.deleteotherforms.isChecked()
@@ -362,6 +333,13 @@ class BasicTab(QWidget):
 
         self.l.addSpacing(10)        
 
+        self.showmarked = QCheckBox("Show added/updated books when finished?",self)
+        self.showmarked.setToolTip("Show added/updated books only when finished.\nYou can also manually search for 'marked:ffdl_success'.\n'marked:ffdl_failed' is also available, or search 'marked:ffdl' for both.")
+        self.showmarked.setChecked(prefs['showmarked'])
+        self.l.addWidget(self.showmarked)
+
+        self.l.addSpacing(10)        
+
         self.urlsfromclip = QCheckBox('Take URLs from Clipboard?',self)
         self.urlsfromclip.setToolTip('Prefill URLs from valid URLs in Clipboard when Adding New.')
         self.urlsfromclip.setChecked(prefs['urlsfromclip'])
@@ -396,6 +374,27 @@ class BasicTab(QWidget):
         self.injectseries.setChecked(prefs['injectseries'])
         self.l.addWidget(self.injectseries)
 
+        self.l.addSpacing(10)        
+
+        horz = QHBoxLayout()
+        
+        self.rejectlist = QPushButton('Edit Reject URL List', self)
+        self.rejectlist.setToolTip("Edit list of URLs FFDL will automatically Reject.")
+        self.rejectlist.clicked.connect(self.show_rejectlist)
+        horz.addWidget(self.rejectlist)
+        
+        self.reject_urls = QPushButton('Add Reject URLs', self)
+        self.reject_urls.setToolTip("Add additional URLs to Reject as text.")
+        self.reject_urls.clicked.connect(self.add_reject_urls)
+        horz.addWidget(self.reject_urls)
+        
+        self.reject_reasons = QPushButton('Edit Reject Reasons List', self)
+        self.reject_reasons.setToolTip("Customize the Reasons presented when Rejecting URLs")
+        self.reject_reasons.clicked.connect(self.show_reject_reasons)
+        horz.addWidget(self.reject_reasons)
+        
+        self.l.addLayout(horz)
+        
         self.l.insertStretch(-1)
         
     def set_collisions(self):
@@ -411,7 +410,53 @@ class BasicTab(QWidget):
     def show_defaults(self):
         text = get_resources('plugin-defaults.ini')
         ShowDefaultsIniDialog(self.windowIcon(),text,self).exec_()
+
+    def show_rejectlist(self):
+        rejectlist = []
+        for (url,note) in rejecturllist.get_list().items():
+            rejectlist.append((None,url,note,note))
+            
+        d = RejectListDialog(self,
+                             rejectlist,
+                             rejectreasons=rejecturllist.get_reject_reasons(),
+                             header="Edit Reject URLs List",
+                             show_delete=False,
+                             show_all_reasons=False)
+        d.exec_()
         
+        if d.result() != d.Accepted:
+            return
+        
+        rejectlist=[]
+        for (bookid,url,note) in d.get_reject_list():
+            rejectlist.append((url,note))
+
+        rejecturllist.add(rejectlist,clear=True)
+        
+    def show_reject_reasons(self):
+        d = EditTextDialog(self,
+                           prefs['rejectreasons'],
+                           icon=self.windowIcon(),
+                           title="Reject Reasons",
+                           label="Customize Reject List Reasons",
+                           tooltip="Customize the Reasons presented when Rejecting URLs")
+        d.exec_()
+        if d.result() == d.Accepted:
+            prefs['rejectreasons'] = d.get_plain_text()
+                
+    def add_reject_urls(self):
+        d = EditTextDialog(self,
+                           "http://example.com?story.php?sid=5,Reason why I rejected it",
+                           icon=self.windowIcon(),
+                           title="Add Reject URLs",
+                           label="Add Reject URLs. Use: <b>http://...,note</b><br>Invalid story URLs will be ignored.",
+                           tooltip="One URL per line, everything after <b>,</b> will be put in the note.",
+                           rejectreasons=rejecturllist.get_reject_reasons(),
+                           reasonslabel='Add this reason to all URLs added:')
+        d.exec_()
+        if d.result() == d.Accepted:
+            rejecturllist.add_text(d.get_plain_text(),d.get_reason_text())
+                
 class PersonalIniTab(QWidget):
 
     def __init__(self, parent_dialog, plugin_action):
@@ -451,7 +496,7 @@ class PersonalIniTab(QWidget):
     def show_defaults(self):
         text = get_resources('plugin-defaults.ini')
         ShowDefaultsIniDialog(self.windowIcon(),text,self).exec_()
-
+        
 class ShowDefaultsIniDialog(QDialog):
 
     def __init__(self, icon, text, parent=None):
@@ -758,7 +803,7 @@ titleLabels = {
     'ships':'Relationships',
     'datePublished':'Published',
     'dateUpdated':'Updated',
-    'dateCreated':'Packaged',
+    'dateCreated':'Created',
     'rating':'Rating',
     'warnings':'Warnings',
     'numChapters':'Chapters',
@@ -769,7 +814,7 @@ titleLabels = {
     'extratags':'Extra Tags',
     'title':'Title',
     'storyUrl':'Story URL',
-    'description':'Summary',
+    'description':'Description',
     'author':'Author',
     'authorUrl':'Author URL',
     'formatname':'File Format',
@@ -916,3 +961,4 @@ class StandardColumnsTab(QWidget):
             self.l.addLayout(horz)
         
         self.l.insertStretch(-1)
+

calibre-plugin/dialogs.py

@@ -8,15 +8,22 @@ __copyright__ = '2011, Jim Miller'
 __docformat__ = 'restructuredtext en'
 
 import traceback
+from functools import partial
+
+import urllib
+import email
 
 from PyQt4 import QtGui
-from PyQt4.Qt import (QDialog, QTableWidget, QMessageBox, QVBoxLayout, QHBoxLayout, QGridLayout,
-                      QPushButton, QProgressDialog, QString, QLabel, QCheckBox, QIcon, QTextCursor,
-                      QTextEdit, QLineEdit, QInputDialog, QComboBox, QClipboard, QVariant,
-                      QProgressDialog, QTimer, QDialogButtonBox, QPixmap, Qt, QAbstractItemView )
+from PyQt4.Qt import (QDialog, QTableWidget, QMessageBox, QVBoxLayout, QHBoxLayout,
+                      QGridLayout, QPushButton, QProgressDialog, QString, QLabel,
+                      QCheckBox, QIcon, QTextCursor, QTextEdit, QLineEdit, QInputDialog,
+                      QComboBox, QClipboard, QVariant, QProgressDialog, QTimer,
+                      QDialogButtonBox, QPixmap, Qt, QAbstractItemView, SIGNAL,
+                      QTableWidgetItem )
 
 from calibre.gui2 import error_dialog, warning_dialog, question_dialog, info_dialog
 from calibre.gui2.dialogs.confirm_delete import confirm
+from calibre.gui2.complete2 import EditWithComplete
 
 from calibre import confirm_config_name
 from calibre.gui2 import dynamic
@@ -26,13 +33,15 @@ from calibre_plugins.fanfictiondownloader_plugin.common_utils \
     import (ReadOnlyTableWidgetItem, ReadOnlyTextIconWidgetItem, SizePersistedDialog,
             ImageTitleLayout, get_icon)
 
-SKIP='Skip'
-ADDNEW='Add New Book'
-UPDATE='Update EPUB if New Chapters'
-UPDATEALWAYS='Update EPUB Always'
-OVERWRITE='Overwrite if Newer'
-OVERWRITEALWAYS='Overwrite Always'
-CALIBREONLY='Update Calibre Metadata Only'
+from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.geturls import get_urls_from_html, get_urls_from_text
+
+SKIP=u'Skip'
+ADDNEW=u'Add New Book'
+UPDATE=u'Update EPUB if New Chapters'
+UPDATEALWAYS=u'Update EPUB Always'
+OVERWRITE=u'Overwrite if Newer'
+OVERWRITEALWAYS=u'Overwrite Always'
+CALIBREONLY=u'Update Calibre Metadata Only'
 collision_order=[SKIP,
                  ADDNEW,
                  UPDATE,
@@ -40,7 +49,24 @@ collision_order=[SKIP,
                  OVERWRITE,
                  OVERWRITEALWAYS,
                  CALIBREONLY,]
+
+anthology_collision_order=[UPDATE,
+                           UPDATEALWAYS,
+                           OVERWRITEALWAYS]
         
+# This is a more than slightly kludgey way to get
+# EditWithComplete to *not* alpha-order the reasons, but leave
+# them in the order entered.  If
+# calibre.gui2.complete2.CompleteModel.set_items ever changes,
+# this function will need to also.
+def complete_model_set_items_kludge(self, items):
+    items = [unicode(x.strip()) for x in items]
+    items = [x for x in items if x]
+    items = tuple(items)
+    self.all_items = self.current_items = items
+    self.current_prefix = ''
+    self.reset()
+    
 class NotGoingToDownload(Exception):
     def __init__(self,error,icon='dialog_error.png'):
         self.error=error
@@ -52,6 +78,46 @@ class NotGoingToDownload(Exception):
 class DroppableQTextEdit(QTextEdit):
     def __init__(self,parent):
         QTextEdit.__init__(self,parent)
+
+    def dropEvent(self,event):
+        # print("event:%s"%event)
+        # print("event.mimeData():%s"%event.mimeData())
+        # print("event.mimeData().text():%s"%str(event.mimeData().text()))
+        # print("event.mimeData().data():%s"%str(event.mimeData().data()))
+        # print("event.mimeData().formats():%s"%[str(f) for f in event.mimeData().formats()])
+        # for f in event.mimeData().formats():
+        #     try:
+        #         print("event.mimeData().data('%s'):%s"%(f,event.mimeData().data(f)))
+        #     except:
+        #         print("failed %s"%f)
+
+        mimetype='text/uri-list'
+        # print("event.mimeData().data('%s'):%s"%(mimetype,event.mimeData().data(mimetype)))
+
+        urllist=[]
+        filelist="%s"%event.mimeData().data(mimetype)
+        for f in filelist.splitlines():
+            #print("filename:%s"%f)
+            if f.endswith(".eml"):
+                fhandle = urllib.urlopen(f)
+                #print("file:\n%s\n\n"%fhandle.read())
+                msg = email.message_from_file(fhandle)
+                if msg.is_multipart():
+                    for part in msg.walk():
+                        #print("part type:%s"%part.get_content_type())
+                        if part.get_content_type() == "text/html":
+                            #print("URL list:%s"%get_urls_from_data(part.get_payload(decode=True)))
+                            urllist.extend(get_urls_from_html(part.get_payload(decode=True)))
+                        if part.get_content_type() == "text/plain":
+                            #print("part content:text/plain")
+                            # print("part content:%s"%part.get_payload(decode=True))
+                            urllist.extend(get_urls_from_text(part.get_payload(decode=True)))
+                else:
+                    urllist.extend(get_urls_from_text("%s"%msg))
+        
+        if urllist:
+            self.append("\n".join(urllist))
+        return QTextEdit.dropEvent(self,event)
         
     def canInsertFromMimeData(self, source):
         if source.hasUrls():
@@ -67,10 +133,23 @@ class DroppableQTextEdit(QTextEdit):
                             
 class AddNewDialog(SizePersistedDialog):
 
-    def __init__(self, gui, prefs, icon, url_list_text):
+    def __init__(self, gui, prefs, icon, url_list_text, merge=False, newmerge=False):
         SizePersistedDialog.__init__(self, gui, 'FanFictionDownLoader plugin:add new dialog')
         self.gui = gui
+        self.merge = merge
+        self.newmerge = newmerge
 
+        if merge:
+            labeltext = 'Story URL(s) for anthology, one per line:'
+            tooltiptext = 'URLs for stories to include in the anthology, one per line.\nWill take URLs from clipboard, but only valid URLs.'
+            collisiontext = 'If Story Already Exists in Anthology?'
+            collisiontooltip = "What to do if there's already an existing story with the same URL in the anthology."
+        else:
+            labeltext = 'Story URL(s), one per line:'
+            tooltiptext = 'URLs for stories, one per line.\nWill take URLs from clipboard, but only valid URLs.\nAdd [1,5] after the URL to limit the download to chapters 1-5.'
+            collisiontext = 'If Story Already Exists?'
+            collisiontooltip = "What to do if there's already an existing story with the same URL or title and author."
+            
         if prefs['adddialogstaysontop']:
             QDialog.setWindowFlags ( self, Qt.Dialog|Qt.WindowStaysOnTopHint )
         
@@ -81,55 +160,58 @@ class AddNewDialog(SizePersistedDialog):
         self.setWindowTitle('FanFictionDownLoader')
         self.setWindowIcon(icon)
 
-        self.l.addWidget(QLabel('Story URL(s), one per line:'))
+        self.l.addWidget(QLabel(labeltext))
         self.url = DroppableQTextEdit(self)
-        self.url.setToolTip('URLs for stories, one per line.\nWill take URLs from clipboard, but only valid URLs.\nAdd [1,5] after the URL to limit the download to chapters 1-5.')
+        self.url.setToolTip(tooltiptext)
         self.url.setLineWrapMode(QTextEdit.NoWrap)
         self.url.setText(url_list_text)
         self.l.addWidget(self.url)
-        
-        horz = QHBoxLayout()
-        label = QLabel('Output &Format:')
-        horz.addWidget(label)
-        self.fileform = QComboBox(self)
-        self.fileform.addItem('epub')
-        self.fileform.addItem('mobi')
-        self.fileform.addItem('html')
-        self.fileform.addItem('txt')
-        self.fileform.setCurrentIndex(self.fileform.findText(prefs['fileform']))
-        self.fileform.setToolTip('Choose output format to create.  May set default from plugin configuration.')
-        self.fileform.activated.connect(self.set_collisions)
-        
-        label.setBuddy(self.fileform)
-        horz.addWidget(self.fileform)
-        self.l.addLayout(horz)
 
-        horz = QHBoxLayout()
-        label = QLabel('If Story Already Exists?')
-        horz.addWidget(label)
-        self.collision = QComboBox(self)
-        self.collision.setToolTip("What to do if there's already an existing story with the same URL or title and author.")
-        # add collision options
-        self.set_collisions()
-        i = self.collision.findText(prefs['collision'])
-        if i > -1:
-            self.collision.setCurrentIndex(i)
-        label.setBuddy(self.collision)
-        horz.addWidget(self.collision)
-        self.l.addLayout(horz)
+        if not merge:
+            horz = QHBoxLayout()
+            label = QLabel('Output &Format:')
+            horz.addWidget(label)
+            self.fileform = QComboBox(self)
+            self.fileform.addItem('epub')
+            self.fileform.addItem('mobi')
+            self.fileform.addItem('html')
+            self.fileform.addItem('txt')
+            self.fileform.setCurrentIndex(self.fileform.findText(prefs['fileform']))
+            self.fileform.setToolTip('Choose output format to create.  May set default from plugin configuration.')
+            self.fileform.activated.connect(self.set_collisions)
+            
+            label.setBuddy(self.fileform)
+            horz.addWidget(self.fileform)
+            self.l.addLayout(horz)
 
-        horz = QHBoxLayout()
-        self.updatemeta = QCheckBox('Update Calibre &Metadata?',self)
-        self.updatemeta.setToolTip("Update metadata for existing stories in Calibre from web site?\n(Columns set to 'New Only' in the column tabs will only be set for new books.)")
-        self.updatemeta.setChecked(prefs['updatemeta'])
-        horz.addWidget(self.updatemeta)
-
-        self.updateepubcover = QCheckBox('Update EPUB Cover?',self)
-        self.updateepubcover.setToolTip('Update book cover image from site or defaults (if found) <i>inside</i> the EPUB when EPUB is updated.')
-        self.updateepubcover.setChecked(prefs['updateepubcover'])
-        horz.addWidget(self.updateepubcover)
-        
-        self.l.addLayout(horz)
+        if not newmerge:
+            horz = QHBoxLayout()
+            label = QLabel(collisiontext)
+            horz.addWidget(label)
+            self.collision = QComboBox(self)
+            self.collision.setToolTip(collisiontooltip)
+            # add collision options
+            self.set_collisions()
+            i = self.collision.findText(prefs['collision'])
+            if i > -1:
+                self.collision.setCurrentIndex(i)
+            label.setBuddy(self.collision)
+            horz.addWidget(self.collision)
+            self.l.addLayout(horz)
+    
+            horz = QHBoxLayout()
+            self.updatemeta = QCheckBox('Update Calibre &Metadata?',self)
+            self.updatemeta.setToolTip("Update metadata for existing stories in Calibre from web site?\n(Columns set to 'New Only' in the column tabs will only be set for new books.)")
+            self.updatemeta.setChecked(prefs['updatemeta'])
+            horz.addWidget(self.updatemeta)
+    
+            if not merge: # hide if anthology merge.
+                self.updateepubcover = QCheckBox('Update EPUB Cover?',self)
+                self.updateepubcover.setToolTip('Update book cover image from site or defaults (if found) <i>inside</i> the EPUB when EPUB is updated.')
+                self.updateepubcover.setChecked(prefs['updateepubcover'])
+                horz.addWidget(self.updateepubcover)
+            
+            self.l.addLayout(horz)
 
         button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
         button_box.accepted.connect(self.accept)
@@ -146,20 +228,39 @@ class AddNewDialog(SizePersistedDialog):
     def set_collisions(self):
         prev=self.collision.currentText()
         self.collision.clear()
-        for o in collision_order:
-            if self.fileform.currentText() == 'epub' or o not in [UPDATE,UPDATEALWAYS]:
+        if self.merge:
+            order = anthology_collision_order
+        else:
+            order = collision_order
+        for o in order:
+            if self.merge or self.fileform.currentText() == 'epub' or o not in [UPDATE,UPDATEALWAYS]:
                 self.collision.addItem(o)
         i = self.collision.findText(prev)
         if i > -1:
             self.collision.setCurrentIndex(i)
         
     def get_ffdl_options(self):
-        return {
-            'fileform': unicode(self.fileform.currentText()),
-            'collision': unicode(self.collision.currentText()),
-            'updatemeta': self.updatemeta.isChecked(),
-            'updateepubcover': self.updateepubcover.isChecked(),
-            }
+        if self.merge:
+            if self.newmerge:
+                updatemeta=True
+                collision=ADDNEW
+            else:
+                updatemeta=self.updatemeta.isChecked()
+                collision=unicode(self.collision.currentText())
+                
+            return {
+                'fileform': 'epub',
+                'collision': collision,
+                'updatemeta': updatemeta,
+                'updateepubcover': True,
+                }
+        else:
+            return {
+                'fileform': unicode(self.fileform.currentText()),
+                'collision': unicode(self.collision.currentText()),
+                'updatemeta': self.updatemeta.isChecked(),
+                'updateepubcover': self.updateepubcover.isChecked(),
+                }
 
     def get_urlstext(self):
         return unicode(self.url.toPlainText())
@@ -176,10 +277,11 @@ class CollectURLDialog(SizePersistedDialog):
     '''
     Collect single url for get urls.
     '''
-    def __init__(self, gui, title, url_text): 
+    def __init__(self, gui, title, url_text, epubmerge_plugin=None): 
         SizePersistedDialog.__init__(self, gui, 'FanFictionDownLoader plugin:get story urls')
         self.gui = gui
         self.status=False
+        self.anthology=False
 
         self.setMinimumWidth(300)
         
@@ -187,28 +289,40 @@ class CollectURLDialog(SizePersistedDialog):
         self.setLayout(self.l)
 
         self.setWindowTitle(title)
-        self.l.addWidget(QLabel(title),0,0,1,2)
+        self.l.addWidget(QLabel(title),0,0,1,3)
         
         self.l.addWidget(QLabel("URL:"),1,0)
         self.url = QLineEdit(self)
         self.url.setText(url_text)
-        self.l.addWidget(self.url,1,1)
+        self.l.addWidget(self.url,1,1,1,2)
    
-        self.ok_button = QPushButton('OK', self)
-        self.ok_button.clicked.connect(self.ok)
-        self.l.addWidget(self.ok_button,2,0)
+        self.indiv_button = QPushButton('For Individual Books', self)
+        self.indiv_button.setToolTip('Get URLs and go to dialog for individual story downloads.')
+        self.indiv_button.clicked.connect(self.indiv)
+        self.l.addWidget(self.indiv_button,2,0)
+
+        self.merge_button = QPushButton('For Anthology Epub', self)
+        self.merge_button.setToolTip('Get URLs and go to dialog for Anthology download.\nRequires EpubMerge 1.3.1+ plugin.')
+        self.merge_button.clicked.connect(self.merge)
+        self.l.addWidget(self.merge_button,2,1)
+        self.merge_button.setEnabled(epubmerge_plugin!=None)
 
         self.cancel_button = QPushButton('Cancel', self)
         self.cancel_button.clicked.connect(self.cancel)
-        self.l.addWidget(self.cancel_button,2,1)
+        self.l.addWidget(self.cancel_button,2,2)
 
         # restore saved size.
         self.resize_dialog()
 
-    def ok(self):
+    def indiv(self):
         self.status=True
         self.accept()
 
+    def merge(self):
+        self.status=True
+        self.anthology=True
+        self.accept()
+
     def cancel(self):
         self.status=False
         self.reject()
@@ -488,84 +602,6 @@ class UpdateExistingDialog(SizePersistedDialog):
             'updateepubcover': self.updateepubcover.isChecked(),
             }
 
-def display_story_list(gui, header, prefs, icon, books,
-                       label_text='',
-                       save_size_name='fanfictiondownloader_plugin:display list dialog',
-                       offer_skip=False):
-    all_good = True
-    for b in books:
-        if not b['good']:
-            all_good=False
-            break
-        
-    ## 
-    if all_good and not dynamic.get(confirm_config_name(save_size_name), True):
-        return True
-        pass
-            ## fake accept?
-    d = DisplayStoryListDialog(gui, header, prefs, icon, books,
-                               label_text,
-                               save_size_name,
-                               offer_skip and all_good)
-    d.exec_()
-    return d.result() == d.Accepted
-
-class DisplayStoryListDialog(SizePersistedDialog):
-    def __init__(self, gui, header, prefs, icon, books,
-                 label_text='',
-                 save_size_name='fanfictiondownloader_plugin:display list dialog',
-                 offer_skip=False):
-        SizePersistedDialog.__init__(self, gui, save_size_name)
-        self.name = save_size_name
-        self.gui = gui
-        
-        self.setWindowTitle(header)
-        self.setWindowIcon(icon)
-        
-        layout = QVBoxLayout(self)
-        self.setLayout(layout)
-        title_layout = ImageTitleLayout(self, 'images/icon.png',
-                                        header)
-        layout.addLayout(title_layout)
-
-        self.books_table = StoryListTableWidget(self)
-        layout.addWidget(self.books_table)
-
-        options_layout = QHBoxLayout()
-        self.label = QLabel(label_text)
-        #self.label.setOpenExternalLinks(True)
-        #self.label.setWordWrap(True)
-        options_layout.addWidget(self.label)
-        
-        if offer_skip:
-            spacerItem1 = QtGui.QSpacerItem(2, 4, QtGui.QSizePolicy.Expanding, QtGui.QSizePolicy.Minimum)
-            options_layout.addItem(spacerItem1)
-            self.again = QCheckBox('Show this again?',self)
-            self.again.setChecked(True)
-            self.again.stateChanged.connect(self.toggle)
-            self.again.setToolTip('Uncheck to skip review and update stories immediately when no problems.')
-            options_layout.addWidget(self.again)
-        
-        button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
-        button_box.accepted.connect(self.accept)
-        button_box.rejected.connect(self.reject)
-
-        options_layout.addWidget(button_box)
-        
-        layout.addLayout(options_layout)
-        
-        # Cause our dialog size to be restored from prefs or created on first usage
-        self.resize_dialog()
-        self.books_table.populate_table(books)
-
-    def get_books(self):
-        return self.books_table.get_books()
-
-    def toggle(self, *args):
-        dynamic[confirm_config_name(self.name)] = self.again.isChecked()
-
-    
-
 class StoryListTableWidget(QTableWidget):
 
     def __init__(self, parent):
@@ -712,3 +748,321 @@ class StoryListTableWidget(QTableWidget):
             self.setItem(dest_row, col, self.takeItem(src_row, col))
         self.removeRow(src_row)
         self.blockSignals(False)
+
+class RejectListTableWidget(QTableWidget):
+
+    def __init__(self, parent,rejectreasons=[]):
+        QTableWidget.__init__(self, parent)
+        self.setSelectionBehavior(QAbstractItemView.SelectRows)
+        self.rejectreasons = rejectreasons
+
+    def on_headersection_clicked(self):
+        self.setSortingEnabled(True)
+        
+    def populate_table(self, reject_list):
+        self.clear()
+        self.setAlternatingRowColors(True)
+        self.setRowCount(len(reject_list))
+        header_labels = ['URL', 'Note']
+        self.setColumnCount(len(header_labels))
+        self.setHorizontalHeaderLabels(header_labels)
+        self.horizontalHeader().setStretchLastSection(True)
+        #self.verticalHeader().setDefaultSectionSize(24)
+        self.verticalHeader().hide()
+
+        # need sortingEnbled to sort, but off to up & down.
+        self.connect(self.horizontalHeader(),
+                     SIGNAL('sectionClicked(int)'),
+                     self.on_headersection_clicked)
+
+        # row is just row number.
+        for row, rejectrow in enumerate(reject_list):
+            self.populate_table_row(row,rejectrow)
+
+        self.resizeColumnsToContents()
+        self.setMinimumColumnWidth(1, 100)
+        self.setMinimumColumnWidth(2, 100)
+        self.setMinimumSize(300, 0)
+
+    def setMinimumColumnWidth(self, col, minimum):
+        if self.columnWidth(col) < minimum:
+            self.setColumnWidth(col, minimum)
+
+    def populate_table_row(self, row, rejectrow):
+
+        (bookid,url,titleauth,oldrejnote) = rejectrow
+        if oldrejnote:
+            noteprefix = note = oldrejnote
+            # incase the existing note ends with one of the known reasons.
+            for reason in self.rejectreasons:
+                if noteprefix.endswith(' - '+reason):
+                    noteprefix = noteprefix[:-len(' - '+reason)]
+                    break
+        else:
+            noteprefix = note = titleauth
+            
+        if len(noteprefix) > 0:
+            noteprefix = noteprefix+' - '
+            
+        url_cell = ReadOnlyTableWidgetItem(url)
+        url_cell.setData(Qt.UserRole, QVariant(bookid))
+        url_cell.setToolTip('URL to add to the Reject List.')
+        self.setItem(row, 0, url_cell)
+
+        note_cell = EditWithComplete(self)
+
+        note_cell.lineEdit().mcompleter.model().set_items = \
+            partial(complete_model_set_items_kludge,
+                    note_cell.lineEdit().mcompleter.model())
+        
+        items = [note]+[ noteprefix+x for x in self.rejectreasons ]
+        note_cell.update_items_cache(items)
+        note_cell.show_initial_value(note)
+        note_cell.set_separator(None)
+        note_cell.setToolTip('Select or Edit Reject Note.')
+        self.setCellWidget(row, 1, note_cell)
+        
+        # note_cell = QTableWidgetItem(note)
+        # note_cell.setToolTip('Double-click to edit note.')
+        # self.setItem(row, 1, note_cell)
+      
+    def get_reject_list(self):
+        rejectrows = []
+        for row in range(self.rowCount()):
+            bookid = self.item(row, 0).data(Qt.UserRole).toPyObject()
+            url = unicode(self.item(row, 0).text())
+            note = unicode(self.cellWidget(row, 1).currentText()).strip()
+            rejectrows.append((bookid,url,note))
+        return rejectrows
+
+    def remove_selected_rows(self):
+        self.setFocus()
+        rows = self.selectionModel().selectedRows()
+        if len(rows) == 0:
+            return
+        message = '<p>Are you sure you want to remove this URL from the list?'
+        if len(rows) > 1:
+            message = '<p>Are you sure you want to remove the %d selected URLs from the list?'%len(rows)
+        if not confirm(message,'ffdl_rejectlist_delete_item_again', self):
+            return
+        first_sel_row = self.currentRow()
+        for selrow in reversed(rows):
+            self.removeRow(selrow.row())
+        if first_sel_row < self.rowCount():
+            self.select_and_scroll_to_row(first_sel_row)
+        elif self.rowCount() > 0:
+            self.select_and_scroll_to_row(first_sel_row - 1)
+
+    def select_and_scroll_to_row(self, row):
+        self.selectRow(row)
+        self.scrollToItem(self.currentItem())
+
+    def move_rows_up(self):
+        self.setFocus()
+        rows = self.selectionModel().selectedRows()
+        if len(rows) == 0:
+            return
+        first_sel_row = rows[0].row()
+        if first_sel_row <= 0:
+            return
+        # Workaround for strange selection bug in Qt which "alters" the selection
+        # in certain circumstances which meant move down only worked properly "once"
+        selrows = []
+        for row in rows:
+            selrows.append(row.row())
+        selrows.sort()
+        for selrow in selrows:
+            self.swap_row_widgets(selrow - 1, selrow + 1)
+        scroll_to_row = first_sel_row - 1
+        if scroll_to_row > 0:
+            scroll_to_row = scroll_to_row - 1
+        self.scrollToItem(self.item(scroll_to_row, 0))
+
+    def move_rows_down(self):
+        self.setFocus()
+        rows = self.selectionModel().selectedRows()
+        if len(rows) == 0:
+            return
+        last_sel_row = rows[-1].row()
+        if last_sel_row == self.rowCount() - 1:
+            return
+        # Workaround for strange selection bug in Qt which "alters" the selection
+        # in certain circumstances which meant move down only worked properly "once"
+        selrows = []
+        for row in rows:
+            selrows.append(row.row())
+        selrows.sort()
+        for selrow in reversed(selrows):
+            self.swap_row_widgets(selrow + 2, selrow)
+        scroll_to_row = last_sel_row + 1
+        if scroll_to_row < self.rowCount() - 1:
+            scroll_to_row = scroll_to_row + 1
+        self.scrollToItem(self.item(scroll_to_row, 0))
+
+    def swap_row_widgets(self, src_row, dest_row):
+        self.blockSignals(True)
+        self.setSortingEnabled(False)
+        self.insertRow(dest_row)
+        for col in range(0, self.columnCount()):
+            self.setItem(dest_row, col, self.takeItem(src_row, col))
+        self.removeRow(src_row)
+        self.blockSignals(False)
+
+class RejectListDialog(SizePersistedDialog):
+    def __init__(self, gui, reject_list,
+                 rejectreasons=[],
+                 header="List of Books to Reject",
+                 icon='rotate-right.png',
+                 show_delete=True,
+                 show_all_reasons=True,
+                 save_size_name='ffdl:reject list dialog'):
+        SizePersistedDialog.__init__(self, gui, save_size_name)
+        self.gui = gui
+      
+        self.setWindowTitle(header)
+        self.setWindowIcon(get_icon(icon))
+      
+        layout = QVBoxLayout(self)
+        self.setLayout(layout)
+        title_layout = ImageTitleLayout(self, icon, header,
+                                        '<i></i>FFDL will remember these URLs and display the note and offer to reject them if you try to download them again later.')
+        layout.addLayout(title_layout)
+        rejects_layout = QHBoxLayout()
+        layout.addLayout(rejects_layout)
+
+        self.rejects_table = RejectListTableWidget(self,rejectreasons=rejectreasons)
+        rejects_layout.addWidget(self.rejects_table)
+
+        button_layout = QVBoxLayout()
+        rejects_layout.addLayout(button_layout)
+        spacerItem = QtGui.QSpacerItem(20, 40, QtGui.QSizePolicy.Minimum, QtGui.QSizePolicy.Expanding)
+        button_layout.addItem(spacerItem)
+        # self.move_up_button = QtGui.QToolButton(self)
+        # self.move_up_button.setToolTip('Move selected books up the list')
+        # self.move_up_button.setIcon(QIcon(I('arrow-up.png')))
+        # self.move_up_button.clicked.connect(self.books_table.move_rows_up)
+        # button_layout.addWidget(self.move_up_button)
+        self.remove_button = QtGui.QToolButton(self)
+        self.remove_button.setToolTip('Remove selected URL(s) from the list')
+        self.remove_button.setIcon(get_icon('list_remove.png'))
+        self.remove_button.clicked.connect(self.remove_from_list)
+        button_layout.addWidget(self.remove_button)
+        # self.move_down_button = QtGui.QToolButton(self)
+        # self.move_down_button.setToolTip('Move selected books down the list')
+        # self.move_down_button.setIcon(QIcon(I('arrow-down.png')))
+        # self.move_down_button.clicked.connect(self.books_table.move_rows_down)
+        # button_layout.addWidget(self.move_down_button)
+        spacerItem1 = QtGui.QSpacerItem(20, 40, QtGui.QSizePolicy.Minimum, QtGui.QSizePolicy.Expanding)
+        button_layout.addItem(spacerItem1)
+
+        if show_all_reasons:
+            self.reason_edit = EditWithComplete(self)
+            self.reason_edit.lineEdit().mcompleter.model().set_items = \
+                partial(complete_model_set_items_kludge,
+                        self.reason_edit.lineEdit().mcompleter.model())
+            
+            items = ['']+rejectreasons
+            self.reason_edit.update_items_cache(items)
+            self.reason_edit.show_initial_value('')
+            self.reason_edit.set_separator(None)
+            self.reason_edit.setToolTip("This will be added to whatever note you've set for each URL above.")
+            
+            horz = QHBoxLayout()
+            label = QLabel("Add this reason to all URLs added:")
+            label.setToolTip("This will be added to whatever note you've set for each URL above.")
+            horz.addWidget(label)
+            horz.addWidget(self.reason_edit)
+            horz.insertStretch(-1)
+            layout.addLayout(horz)
+                    
+        options_layout = QHBoxLayout()
+
+        if show_delete:
+            self.deletebooks = QCheckBox('Delete Books (including books without FanFiction URLs)?',self)
+            self.deletebooks.setToolTip("Delete the selected books after adding them to the Rejected URLs list.")
+            self.deletebooks.setChecked(True)
+            options_layout.addWidget(self.deletebooks)
+        
+        button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
+        button_box.accepted.connect(self.accept)
+        button_box.rejected.connect(self.reject)
+        options_layout.addWidget(button_box)
+      
+        layout.addLayout(options_layout)
+      
+        # Cause our dialog size to be restored from prefs or created on first usage
+        self.resize_dialog()
+        self.rejects_table.populate_table(reject_list)
+
+    def remove_from_list(self):
+        self.rejects_table.remove_selected_rows()
+
+    def get_reject_list(self):
+        return self.rejects_table.get_reject_list()
+
+    def get_reason_text(self):
+        return unicode(self.reason_edit.currentText()).strip()
+    
+    def get_deletebooks(self):
+        return self.deletebooks.isChecked()
+
+class EditTextDialog(QDialog):
+
+    def __init__(self, parent, text,
+                 icon=None, title=None, label=None, tooltip=None,
+                 rejectreasons=[],reasonslabel=None
+                 ):
+        QDialog.__init__(self, parent)
+        self.resize(600, 500)
+        self.l = QVBoxLayout()
+        self.setLayout(self.l)
+        self.label = QLabel(label)
+        if title:
+            self.setWindowTitle(title)
+        if icon:
+            self.setWindowIcon(icon)
+        self.l.addWidget(self.label)
+        
+        self.textedit = QTextEdit(self)
+        self.textedit.setLineWrapMode(QTextEdit.NoWrap)
+        self.textedit.setText(text)
+        self.l.addWidget(self.textedit)
+
+        if tooltip:
+            self.label.setToolTip(tooltip)
+            self.textedit.setToolTip(tooltip)
+
+        if rejectreasons or reasonslabel:
+            self.reason_edit = EditWithComplete(self)
+
+            self.reason_edit.lineEdit().mcompleter.model().set_items = \
+                partial(complete_model_set_items_kludge,
+                        self.reason_edit.lineEdit().mcompleter.model())
+            
+            items = ['']+rejectreasons
+            self.reason_edit.update_items_cache(items)
+            self.reason_edit.show_initial_value('')
+            self.reason_edit.set_separator(None)
+            self.reason_edit.setToolTip(reasonslabel)
+            
+            if reasonslabel:
+                horz = QHBoxLayout()
+                label = QLabel(reasonslabel)
+                label.setToolTip(reasonslabel)
+                horz.addWidget(label)
+                horz.addWidget(self.reason_edit)
+                self.l.addLayout(horz)
+            else:
+                self.l.addWidget(self.reason_edit)
+            
+        button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
+        button_box.accepted.connect(self.accept)
+        button_box.rejected.connect(self.reject)
+        self.l.addWidget(button_box)
+
+    def get_plain_text(self):
+        return unicode(self.textedit.toPlainText())
+
+    def get_reason_text(self):
+        return unicode(self.reason_edit.currentText()).strip()
+    

calibre-plugin/ffdl_util.py

@@ -0,0 +1,43 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2013, Jim Miller'
+__docformat__ = 'restructuredtext en'
+
+from StringIO import StringIO
+
+from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapters, exceptions
+from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.configurable import Configuration
+from calibre_plugins.fanfictiondownloader_plugin.prefs import (prefs)
+
+def get_ffdl_personalini():
+    if prefs['includeimages']:
+        # this is a cheat to make it easier for users.
+        return '''[epub]
+include_images:true
+keep_summary_html:true
+make_firstimage_cover:true
+''' + prefs['personal.ini']
+    else:
+        return prefs['personal.ini']
+
+def get_ffdl_config(url,fileform="epub",personalini=None):
+    if not personalini:
+        personalini = get_ffdl_personalini()
+    site='unknown'
+    try:
+        site = adapters.getConfigSectionFor(url)
+    except Exception as e:
+        print("Failed trying to get ini config for url(%s): %s, using section [%s] instead"%(url,e,site))
+    configuration = Configuration(site,fileform)
+    configuration.readfp(StringIO(get_resources("plugin-defaults.ini")))
+    configuration.readfp(StringIO(personalini))
+
+    return configuration
+
+def get_ffdl_adapter(url,fileform="epub",personalini=None):
+    return adapters.getAdapter(get_ffdl_config(url,fileform,personalini),url)
+    

calibre-plugin/jobs.py

@@ -18,9 +18,9 @@ from calibre.utils.ipc.job import ParallelJob
 from calibre_plugins.fanfictiondownloader_plugin.dialogs import (NotGoingToDownload,
     OVERWRITE, OVERWRITEALWAYS, UPDATE, UPDATEALWAYS, ADDNEW, SKIP, CALIBREONLY)
 from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapters, writers, exceptions
-from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.configurable import Configuration
 from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.epubutils import get_update_data
 
+from calibre_plugins.fanfictiondownloader_plugin.ffdl_util import (get_ffdl_adapter, get_ffdl_config)
 # ------------------------------------------------------------------------------
 #
 #              Functions to perform downloads using worker jobs
@@ -39,11 +39,12 @@ def do_download_worker(book_list, options,
 
     print(options['version'])
     total = 0
+    alreadybad = []
     # Queue all the jobs
     print("Adding jobs for URLs:")
     for book in book_list:
+        print("%s"%book['url'])
         if book['good']:
-            print("%s"%book['url'])
             total += 1
             args = ['calibre_plugins.fanfictiondownloader_plugin.jobs',
                     'do_download_for_worker',
@@ -58,6 +59,9 @@ def do_download_worker(book_list, options,
             # job._modified_date = modified_date
             # job._existing_isbn = existing_isbn
             server.add_job(job)
+        else:
+            # was already bad before the subprocess ever started.
+            alreadybad.append(book)
     
     # This server is an arbitrary_n job, so there is a notifier available.
     # Set the % complete to a small number to avoid the 'unavailable' indicator
@@ -90,11 +94,11 @@ def do_download_worker(book_list, options,
             print("Successfully downloaded:")
             for book in book_list:
                 if book['good']:
-                    print(book['title'])
+                    print("%s %s"%(book['title'],book['url']))
             print("\nUnsuccessful:")
             for book in book_list:
                 if not book['good']:
-                    print(book['title'])
+                    print("%s %s"%(book['title'],book['url']))
             break
 
     server.close()
@@ -110,9 +114,9 @@ def do_download_for_worker(book,options):
     try:
         book['comment'] = 'Download started...'
 
-        configuration = Configuration(adapters.getConfigSectionFor(book['url']),options['fileform'])
-        configuration.readfp(StringIO(get_resources("plugin-defaults.ini")))
-        configuration.readfp(StringIO(options['personal.ini']))
+        configuration = get_ffdl_config(book['url'],
+                                        options['fileform'],
+                                        options['personal.ini'])
         
         if not options['updateepubcover'] and 'epub_for_update' in book and options['collision'] in (UPDATE, UPDATEALWAYS):
             configuration.set("overrides","never_make_cover","true")
@@ -149,6 +153,22 @@ def do_download_for_worker(book,options):
         elif options['collision'] in (ADDNEW, SKIP, OVERWRITE, OVERWRITEALWAYS) or \
                 ('epub_for_update' not in book and options['collision'] in (UPDATE, UPDATEALWAYS)):
 
+            # preserve logfile even on overwrite.
+            if 'epub_for_update' in book:
+                (urlignore,
+                 chaptercountignore,
+                 oldchaptersignore,
+                 oldimgsignore,
+                 oldcoverignore,
+                 calibrebookmarkignore,
+                 # only logfile set in adapter, so others aren't used.
+                 adapter.logfile) = get_update_data(book['epub_for_update'])
+
+                # change the existing entries id to notid so
+                # write_epub writes a whole new set to indicate overwrite.
+                if adapter.logfile:
+                    adapter.logfile = adapter.logfile.replace("span id","span notid")
+            
             print("write to %s"%outfile)
             writer.writeStory(outfilename=outfile, forceOverwrite=True)
             book['comment'] = 'Download %s completed, %s chapters.'%(options['fileform'],story.getMetadata("numChapters"))
@@ -167,6 +187,17 @@ def do_download_for_worker(book,options):
              adapter.calibrebookmark,
              adapter.logfile) = get_update_data(book['epub_for_update'])
 
+            # dup handling from ffdl_plugin needed for anthology updates.
+            if options['collision'] == UPDATE:
+                if chaptercount == urlchaptercount:
+                    book['comment']="Already contains %d chapters.  Reuse as is."%chaptercount
+                    book['outfile'] = book['epub_for_update'] # for anthology merge ops.
+                    return book
+
+            # dup handling from ffdl_plugin needed for anthology updates.
+            if chaptercount > urlchaptercount:
+                raise NotGoingToDownload("Existing epub contains %d chapters, web site only has %d. Use Overwrite to force update." % (chaptercount,urlchaptercount),'dialog_error.png')
+                            
             print("Do update - epub(%d) vs url(%d)" % (chaptercount, urlchaptercount))
             print("write to %s"%outfile)
 

calibre-plugin/prefs.py

@@ -0,0 +1,141 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2013, Jim Miller'
+__docformat__ = 'restructuredtext en'
+
+import copy
+
+from calibre.utils.config import JSONConfig
+from calibre.gui2.ui import get_gui
+
+from calibre_plugins.fanfictiondownloader_plugin.dialogs import OVERWRITE
+from calibre_plugins.fanfictiondownloader_plugin.common_utils import get_library_uuid
+PREFS_NAMESPACE = 'FanFictionDownLoaderPlugin'
+PREFS_KEY_SETTINGS = 'settings'
+
+# Set defaults used by all.  Library specific settings continue to
+# take from here.
+default_prefs = {}
+default_prefs['personal.ini'] = get_resources('plugin-example.ini')
+default_prefs['rejecturls'] = ''
+default_prefs['rejectreasons'] = '''Sucked
+Boring
+Dup from another site'''
+
+default_prefs['updatemeta'] = True
+default_prefs['updatecover'] = False
+default_prefs['updateepubcover'] = False
+default_prefs['keeptags'] = False
+default_prefs['showmarked'] = False
+default_prefs['urlsfromclip'] = True
+default_prefs['updatedefault'] = True
+default_prefs['fileform'] = 'epub'
+default_prefs['collision'] = OVERWRITE
+default_prefs['deleteotherforms'] = False
+default_prefs['adddialogstaysontop'] = False
+default_prefs['includeimages'] = False
+default_prefs['lookforurlinhtml'] = False
+default_prefs['injectseries'] = False
+
+default_prefs['send_lists'] = ''
+default_prefs['read_lists'] = ''
+default_prefs['addtolists'] = False
+default_prefs['addtoreadlists'] = False
+default_prefs['addtolistsonread'] = False
+
+default_prefs['gcnewonly'] = False
+default_prefs['gc_site_settings'] = {}
+default_prefs['allow_gc_from_ini'] = True
+
+default_prefs['countpagesstats'] = []
+
+default_prefs['errorcol'] = ''
+default_prefs['custom_cols'] = {}
+default_prefs['custom_cols_newonly'] = {}
+default_prefs['allow_custcol_from_ini'] = True
+
+default_prefs['std_cols_newonly'] = {}
+
+def set_library_config(library_config,db):
+    db.prefs.set_namespaced(PREFS_NAMESPACE,
+                            PREFS_KEY_SETTINGS,
+                            library_config)
+
+def get_library_config(db):
+    library_id = get_library_uuid(db)
+    library_config = None
+    # Check whether this is a configuration needing to be migrated
+    # from json into database.  If so: get it, set it, rename it in json.
+    if library_id in old_prefs:
+        #print("get prefs from old_prefs")
+        library_config = old_prefs[library_id]
+        set_library_config(library_config)
+        old_prefs["migrated to library db %s"%library_id] = old_prefs[library_id]
+        del old_prefs[library_id]
+
+    if library_config is None:
+        #print("get prefs from db")
+        library_config = db.prefs.get_namespaced(PREFS_NAMESPACE, PREFS_KEY_SETTINGS,
+                                                 copy.deepcopy(default_prefs))
+    return library_config
+
+# This is where all preferences for this plugin *were* stored
+# Remember that this name (i.e. plugins/fanfictiondownloader_plugin) is also
+# in a global namespace, so make it as unique as possible.
+# You should always prefix your config file name with plugins/,
+# so as to ensure you dont accidentally clobber a calibre config file
+old_prefs = JSONConfig('plugins/fanfictiondownloader_plugin')
+
+# fake out so I don't have to change the prefs calls anywhere.  The
+# Java programmer in me is offended by op-overloading, but it's very
+# tidy.
+class PrefsFacade():
+    def _get_db(self):
+        if self.passed_db:
+            return self.passed_db
+        else:
+            # In the GUI plugin we want current db so we detect when
+            # it's changed.  CLI plugin calls need to pass db in.
+            return get_gui().current_db
+    
+    def __init__(self,passed_db=None):
+        self.default_prefs = default_prefs
+        self.libraryid = None
+        self.current_prefs = None
+        self.passed_db=passed_db
+        
+    def _get_prefs(self):
+        libraryid = get_library_uuid(self._get_db())
+        if self.current_prefs == None or self.libraryid != libraryid:
+            #print("self.current_prefs == None(%s) or self.libraryid != libraryid(%s)"%(self.current_prefs == None,self.libraryid != libraryid))
+            self.libraryid = libraryid
+            self.current_prefs = get_library_config(self._get_db())
+        return self.current_prefs
+        
+    def __getitem__(self,k):            
+        prefs = self._get_prefs()
+        if k not in prefs:
+            # pulls from default_prefs.defaults automatically if not set
+            # in default_prefs
+            return self.default_prefs[k]
+        return prefs[k]
+
+    def __setitem__(self,k,v):
+        prefs = self._get_prefs()
+        prefs[k]=v
+        # self._save_prefs(prefs)
+
+    def __delitem__(self,k):
+        prefs = self._get_prefs()
+        if k in prefs:
+            del prefs[k]
+
+    def save_to_db(self):
+        set_library_config(self._get_prefs(),self._get_db())
+        
+prefs = PrefsFacade()
+

defaults.ini

@@ -320,12 +320,13 @@ background_color: ffffff
 ## Allow customization of CSS.  Make sure to keep at least one space
 ## at the start of each line and to escape % to %%.  Also need
 ## background_color to be in the same section, if included in CSS.
-## 'adobe-text-layout: optimizeSpeed;' prevents hyphenation on newer Nooks
+## 'adobe-hyphenate: none;' prevents hyphenation on newer Nooks
+## STR(wG) (1.2.1+ for sure)
 output_css: 
  body { background-color: #%(background_color)s;
         text-align: justify;
         margin: 2%%;
-	adobe-text-layout: optimizeSpeed; }
+	adobe-hyphenate: none; }
  pre { font-size: x-small; }
  sml { font-size: small; }
  h1 { text-align: center; }
@@ -368,7 +369,7 @@ output_css:
 ## It can be either a 'file:' or 'http:' url.
 ## Note that if you enable make_firstimage_cover in [epub], but want
 ## to use default_cover_image for a specific site, use the site:format
-## section, for example: [www.ficwad.com:epub]
+## section, for example: [ficwad.com:epub]
 ## default_cover_image is a python string Template string with
 ## ${title}, ${author} etc, same as titlepage_entries. Unless
 ## allow_unsafe_filename is true, invalid filename chars will be
@@ -386,9 +387,20 @@ output_css:
 image_max_size: 580, 725
 
 ## Change image to grayscale, if graphics library allows, to save
-## space.
+## space.  Transparency removed as if remove_transparency: true
 #grayscale_images: false
 
+## jpg or png
+## -- jpg produces smaller images, and may be supported by more
+## readers, but it's older and doesn't allow transparency.
+## Transparency removed as if remove_transparency: true
+## -- png is newer but does allow transparency, but only in CLI.
+## It doesn't work in calibre PI due to limitations of the API.
+convert_images_to: jpg
+
+## Remove transparency and fill with background_color if true.
+remove_transparency: true
+
 ## if the <img> tag doesn't have a div or a p around it, nook gets
 ## confused and displays it on every page after that under the text
 ## for the rest of the chapter.  I doubt adding a div around the img
@@ -458,20 +470,26 @@ extratags: FanFiction,Testing,HTML
 #is_adult:true
 
 ## AO3 adapter defines a few extra metadata entries.
-extra_valid_entries:fandoms,freefromtags,ao3categories,comments,kudos,hits,bookmarks
+extra_valid_entries:fandoms,freeformtags,freefromtags,ao3categories,comments,kudos,hits,bookmarks,collections
 fandoms_label:Fandoms
+freeformtags_label:Freeform Tags
 freefromtags_label:Freeform Tags
 ao3categories_label:AO3 Categories
 comments_label:Comments
 kudos_label:Kudos
 hits_label:Hits
-bookmarks:Bookmarks
+collections_label:Collections
+bookmarks_label:Bookmarks
+
+## freeformtags was previously typo'ed as freefromtags.  This way,
+## freefromtags will still work for people who've used it.
+include_in_freefromtags:freeformtags
 
 ## adds to titlepage_entries instead of replacing it.
-#extra_titlepage_entries: fandoms,freefromtags,ao3categories,comments,kudos,hits,bookmarks
+#extra_titlepage_entries: fandoms,freeformtags,ao3categories,comments,kudos,hits,bookmarks
 
 ## adds to include_subject_tags instead of replacing it.
-#extra_subject_tags:fandoms,freefromtags,ao3categories
+#extra_subject_tags:fandoms,freeformtags,ao3categories
 
 [ashwinder.sycophanthex.com]
 ## Site dedicated to these categories/characters/ships
@@ -502,6 +520,15 @@ extracategories:Blood Ties
 ## personal.ini, not defaults.ini.
 #is_adult:true
 
+[buffynfaith.net]
+## Site dedicated to these categories/characters/ships
+extracategories:Buffy: The Vampire Slayer
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
 [castlefans.org]
 ## Site dedicated to these categories/characters/ships
 extracategories:Castle
@@ -559,6 +586,10 @@ extraships:Draco Malfoy/Hermione Granger
 ## personal.ini, not defaults.ini.
 #is_adult:true
 
+## some sites include images that we don't ever want becoming the
+## cover image.  This lets you exclude them.
+cover_exclusion_regexp:/images/.*?ribbon.gif
+
 ## Some adapters collect additional meta information beyond the
 ## standard ones.  They need to be defined in extra_valid_entries to
 ## tell the rest of the FFDL system about them.  They can be used in
@@ -592,6 +623,10 @@ cliches_label:Character Cliches
 #extra_logpage_entries: themes,timeline,cliches
 #extra_subject_tags: themes,timeline,cliches
 
+[efiction.esteliel.de]
+## Site dedicated to these categories/characters/ships
+extracategories:Lord of the Rings
+
 [erosnsappho.sycophanthex.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter
@@ -668,6 +703,19 @@ extracharacters:Hermione Granger
 ## Site dedicated to these categories/characters/ships
 extracategories:Highlander
 
+[imagine.e-fic.com]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
 [indeath.net]
 ## Site dedicated to these categories/characters/ships
 extracategories:In Death
@@ -753,6 +801,22 @@ extracategories:One Direction
 ## personal.ini, not defaults.ini.
 #is_adult:true
 
+[pommedesang.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Anita Blake Vampire Hunter 
+
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
 [ponyfictionarchive.net]
 ## Site dedicated to these categories/characters/ships
 extracategories:My Little Pony: Friendship is Magic
@@ -905,6 +969,14 @@ extracategories:InuYasha
 extracharacters:Sesshoumaru,Kagome
 extraships:Sesshoumaru/Kagome
 
+[www.dotmoon.net]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
 [www.efpfanfic.net]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -959,7 +1031,7 @@ extratags:
 ## for examples of how to use them.
 extra_valid_entries:reviews,favs,follows
 
-[www.ficwad.com]
+[ficwad.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -999,6 +1071,10 @@ extracategories:Harry Potter
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+[www.henneth-annun.net]
+## Site dedicated to these categories/characters/ships
+extracategories:The Hobbit
+
 [www.hpfandom.net]
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter
@@ -1078,10 +1154,23 @@ extraships:Harry Potter/Ginny Weasley
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter
 
+[www.potterfics.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
 [www.prisonbreakfic.net]
 ## Site dedicated to these categories/characters/ships
 extracategories:Prison Break
 
+[www.psychfic.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Psych
+
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
 [www.qaf-fic.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:Queer as Folk
@@ -1091,6 +1180,16 @@ extracategories:Queer as Folk
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+[www.restrictedsection.org]
+extracategories:Harry Potter
+extragenres:Erotica
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
 [www.scarvesandcoffee.net]
 ## Site dedicated to these categories/characters/ships
 extracategories:Glee
@@ -1240,12 +1339,6 @@ extracategories:Stargate: Atlantis
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
-[www.yourfanfiction.com]
-## Some sites do not require a login, but do require the user to
-## confirm they are adult for adult content.  In commandline version,
-## this should go in your personal.ini, not defaults.ini.
-#is_adult:true
-
 [overrides]
 ## It may sometimes be useful to override all of the specific format,
 ## site and site:format sections in your private configuration.  For

downloader.py

@@ -32,10 +32,17 @@ if sys.version_info >= (2, 7):
     loghandler.setFormatter(logging.Formatter("(=====)(levelname)s:%(message)s"))
     rootlogger.addHandler(loghandler)
 
-from fanficdownloader import adapters,writers,exceptions
-from fanficdownloader.configurable import Configuration
-from fanficdownloader.epubutils import get_dcsource_chaptercount, get_update_data
-from fanficdownloader.geturls import get_urls_from_page
+try:
+    from fanficdownloader import adapters,writers,exceptions
+    from fanficdownloader.configurable import Configuration
+    from fanficdownloader.epubutils import get_dcsource_chaptercount, get_update_data
+    from fanficdownloader.geturls import get_urls_from_page
+except:
+    # running under calibre
+    from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapters,writers,exceptions
+    from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.configurable import Configuration
+    from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.epubutils import get_dcsource_chaptercount, get_update_data
+    from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.geturls import get_urls_from_page
 
 if sys.version_info < (2, 5):
     print "This program requires Python 2.5 or newer."
@@ -48,15 +55,23 @@ def writeStory(config,adapter,writeformat,metaonly=False,outstream=None):
     del writer
     return output_filename
 
-def main():
+def main(argv,
+         parser=None,
+         passed_defaultsini=None,
+         passed_personalini=None):
    # read in args, anything starting with -- will be treated as --<varible>=<value>
-   usage = "usage: %prog [options] storyurl"
-   parser = OptionParser(usage)
+   if not parser:
+       parser = OptionParser("usage: %prog [options] storyurl")
    parser.add_option("-f", "--format", dest="format", default="epub",
-                     help="write story as FORMAT, epub(default), text or html", metavar="FORMAT")
+                     help="write story as FORMAT, epub(default), mobi, text or html", metavar="FORMAT")
+
+   if passed_defaultsini:
+       config_help="read config from specified file(s) in addition to calibre plugin personal.ini, ~/.fanficdownloader/personal.ini, and ./personal.ini"
+   else:
+       config_help="read config from specified file(s) in addition to ~/.fanficdownloader/defaults.ini, ~/.fanficdownloader/personal.ini, ./defaults.ini, and ./personal.ini"
    parser.add_option("-c", "--config",
                      action="append", dest="configfile", default=None,
-                     help="read config from specified file(s) in addition to ~/.fanficdownloader/defaults.ini, ~/.fanficdownloader/personal.ini, ./defaults.ini, ./personal.ini", metavar="CONFIG")
+                     help=config_help, metavar="CONFIG")
    parser.add_option("-b", "--begin", dest="begin", default=None,
                      help="Begin with Chapter START", metavar="START")
    parser.add_option("-e", "--end", dest="end", default=None,
@@ -69,13 +84,13 @@ def main():
                      help="Retrieve metadata and stop.  Or, if --update-epub, update metadata title page only.",)
    parser.add_option("-u", "--update-epub",
                      action="store_true", dest="update",
-                     help="Update an existing epub with new chapter, give epub filename instead of storyurl.",)
+                     help="Update an existing epub with new chapters, give epub filename instead of storyurl.",)
    parser.add_option("--update-cover",
                      action="store_true", dest="updatecover",
                      help="Update cover in an existing epub, otherwise existing cover (if any) is used on update.  Only valid with --update-epub.",)
    parser.add_option("--force",
                      action="store_true", dest="force",
-                     help="Force overwrite or update of an existing epub, download and overwrite all chapters.",)
+                     help="Force overwrite of an existing epub, download and overwrite all chapters.",)
    parser.add_option("-l", "--list",
                      action="store_true", dest="list",
                      help="Get list of valid story URLs from page given.",)
@@ -83,7 +98,7 @@ def main():
                      action="store_true", dest="debug",
                      help="Show debug output while downloading.",)
    
-   (options, args) = parser.parse_args()
+   (options, args) = parser.parse_args(argv)
 
    if not options.debug:
        logger = logging.getLogger("fanficdownloader")
@@ -107,12 +122,18 @@ def main():
 
    conflist = []
    homepath = join(expanduser("~"),".fanficdownloader")
+
+   if passed_defaultsini:
+       configuration.readfp(passed_defaultsini)
    
    if isfile(join(homepath,"defaults.ini")):
        conflist.append(join(homepath,"defaults.ini"))
    if isfile("defaults.ini"):
        conflist.append("defaults.ini")
        
+   if passed_personalini:
+       configuration.readfp(passed_personalini)
+   
    if isfile(join(homepath,"personal.ini")):
        conflist.append(join(homepath,"personal.ini"))
    if isfile("personal.ini"):
@@ -155,9 +176,7 @@ def main():
        return
 
    try:
-
        adapter = adapters.getAdapter(configuration,url)
-
        adapter.setChaptersRange(options.begin,options.end)
            
        ## Check for include_images and absence of PIL, give warning.
@@ -173,7 +192,6 @@ def main():
                    print "You have include_images enabled, but Python Image Library(PIL) isn't found.\nImages will be included full size in original format.\nContinue? (y/n)?"
                    if not sys.stdin.readline().strip().lower().startswith('y'):
                        return
-               
 
        ## three tries, that's enough if both user/pass & is_adult needed,
        ## or a couple tries of one or the other
@@ -243,5 +261,5 @@ def main():
 if __name__ == "__main__":
     #import time
     #start = time.time()
-    main()
+    main(sys.argv[1:])
     #print("Total time seconds:%f"%(time.time()-start))

fanficdownloader/adapters/__init__.py

@@ -23,6 +23,7 @@ import urlparse as up
 logger = logging.getLogger(__name__)
 
 from .. import exceptions as exceptions
+from ..configurable import Configuration
 
 ## must import each adapter here.
 
@@ -72,7 +73,6 @@ import adapter_iketernalnet
 import adapter_onedirectionfanfictioncom
 import adapter_prisonbreakficnet
 import adapter_storiesofardacom
-import adapter_yourfanfictioncom
 import adapter_samdeanarchivenu
 import adapter_destinysgatewaycom
 import adapter_ncisfictionnet
@@ -106,7 +106,15 @@ import adapter_indeathnet
 import adapter_jlaunlimitedcom
 import adapter_qafficcom
 import adapter_efpfanficnet
-
+import adapter_potterficscom
+import adapter_efictionestelielde
+import adapter_dotmoonnet
+import adapter_pommedesangcom
+import adapter_restrictedsectionorg
+import adapter_imagineeficcom
+import adapter_buffynfaithnet
+import adapter_psychficcom
+import adapter_hennethannunnet
 
 ## This bit of complexity allows adapters to be added by just adding
 ## importing.  It eliminates the long if/else clauses we used to need
@@ -125,9 +133,25 @@ for x in imports():
         #print x
         __class_list.append(sys.modules[x].getClass())
 
+def getNormalStoryURL(url):
+    if not getNormalStoryURL.__dummyconfig:
+        getNormalStoryURL.__dummyconfig = Configuration("test1.com","EPUB")
+    # pulling up an adapter is pretty low over-head.  If
+    # it fails, it's a bad url.
+    try:
+        adapter = getAdapter(getNormalStoryURL.__dummyconfig,url)
+        url = adapter.url
+        del adapter
+        return url
+    except:
+        return None;
+
+# kludgey function static/singleton
+getNormalStoryURL.__dummyconfig = None
+
 def getAdapter(config,url):
 
-    logger.debug("trying url:"+url)
+    #logger.debug("trying url:"+url)
     (cls,fixedurl) = getClassFor(url)
     logger.debug("fixedurl:"+fixedurl)
     if cls:
@@ -155,10 +179,6 @@ def getClassFor(url):
     ## remove any trailing '#' locations.
     fixedurl = re.sub(r"#.*$","",fixedurl)
     
-    ## remove any trailing '&' parameters--?sid=999 will be left.
-    ## that's all that any of the current adapters need or want.
-    fixedurl = re.sub(r"&.*$","",fixedurl)
-    
     parsedUrl = up.urlparse(fixedurl)
     domain = parsedUrl.netloc.lower()
     if( domain != parsedUrl.netloc ):
@@ -167,14 +187,16 @@ def getClassFor(url):
     cls = getClassFromList(domain)
     if not cls and domain.startswith("www."):
         domain = domain.replace("www.","")
-        logger.debug("trying site:without www: "+domain)
+        #logger.debug("trying site:without www: "+domain)
         cls = getClassFromList(domain)
         fixedurl = fixedurl.replace("http://www.","http://")
     if not cls:
-        logger.debug("trying site:www."+domain)
+        #logger.debug("trying site:www."+domain)
         cls = getClassFromList("www."+domain)
         fixedurl = fixedurl.replace("http://","http://www.")
         
+    fixedurl = cls.stripURLParameters(fixedurl)
+    
     return (cls,fixedurl)
     
 def getClassFromList(domain):

fanficdownloader/adapters/adapter_archiveofourownorg.py

@@ -82,7 +82,8 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
 
     def getSiteURLPattern(self):
         # http://archiveofourown.org/collections/Smallville_Slash_Archive/works/159770
-        return re.escape("http://")+re.escape(self.getSiteDomain())+r"(/collections/[^/]+)?/works/(?P<id>\d+)"
+        # Discard leading zeros from story ID numbers--AO3 doesn't use them in it's own chapter URLs.
+        return re.escape("http://")+re.escape(self.getSiteDomain())+r"(/collections/[^/]+)?/works/0*(?P<id>\d+)"
         
     ## Login
     def needToLoginCheck(self, data):
@@ -217,7 +218,7 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
         if a != None:
             genres = a.findAll('a',{'class':"tag"})
             for genre in genres:
-                self.story.addToList('freefromtags',genre.string)
+                self.story.addToList('freeformtags',genre.string)
                 self.story.addToList('genre',genre.string)
                 
         a = metasoup.find('dd',{'class':"category tags"})
@@ -233,13 +234,19 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
             chars = a.findAll('a',{'class':"tag"})
             for char in chars:
                 self.story.addToList('characters',char.string)
+                
         a = metasoup.find('dd',{'class':"relationship tags"})
         if a != None:
             ships = a.findAll('a',{'class':"tag"})
             for ship in ships:
                 self.story.addToList('ships',ship.string)
 		
-
+        a = metasoup.find('dd',{'class':"collections"})
+        if a != None:
+            collections = a.findAll('a')
+            for collection in collections:
+                self.story.addToList('collections',collection.string)
+		
         stats = metasoup.find('dl',{'class':'stats'})
         dt = stats.findAll('dt')
         dd = stats.findAll('dd')

fanficdownloader/adapters/adapter_archiveskyehawkecom.py

@@ -66,14 +66,15 @@ class ArchiveSkyeHawkeComAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'archive.skyehawke.com'
 
+    @classmethod
+    def getAcceptDomains(cls):
+        return ['archive.skyehawke.com','www.skyehawke.com']
+
     def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/story.php?no=1234"
+        return "http://archive.skyehawke.com/story.php?no=1234 http://www.skyehawke.com/archive/story.php?no=1234  http://skyehawke.com/archive/story.php?no=1234"
 
     def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/story.php?no=")+r"\d+$"
-
-
-        
+        return re.escape("http://")+r"(archive|www)\.skyehawke\.com/(archive/)?story\.php\?no=\d+$"
 
     ## Getting the chapter list and the meta data, plus 'is adult' checking.
     def extractChapterUrlsAndMetadata(self):

fanficdownloader/adapters/adapter_buffynfaithnet.py

@@ -0,0 +1,290 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+import cookielib as cl
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+
+# This function is called by the downloader in all adapter_*.py files
+# in this dir to register the adapter class.  So it needs to be
+# updated to reflect the class below it.  That, plus getSiteDomain()
+# take care of 'Registering'.
+def getClass():
+    return BuffyNFaithNetAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class BuffyNFaithNetAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+        
+        self.setHeader()
+        
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query correct
+        m = re.match(self.getSiteURLPattern(),url)
+        if m:
+            self.story.setMetadata('storyId',m.group('id'))
+            logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+            # normalized story URL. gets rid of chapter if there, left with ch 1 URL on this site
+            nurl = "http://"+self.getSiteDomain()+"/fanfictions/index.php?act=vie&id="+self.story.getMetadata('storyId')
+            self._setURL(nurl)
+            #argh, this mangles the ampersands I need on metadata['storyUrl']
+            #will set it this way
+            self.story.setMetadata('storyUrl',nurl,condremoveentities=False)
+        else:
+            raise exceptions.InvalidStoryURL(url,
+                                             self.getSiteDomain(),
+                                             self.getSiteExampleURLs())
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','bnfnet')
+
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'buffynfaith.net'
+
+    @classmethod
+    def stripURLParameters(cls,url):
+        "Only needs to be overriden if URL contains more than one parameter"
+        ## This adapter needs at least two parameters left on the URL, act and id
+        return re.sub(r"(\?act=(vie|ovr)&id=\d+)&.*$",r"\1",url)
+    
+    def setHeader(self):
+        "buffynfaith.net wants a Referer for images.  Used both above and below(after cookieproc added)"
+        self.opener.addheaders = [('Referer', 'http://'+self.getSiteDomain()+'/')]
+        
+    def getSiteExampleURLs(self):
+        return "http://buffynfaith.net/fanfictions/index.php?act=vie&id=963 http://buffynfaith.net/fanfictions/index.php?act=vie&id=949 http://buffynfaith.net/fanfictions/index.php?act=vie&id=949&ch=2"
+
+    def getSiteURLPattern(self):
+        #http://buffynfaith.net/fanfictions/index.php?act=vie&id=963
+        #http://buffynfaith.net/fanfictions/index.php?act=vie&id=949
+        #http://buffynfaith.net/fanfictions/index.php?act=vie&id=949&ch=2
+        p = re.escape("http://"+self.getSiteDomain()+"/fanfictions/index.php?act=")+\
+            r"(vie|ovr)&id=(?P<id>\d+)(&ch=(?P<ch>\d+))?$"
+        return p
+
+    def extractChapterUrlsAndMetadata(self):
+
+        dateformat = "%d %B %Y"
+        url = self.url
+        logger.debug("URL: "+url)
+        
+        #set a cookie to get past adult check
+        if self.is_adult or self.getConfig("is_adult"):
+            cookieproc = urllib2.HTTPCookieProcessor()
+            cookie = cl.Cookie(version=0, name='my_age', value='yes',
+                               port=None, port_specified=False,
+                               domain=self.getSiteDomain(), domain_specified=False, domain_initial_dot=False,
+                               path='/', path_specified=True,
+                               secure=False,
+                               expires=time.time()+10000,
+                               discard=False,
+                               comment=None,
+                               comment_url=None,
+                               rest={'HttpOnly': None},
+                               rfc2109=False)
+            cookieproc.cookiejar.set_cookie(cookie)
+            self.opener = urllib2.build_opener(cookieproc)        
+            self.setHeader()
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        #print data
+
+        if "ADULT CONTENT WARNING" in data: 
+            raise exceptions.AdultCheckRequired(self.url)
+
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        #stuff in <head>: description
+        svalue = soup.head.find('meta',attrs={'name':'description'})['content']
+        #self.story.setMetadata('description',svalue)
+        self.setDescription(url,svalue)
+        
+        #useful stuff in rest of doc, all contained in this:
+        doc = soup.body.find('div', id='my_wrapper')
+        
+        #first the site category (more of a genre to me, meh) and title, in this element:
+        mt = doc.find('div',attrs={'class':'maintitle'})
+        self.story.addToList('genre',mt.findAll('a')[1].string)
+        self.story.setMetadata('title',mt.findAll('a')[1].nextSibling[len('&nbsp;&raquo;&nbsp;'):])
+        del mt
+
+        #the actual category, for me, is 'Buffy: The Vampire Slayer'
+        #self.story.addToList('category','Buffy: The Vampire Slayer')
+        #No need to do it here, it is better to set it in in plugin-defaults.ini and defaults.ini
+        
+        #then a block that sits in a table cell like so:
+        #(contains a lot of metadata)
+        mblock = doc.find('td', align='left', width = '70%').contents
+        while len(mblock) > 0:
+            i = mblock.pop(0)
+            if 'Author:' in i.string:
+                #drop empty space
+                mblock.pop(0)
+                #get author link
+                a = mblock.pop(0)
+                authre = re.escape('./index.php?act=bio&id=')+'(?P<authid>\d+)'
+                m = re.match(authre,a['href'])
+                self.story.setMetadata('author',a.string)
+                self.story.setMetadata('authorId',m.group('authid'))
+                authurl = u'http://%s/fanfictions/index.php?act=bio&id=%s' % ( self.getSiteDomain(),
+                                            self.story.getMetadata('authorId'))
+                self.story.setMetadata('authorUrl',authurl,condremoveentities=False)
+                #drop empty space
+                mblock.pop(0)
+            if 'Rating:' in i.string:
+                self.story.setMetadata('rating',mblock.pop(0).strip())
+            if 'Published:' in i.string:
+                date = mblock.pop(0).strip()
+                #get rid of 'st', 'nd', 'rd', 'th' after day number
+                date = date[0:2]+date[4:]
+                self.story.setMetadata('datePublished',makeDate(date, dateformat))
+            if 'Last Updated:' in i.string:
+                date = mblock.pop(0).strip()
+                #get rid of 'st', 'nd', 'rd', 'th' after day number
+                date = date[0:2]+date[4:]
+                self.story.setMetadata('dateUpdated',makeDate(date, dateformat))
+            if 'Genre:' in i.string:
+                genres = mblock.pop(0).strip()
+                genres = genres.split('/')
+                for genre in genres: self.story.addToList('genre',genre)
+            #end ifs
+        #end while
+
+        # Find the chapter selector 
+        select = soup.find('select', { 'name' : 'ch' } )
+         
+        if select is None:
+           # no selector found, so it's a one-chapter story.
+           #self.chapterUrls.append((self.story.getMetadata('title'),url))
+           self.chapterUrls.append((self.story.getMetadata('title'),url))
+        else:
+            allOptions = select.findAll('option')
+            for o in allOptions:
+                url = u'http://%s/fanfictions/index.php?act=vie&id=%s&ch=%s' % ( self.getSiteDomain(),
+                                            self.story.getMetadata('storyId'),
+                                            o['value'])
+                title = u"%s" % o
+                title = stripHTML(title)
+                ts = title.split(' ',1)
+                title = ts[0]+'. '+ts[1]
+                self.chapterUrls.append((title,url))
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+        
+        ## Go scrape the rest of the metadata from the author's page.
+        data = self._fetchUrl(self.story.getMetadata('authorUrl'))
+        soup = bs.BeautifulSoup(data)
+        #find the story link and its parent div
+        storya = soup.find('a',{'href':self.story.getMetadata('storyUrl')})
+        storydiv = storya.parent
+        #warnings come under a <spawn> tag. Never seen that before...
+        #appears to just be a line of freeform text, not necessarily a list
+        #optional
+        spawn = storydiv.find('spawn',{'id':'warnings'})
+        if spawn is not None:
+            warns = spawn.nextSibling.strip()
+            self.story.addToList('warnings',warns)
+        #some meta in spans - this should get all, even the ones jammed in a table
+        spans = storydiv.findAll('span')
+        for s in spans:
+            if s.string == 'Ship:':
+                list = s.nextSibling.strip().split()
+                self.story.extendList('ships',list)
+            if s.string == 'Characters:':
+                list = s.nextSibling.strip().split(',')
+                self.story.extendList('characters',list)
+            if s.string == 'Status:':
+                st = s.nextSibling.strip()
+                self.story.setMetadata('status',st)
+            if s.string == 'Words:':
+                st = s.nextSibling.strip()
+                self.story.setMetadata('numWords',st)
+
+        #reviews - is this worth having?
+        #ffnet adapter gathers it, don't know if anything else does
+        #or if it's ever going to be used!
+        a = storydiv.find('a',{'id':'bold-blue'})
+        if a:
+            revs = a.nextSibling.strip()[1:-1]
+            self.story.setMetadata('reviews',st)
+        else:
+            revs = '0'
+            self.story.setMetadata('reviews',st)
+
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'fanfiction'})
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+        
+        #remove all the unnecessary bookmark tags
+        [s.extract() for s in div('div',{'class':"tiny_box2"})]
+        
+        #is there a review link?
+        r = div.find('a',href=re.compile(re.escape("./index.php?act=irv")+".*$"))
+        if r is not None:
+        #remove the review link and its parent div
+            r.parent.extract()
+        
+        #There might also be a link to the sequel on the last chapter
+        #I'm inclined to keep it in, but the URL needs to be changed from relative to absolute
+        #Shame there isn't proper series metadata available
+        #(I couldn't find it anyway)
+        s = div.find('a',href=re.compile(re.escape("./index.php?act=ovr")+".*$"))
+        if s is not None:
+            s['href'] = 'http://'+self.getSiteDomain()+'/fanfictions'+s['href'][1:]
+
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_dotmoonnet.py

@@ -0,0 +1,216 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return DotMoonNetAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class DotMoonNetAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL. www.dotmoon.net/library_view.php?storyid=3
+        self._setURL('http://' + self.getSiteDomain() + '/library_view.php?storyid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','dotm')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%Y-%m-%d"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.dotmoon.net'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/library_view.php?storyid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/library_view.php?storyid=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'You must be logged in to read adult-rated stories' in data \
+                or 'Password incorrect' in data \
+                or "That username does not exist" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['user'] = self.username
+            params['passwrd'] = self.password
+        else:
+            params['user'] = self.getConfig("username")
+            params['passwrd'] = self.getConfig("password")
+
+        loginUrl = 'http://' + self.getSiteDomain() + '/board/index.php'
+        
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['user']))
+    
+        d = self._fetchUrl(loginUrl+'?action=login2&user='+params['user']+'&passwrd='+params['passwrd'])
+        d = self._fetchUrl(loginUrl)
+    
+        if "Show unread posts since last visit" not in d : #Member Account
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['user']))
+            raise exceptions.FailedToLogin(url,params['user'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+            
+        if "Invalid story ID" in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Invalid story ID.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        body=soup.findAll('body')[1]
+        body.find('table').extract()
+        
+        ## Title
+        a = body.find('b')
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url. http://www.dotmoon.net/board/index.php?action=profile;u=1'
+        a = body.find('a', href=re.compile(r"index.php\?action=profile;u=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[2])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters: 'library_storyview.php?chapterid=3
+        chapters=body.findAll('a', href=re.compile(r"library_storyview.php\?chapterid=\d+$"))
+        if len(chapters)==0:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: No php/html chapters found.")
+        if len(chapters)==1:
+            self.chapterUrls.append((self.story.getMetadata('title'),'http://'+self.host+'/'+chapters[0]['href']))
+        else:
+            for chapter in chapters:
+                # just in case there's tags, like <i> in chapter titles.
+                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # other tags
+
+        labels = body.find('table', {'width':'390'}).findAll('td')
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+            
+            if label != None:
+                if 'Fandom' in label:
+                    self.story.addToList('category',value.string)
+                
+                if 'Setting' in label:
+                    self.story.addToList('genre',value.string)
+                
+                if 'Genre' in label:
+                    self.story.addToList('genre',value.string)
+                
+                if 'Style' in label:
+                    self.story.addToList('genre',value.string)
+                
+                if 'Rating' in label:
+                    self.story.addToList('rating',value.string)
+                
+                if 'Created' in label:
+                    self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+                if 'Updated' in label:
+                    self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+        
+                if 'Status' in label:
+                    if 'Completed' in value.string:
+                        self.story.setMetadata('status', 'Completed')
+                    else:
+                        self.story.setMetadata('status', 'In-Progress')
+                        
+        table=body.findAll('table', {'width':'400'})[1].find('td')
+        self.setDescription(url,stripHTML(table).split('Summary: ')[1])
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('blockquote')
+        div.name='div'
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_efictionestelielde.py

@@ -0,0 +1,221 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return EfictionEstelielDeAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class EfictionEstelielDeAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','eesd')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%B %d, %Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'efiction.esteliel.de'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+       
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title and author
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+
+        pagetitle = soup.find('div',{'id':'pagetitle'})
+        ## Title
+        a = pagetitle.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = pagetitle.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        list = soup.find('div', {'class':'listbox'})
+        labelspan=list.find('span',{'class':'label'})
+        value = labelspan.nextSibling
+        label = labelspan.string
+        genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+        for genre in genres:
+            self.story.addToList('genre',genre.string)
+        
+        labels = list.findAll('b')
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while 'Rating' not in str(value):
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rating' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Words' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Category' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            if list.find('a', href=re.compile(r"series.php")) != None:
+                for series in asoup.findAll('a', href=re.compile(r"series.php\?seriesid=\d+")):
+                    # Find Series name from series URL.
+                    series_url = 'http://'+self.host+'/'+series['href']
+                    # use BeautifulSoup HTML parser to make everything easier to find.
+                    seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+                    storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+                    i=1
+                    for a in storyas:
+                        if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                            name=seriessoup.find('div', {'id' : 'pagetitle'})
+                            name.find('a').extract()
+                            self.setSeries(name.text.split(' by[')[0], i)
+                            i=0
+                            break
+                        i+=1
+                    if i == 0:
+                        break
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+                                     
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_efpfanficnet.py

@@ -158,6 +158,7 @@ class EFPFanFicNet(BaseSiteAdapter):
                 self.chapterUrls.append((title,url))
 
         self.story.setMetadata('numChapters',len(self.chapterUrls))
+        self.story.setMetadata('language','Italian')
         
         # normalize story URL to first chapter if later chapter URL was given:
         url = self.chapterUrls[0][1].replace('&i=1','')

fanficdownloader/adapters/adapter_fanfictionnet.py

@@ -68,7 +68,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         return "http://www.fanfiction.net/s/1234/1/ http://www.fanfiction.net/s/1234/12/ http://www.fanfiction.net/s/1234/1/Story_Title"
 
     def getSiteURLPattern(self):
-        return r"http://(www|m)?\.fanfiction\.net/s/\d+(/\d+)?(/|/[a-zA-Z0-9_-]+)?/?$"
+        return r"http://(www|m)?\.fanfiction\.net/s/\d+(/\d+)?(/|/[^/]+)?/?$"
 
     def extractChapterUrlsAndMetadata(self):
 
@@ -215,9 +215,14 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         else:
             self.story.setMetadata('status', 'In-Progress')
 
-        img = soup.find('img',{'class':'cimage'})
-        if img:
-            self.setCoverImage(url,img['src'])
+        # Try the larger image first.
+        try:
+            img = soup.find('img',{'class':'lazy cimage'})
+            self.setCoverImage(url,img['data-original'])
+        except:
+            img = soup.find('img',{'class':'cimage'})
+            if img:
+                self.setCoverImage(url,img['src'])
             
         # Find the chapter selector 
         select = soup.find('select', { 'name' : 'chapter' } )

fanficdownloader/adapters/adapter_ficwadcom.py

@@ -43,10 +43,10 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
 
     @staticmethod
     def getSiteDomain():
-        return 'www.ficwad.com'
+        return 'ficwad.com'
 
     def getSiteExampleURLs(self):
-        return "http://www.ficwad.com/story/137169"
+        return "http://ficwad.com/story/137169"
 
     def getSiteURLPattern(self):
         return re.escape(r"http://"+self.getSiteDomain())+"/story/\d+?$"

fanficdownloader/adapters/adapter_fimfictionnet.py

@@ -28,8 +28,6 @@ from .. import BeautifulSoup as bs
 from ..htmlcleanup import stripHTML
 from .. import exceptions as exceptions
 
-from ..bbcodeutils.bbcodeparser import bbcodeparser
-
 from base_adapter import BaseSiteAdapter,  makeDate
 
 def getClass():
@@ -81,8 +79,10 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
             apiResponse = urllib2.urlopen("http://www.fimfiction.net/api/story.php?story=%s" % (self.story.getMetadata("storyId"))).read()
             apiData = json.loads(apiResponse)
             
-            # Unfortunately, we still need to load the story index page to parse the characters
+            # Unfortunately, we still need to load the story index
+            # page to parse the characters.  And chapters, now, too.
             data = self._fetchUrl(self.url)
+            soup = bs.BeautifulSoup(data)
         except urllib2.HTTPError, e:
             if e.code == 404:
                 raise exceptions.StoryDoesNotExist(self.url)
@@ -95,9 +95,6 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
         if "/images/missing_story.png" in data:
             raise exceptions.StoryDoesNotExist(self.url)
         
-        if "Invalid story id" in apiData.values():
-            raise exceptions.StoryDoesNotExist(self.url)
-        
         if "This story has been marked as having adult content." in data:
             raise exceptions.AdultCheckRequired(self.url)
         
@@ -112,18 +109,39 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
             else:
                 raise exceptions.FailedToLogin(self.url,"Story requires individual password",passwdonly=True)
          
+        if "Invalid story id" in apiData.values():
+            raise exceptions.StoryDoesNotExist(self.url)
+        
         storyMetadata = apiData["story"]    
             
-        self.story.setMetadata("title", storyMetadata["title"])
+        ## Title
+        a = soup.find('a', href=re.compile(r'^/story/'+self.story.getMetadata('storyId')))
+        self.story.setMetadata('title',a.string)
+        
+        # self.story.setMetadata("title", storyMetadata["title"])
+        # if not storyMetadata["title"]:
+        #     raise exceptions.FailedToDownload("%s doesn't have a title in the API.  This is a known fimfiction.net bug with titles containing ."%self.url)
+        
         self.story.setMetadata("author", storyMetadata["author"]["name"])
         self.story.setMetadata("authorId", storyMetadata["author"]["id"])
         self.story.setMetadata("authorUrl", "http://%s/user/%s" % (self.getSiteDomain(), storyMetadata["author"]["name"]))
+
+        # chapters = [{"chapterTitle": chapter["title"], "chapterURL": chapter["link"]} for chapter in storyMetadata["chapters"]]
         
-        chapters = [{"chapterTitle": chapter["title"], "chapterURL": chapter["link"]} for chapter in storyMetadata["chapters"]]
-        for chapter in chapters:
-            self.chapterUrls.append((chapter["chapterTitle"], chapter["chapterURL"]))
-        self.story.setMetadata("numChapters", len(self.chapterUrls))
+        # ## this is bit of a kludge based on the assumption all the
+        # ## 'bad' chapters will be at the end.
+        # ## limit down to the number of chapters reported by chapter_count.
+        # chapters = chapters[:storyMetadata["chapter_count"]] 
         
+        # for chapter in chapters:
+        #     self.chapterUrls.append((chapter["chapterTitle"], chapter["chapterURL"]))
+        # self.story.setMetadata("numChapters", len(self.chapterUrls))
+
+        for chapter in soup.findAll('a',{'class':'chapter_link'}):
+            self.chapterUrls.append((stripHTML(chapter), 'http://'+self.host+chapter['href']))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+            
         # In the case of fimfiction.net, possible statuses are 'Completed', 'Incomplete', 'On Hiatus' and 'Cancelled'
         # For the sake of bringing it in line with the other adapters, 'Incomplete' becomes 'In-Progress'
         # and 'Complete' beomes 'Completed'. 'Cancelled' seems an important enough (not to mention more strictly true) 
@@ -133,7 +151,17 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
         status = storyMetadata["status"].replace("Incomplete", "In-Progress").replace("Complete", "Completed")
         self.story.setMetadata("status", status)
         self.story.setMetadata("rating", storyMetadata["content_rating_text"])
-            
+
+        ## Warnings aren't included in the API.
+        bottomli = soup.find('li',{'class':'bottom'})
+        if bottomli:
+            bottomspans = bottomli.findAll('span')
+            # the first span in bottom is the rating, obtained above.
+            if bottomspans and len(bottomspans) > 1:
+                for warning in bottomspans[1:]:
+                    self.story.addToList('warnings',warning.string)
+                
+        
         for category in storyMetadata["categories"]:
             if storyMetadata["categories"][category]:
                 self.story.addToList("genre", category) 
@@ -146,14 +174,16 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
                 coverurl = storyMetadata["full_image"]
             else:
                 coverurl = storyMetadata["image"]
-            if coverurl.startswith('//static.fimfiction.net'): # fix for img urls missing 'http:'
+            if coverurl.startswith('//'): # fix for img urls missing 'http:'
                 coverurl = "http:"+coverurl
+
             self.setCoverImage(self.url,coverurl)
 
-        # the fimfic API gives bbcode for desc, not html.
-        # btw, bbcode honors newlines, html doesn't.  change newlines to br tags.
-        self.setDescription(self.url,
-                            bbcodeparser().parse(storyMetadata["description"]).html(doDeepCopy=False).replace('\r','').replace('\n','<br />'))
+        # fimf has started including extra stuff inside the description div.
+        descdivstr = "%s"%soup.find("div", {"class":"description"})
+        hrstr="<hr />"
+        descdivstr = '<div class="description">'+descdivstr[descdivstr.index(hrstr)+len(hrstr):]
+        self.setDescription(self.url,descdivstr)
         
         # Dates are in Unix time
         # Take the publish date from the first chapter posted
@@ -162,11 +192,11 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
         rawDateUpdated = storyMetadata["date_modified"]
         self.story.setMetadata("dateUpdated", datetime.fromtimestamp(rawDateUpdated))
 
-        soup = bs.BeautifulSoup(data).find("div", {"class":"story"})
+        chars = soup.find("div", {"class":"inner_data"})
         # fimfic stopped putting the char name on or around the char
         # icon now for some reason.  Pull it from the image name with
         # some heuristics.
-        for character in [character_icon["src"] for character_icon in soup.findAll("img", {"class":"character_icon"})]:
+        for character in [character_icon["src"] for character_icon in chars.findAll("img", {"class":"character_icon"})]:
             # //static.fimfiction.net/images/characters/twilight_sparkle.png
             # 5th split /, remove last four, replace _, capitolize every word(title())
             char = character.split('/')[5][:-4].replace('_',' ').title()
@@ -188,7 +218,7 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
             
     def getChapterText(self, url):
         logger.debug('Getting chapter text from: %s' % url)
-        soup = bs.BeautifulSoup(self._fetchUrl(url),selfClosingTags=('br','hr')).find('div', {'id' : 'chapter_container'})
+        soup = bs.BeautifulSoup(self._fetchUrl(url),selfClosingTags=('br','hr')).find('div', {'class' : 'chapter_content'})
         if soup == None:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
         return self.utf8FromSoup(url,soup)

fanficdownloader/adapters/adapter_hennethannunnet.py

@@ -0,0 +1,171 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return HennethAnnunNetAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class HennethAnnunNetAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/stories/chapter.cfm?stid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','htan')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%m/%d/%y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.henneth-annun.net'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/stories/chapter.cfm?stid=1234"
+
+    def getSiteURLPattern(self):
+        return "http://"+self.getSiteDomain()+"/stories/chapter(_view)?.cfm\?stid="+r"\d+$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+            
+        if "We're sorry. This story is not available." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: This story is not available.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('h2', {'id':'page_heading'})
+        self.story.setMetadata('title',a.string)
+
+        # Find the chapters: chapter_view.cfm?stid=6663&spordinal=1" 
+        for chapter in soup.findAll('a', href=re.compile(r'chapter_view.cfm\?stid='+self.story.getMetadata('storyId')+"&spordinal=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/stories/'+chapter['href']))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+        
+        self.story.setMetadata('numWords', soup.find('tr', {'class':'foot'}).findAll('td')[1].text)
+
+        self.setDescription(url,soup.find('div', {'id':'summary'}))
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        info = soup.find('div', {'id':'storyinformation'})
+        labels=info.findAll('b')
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+                
+            if 'Completion' in label:
+                if 'Complete' in value.string:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Rating' in label:
+                self.story.setMetadata('rating', value.string)
+
+            if 'Era:' in label:
+                self.story.addToList('category',value.string)
+
+            if 'Genre' in label:
+                self.story.addToList('genre',value.string)
+                
+        labels=info.findAll('strong')
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+            
+            if 'Author' in label:
+                value=value.nextSibling
+                self.story.setMetadata('authorId',value['href'].split('=')[1])
+                self.story.setMetadata('authorUrl','http://'+self.host+'/'+value['href'])
+                self.story.setMetadata('author',value.string)
+                
+            if 'Post' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated:' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+                
+        for char in soup.findAll('a', href=re.compile(r"/resources/bios_view.cfm\?scid=\d+")):
+            self.story.addToList('characters',stripHTML(char))
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'class' : 'block chapter'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_imagineeficcom.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-# Copyright 2012 Fanficdownloader team
+# Copyright 2013 Fanficdownloader team
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -28,22 +28,15 @@ from .. import exceptions as exceptions
 from base_adapter import BaseSiteAdapter,  makeDate
 
 def getClass():
-    return YourFanfictionComAdapter
+    return ImagineEFicComAdapter
 
 # Class name has to be unique.  Our convention is camel case the
 # sitename with Adapter at the end.  www is skipped.
-class YourFanfictionComAdapter(BaseSiteAdapter):
+class ImagineEFicComAdapter(BaseSiteAdapter):
 
     def __init__(self, config, url):
         BaseSiteAdapter.__init__(self, config, url)
 
-        # yourfanfiction.com blocks the default user-agent.  However,
-        # when asked, they said it was just general anti-spam, not
-        # targeted as us and offered to 'whitelist our IP'.  Clearly,
-        # that wouldn't work, but it does let me do this in good
-        # conscience:
-        self.opener.addheaders = [('User-agent', 'FFDL/1.6')]
-
         self.decode = ["Windows-1252",
                        "utf8"] # 1252 is a superset of iso-8859-1.
                                # Most sites that claim to be
@@ -61,16 +54,16 @@ class YourFanfictionComAdapter(BaseSiteAdapter):
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
         
         # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','yff')
+        self.story.setMetadata('siteabbrev','ime')
 
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%d %b %Y"
+        self.dateformat = "%Y.%m.%d"
             
     @staticmethod # must be @staticmethod, don't remove it.
     def getSiteDomain():
         # The site domain.  Does have www here, if it uses it.
-        return 'www.yourfanfiction.com'
+        return 'imagine.e-fic.com'
 
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
@@ -78,6 +71,41 @@ class YourFanfictionComAdapter(BaseSiteAdapter):
     def getSiteURLPattern(self):
         return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
 
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Registered Users Only' in data \
+                or 'There is no such account on our website' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['cookiecheck'] = '1'
+        params['submit'] = 'Submit'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "Member Account" not in d : #Member Account
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
     ## Getting the chapter list and the meta data, plus 'is adult' checking.
     def extractChapterUrlsAndMetadata(self):
 
@@ -103,20 +131,12 @@ class YourFanfictionComAdapter(BaseSiteAdapter):
             else:
                 raise e
 
-        # The actual text that is used to announce you need to be an
-        # adult varies from site to site.  Again, print data before
-        # the title search to troubleshoot.
-            
-        # Since the warning text can change by warning level, let's
-        # look for the warning pass url.  ksarchive uses
-        # &warning= -- actually, so do other sites.  Must be an
-        # eFiction book.
-            
-        # viewstory.php?sid=1882&warning=4
-        # viewstory.php?sid=1654&ageconsent=ok&warning=5
-        #print data
-        #m = re.search(r"'viewstory.php\?sid=1882(&warning=4)'",data)
-        m = re.search(r"'viewstory.php\?sid=\d+((&ageconsent=ok)?&warning=\d+)'",data)
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
         if m != None:
             if self.is_adult or self.getConfig("is_adult"):
                 # We tried the default and still got a warning, so
@@ -124,8 +144,7 @@ class YourFanfictionComAdapter(BaseSiteAdapter):
                 # link and reload data.
                 addurl = m.group(1)
                 # correct stupid & error in url.
-                # explicitly put ageconsent because google appengine regexp doesn't include it for some reason.
-                addurl = addurl.replace("&","&")+'&ageconsent=ok'
+                addurl = addurl.replace("&","&")
                 url = self.url+'&index=1'+addurl
                 logger.debug("URL 2nd try: "+url)
 
@@ -142,17 +161,9 @@ class YourFanfictionComAdapter(BaseSiteAdapter):
         if "Access denied. This story has not been validated by the adminstrators of this site." in data:
             raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
             
-        # because for some reason, this works while simple 'print data' errors on ascii conversion.
-        # loopdata = data
-        # chklen=5000
-        # while len(loopdata) > 0:
-        #     if len(loopdata) < 5000:
-        #         chklen = len(loopdata)
-        #     logger.info("loopdata: %s" % loopdata[:chklen])
-        #     loopdata = loopdata[chklen:]
-            
         # use BeautifulSoup HTML parser to make everything easier to find.
         soup = bs.BeautifulSoup(data)
+        # print data
 
         # Now go hunting for all the meta data and the chapter list.
         
@@ -182,6 +193,7 @@ class YourFanfictionComAdapter(BaseSiteAdapter):
                 return d[k]
             except:
                 return ""
+                
 
         # <span class="label">Rated:</span> NC-17<br /> etc
         labels = soup.findAll('span',{'class':'label'})
@@ -192,11 +204,9 @@ class YourFanfictionComAdapter(BaseSiteAdapter):
             if 'Summary' in label:
                 ## Everything until the next span class='label'
                 svalue = ""
-                while value and not defaultGetattr(value,'class') == 'label':
+                while not defaultGetattr(value,'class') == 'label':
                     svalue += str(value)
                     value = value.nextSibling
-                # sometimes poorly formated desc (<p> w/o </p>) leads
-                # to all labels being included.
                 self.setDescription(url,svalue)
                 #self.story.setMetadata('description',stripHTML(svalue))
 
@@ -217,17 +227,12 @@ class YourFanfictionComAdapter(BaseSiteAdapter):
                     self.story.addToList('characters',char.string)
 
             if 'Genre' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=5'))
-                for genre in genres:
-                    self.story.addToList('genre',genre.string)
-                    
-            if 'Tags' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=7'))
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
                 for genre in genres:
                     self.story.addToList('genre',genre.string)
 
             if 'Warnings' in label:
-                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=6'))
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
                 for warning in warnings:
                     self.story.addToList('warnings',warning.string)
 
@@ -241,8 +246,6 @@ class YourFanfictionComAdapter(BaseSiteAdapter):
                 self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
             
             if 'Updated' in label:
-                # there's a stray [ at the end.
-                #value = value[0:-1]
                 self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
 
         try:

fanficdownloader/adapters/adapter_pommedesangcom.py

@@ -0,0 +1,298 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return PommeDeSangComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class PommeDeSangComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # pommedesang.com has two 'sections', shown in URL as
+        # 'efiction' and 'sds' that change how things should be
+        # handled.
+        # http://pommedesang.com/efiction/viewstory.php?sid=1234
+        # http://pommedesang.com/sds/viewstory.php?sid=1234
+        self.section=self.parsedUrl.path.split('/',)[1]
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/'+self.section+'/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','pmds')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        if 'efiction' in self.section:
+            self.dateformat = "%b %d, %Y"
+        else:
+            self.dateformat = "%m/%d/%y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'pommedesang.com'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/efiction/viewstory.php?sid=1234 http://"+self.getSiteDomain()+"/sds/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return r"http://"+self.getSiteDomain()+"/(efiction|sds)?/viewstory.php\?sid=\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Registered Users Only' in data \
+                or 'There is no such account on our website' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['cookiecheck'] = '1'
+        params['submit'] = 'Submit'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/'+self.section+'/user.php?action=login'
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "Member Account" not in d : #Member Account
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=5"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid & error in url.
+                addurl = addurl.replace("&","&")
+                url = self.url+'&index=1'+addurl
+                logger.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile('viewstory.php\?sid=\d+'))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+self.section+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+                
+# summary, rated, word count, categories, characters, genre, warnings, completed, published, updated, seires
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) # XXX
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+self.section+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
+            storyas = seriessoup.findAll('a', href=re.compile('viewstory.php\?sid=\d+'))
+            i=1
+            for a in storyas:
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if ('viewstory.php?sid='+self.story.getMetadata('storyId')) in a['href']:
+                        self.setSeries(series_name, i)
+                        break
+                    i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_potterficscom.py

@@ -0,0 +1,247 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import datetime
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter
+
+# This function is called by the downloader in all adapter_*.py files
+# in this dir to register the adapter class.  So it needs to be
+# updated to reflect the class below it.  That, plus getSiteDomain()
+# take care of 'Registering'.
+def getClass():
+    return PotterFicsComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class PotterFicsComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+        
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query correct
+        m = re.match(self.getSiteURLPattern(),url)
+        if m:
+            self.story.setMetadata('storyId',m.group('id'))
+            logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+            # normalized story URL. gets rid of chapter if there, left with chapter index URL
+            nurl = "http://"+self.getSiteDomain()+"/historias/"+self.story.getMetadata('storyId')
+            self._setURL(nurl)
+        else:
+            raise exceptions.InvalidStoryURL(url,
+                                             self.getSiteDomain(),
+                                             self.getSiteExampleURLs())
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','potficscom')
+
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.potterfics.com'
+
+    def getSiteExampleURLs(self):
+        return \
+                "http://www.potterfics.com/historias/127583 "\
+                "http://www.potterfics.com/historias/127583/capitulo-1 "\
+                "http://www.potterfics.com/historias/127583/capitulo-4 "\
+                "http://www.potterfics.com/historias/92810 "\
+                "http://www.potterfics.com/historias/111194"
+
+    def getSiteURLPattern(self):
+        #http://www.potterfics.com/historias/127583
+        #http://www.potterfics.com/historias/127583/capitulo-1
+        #http://www.potterfics.com/historias/127583/capitulo-4
+        #http://www.potterfics.com/historias/92810              -> Complete story
+        #http://www.potterfics.com/historias/111194             -> Complete, single chap
+        p = re.escape("http://"+self.getSiteDomain()+"/historias/")+\
+            r"(?P<id>\d+)(/capitulo-(?P<ch>\d+))?/?$"
+        return p
+
+    def extractChapterUrlsAndMetadata(self):
+
+        #this converts '/historias/12345' to 'http://www.potterfics.com/historias/12345'
+        def makeAbsoluteURL(url):
+            if url[0] == '/':
+                url = 'http://'+self.getSiteDomain()+url
+            return url
+
+        #use this to get month numbers from Spanish months
+        SpanishMonths = {
+            'enero'      : '01',
+            'febrero'    : '02',
+            'marzo'      : '03',
+            'abril'      : '04',
+            'mayo'       : '05',
+            'junio'      : '06',
+            'julio'      : '07',
+            'agosto'     : '08',
+            'septiembre' : '09',
+            'octubre'    : '10',
+            'noviembre'  : '11',
+            'diciembre'  : '12'
+            }
+
+        url = self.url
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        #print data
+
+        #deal with adult content warnings - doesn't seem to apply to this site
+
+        #set constant meta for this site:
+        #Set Language = Spanish
+        self.story.setMetadata('language', 'Spanish')
+        #Set Category = Harry Potter
+        #  This is better done in plugin-defaults.ini and defaults.ini
+        #  by adding a section for this site with the line:
+        #  extracategories:Harry Potter
+        #self.story.addToList('category','Harry Potter')
+        
+        #get the rest of the meta
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        #self closing br and img present!
+        soup = bs.BeautifulSoup(data,selfClosingTags=('br','img'))
+    
+        #we want the second table directly under the body, contains all the metadata
+        table = soup.html.body.findAll('table', recursive=False)[1]
+        #within that, we want the second row, first cell
+        cell = table.tr.findNextSibling('tr').td
+
+        #find first metadata block
+        mb = cell.div.findNextSibling('div')
+        #Get meta...
+        self.story.setMetadata('title', mb.b.string)
+        #strip out brackets on rating
+        self.story.setMetadata('rating', mb.span.string[1:-1])
+        #Completion status is denoted by the presence of this image:
+        if mb.find('img',title="Historia terminada"):
+            self.story.setMetadata('status', 'Completed')
+        else:
+            self.story.setMetadata('status', 'In-Progress')
+
+        #find next metadata block
+        #author details
+        mb = mb.findNextSibling('div')
+        self.story.setMetadata('author', mb.b.a.string.strip())
+        self.story.setMetadata('authorUrl', makeAbsoluteURL(mb.b.a['href']))
+        self.story.setMetadata('authorId', self.story.getMetadata('authorUrl').split('/')[4])
+        #dates and times
+        mb = mb.find('span')
+        #posted/published = Escrita
+        date = mb.find(text=re.compile('Escrita el ')).strip().split()
+        year = int(date[7][:-1]) # need to remove the last char from year, it is a comma
+        month = int(SpanishMonths[date[5].lower()])
+        day = int(date[3])
+        time = date[8].split(':')
+        hour = int(time[0])
+        minute = int(time[1])
+        self.story.setMetadata('datePublished', datetime.datetime(year, month, day, hour, minute))
+        #updated = Actualizada
+        date = mb.find(text=re.compile('Actualizada el ')).strip().split()
+        year = int(date[7][:-1]) # need to remove the last char from year, it is a comma
+        month = int(SpanishMonths[date[5].lower()])
+        day = int(date[3])
+        time = date[8].split(':')
+        hour = int(time[0])
+        minute = int(time[1])
+        self.story.setMetadata('dateUpdated', datetime.datetime(year, month, day, hour, minute))
+        
+        mb = mb.span.findNextSibling('span').findNextSibling('span')
+        wc = mb.find(text=re.compile(' palabras en total')).strip()
+        self.story.setMetadata('numWords', wc.split()[0])
+        
+        #then we come to categories and genres. Oh dear. On this site, categories hold everything from genre, to ships, to crossovers.
+        #To make things worse, there is also another genre field, which often holds similar/duplicate info. Links to genre pages do not work
+        #though, so perhaps those will be phased out?
+        #for now, put them all into the genre list
+        links = mb.findAll('a',href=re.compile('/(categorias|generos)/\d+'))
+        genlist = [i.string.strip() for i in links]
+        self.story.extendList('genre',genlist)
+        
+        #get the chapter urls
+        #we can go back to the table cell we found before
+        #get its last element and work backwards to find the last ordered list on the page
+        list = cell.contents[len(cell)-1].findPrevious('ol')
+        chapters = []
+        revs = 0
+        chnum = 0
+        for li in list:
+            chnum += 1
+            chTitle = str(chnum) + '. ' + li.a.b.string.strip()
+            chURL = makeAbsoluteURL(li.a['href'])
+            chapters.append((chTitle,chURL))
+            #Get reviews, add to total
+            revs += int(li.div.a.string.split()[0])
+        
+        self.chapterUrls.extend(chapters)
+        self.story.setMetadata('numChapters', len(chapters))
+        self.story.setMetadata('reviews', revs)
+        
+        #Now for the description... this may be tricky...
+        #if it is there (doesn't have to be), it will be before the chapter list,
+        #separated by a horizontal rule, and after the google ad bar
+        
+        #get list's parent div
+        mb = list.parent
+        #get the div before that, will either be the description, or the google ad bar
+        mb = mb.findPreviousSibling('div')
+        if 'google_ad_client' in str(mb):
+            #couldn't find description, leaving it blank
+            pass
+        else:
+            self.setDescription(url,mb)
+        
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr','img'))
+        
+        div = soup.find('div', {'id' : 'cuerpoHistoria'})
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+        
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_psychficcom.py

@@ -0,0 +1,246 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return PsychFicComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class PsychFicComAdapter(BaseSiteAdapter): 
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','psyf')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%B %d, %Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.psychfic.com'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=4"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid & error in url.
+                addurl = addurl.replace("&","&")
+                url = self.url+'&index=1'+addurl
+                logger.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.text
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            i=1
+            for a in storyas:
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        break
+                    i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_restrictedsectionorg.py

@@ -0,0 +1,264 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+import cookielib as cl
+from datetime import datetime
+import json
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return RestrictedSectionOrgSiteAdapter
+
+class RestrictedSectionOrgSiteAdapter(BaseSiteAdapter):
+    
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+            
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        
+        # normalized story URL.
+        # get story/file and storyId from url--url validation guarantees query correct
+        m = re.match(self.getSiteURLPattern(),url)
+        if m:
+            self.story.setMetadata('storyId',m.group('id'))
+            logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+            # normalized story URL.
+            self._setURL('http://' + self.getSiteDomain() + '/' + m.group('filestory') + '.php?' + m.group('filestory') + '=' + self.story.getMetadata('storyId'))
+            logger.debug("storyUrl: (%s)"%self.story.getMetadata('storyUrl'))
+        else:
+            raise exceptions.InvalidStoryURL(url,
+                                             self.getSiteDomain(),
+                                             self.getSiteExampleURLs())
+        
+        self.story.setMetadata('siteabbrev','ressec')
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%d %b %Y" # 20 Nov 2005
+        
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        return 'www.restrictedsection.org'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/story.php?story=1234 http://"+self.getSiteDomain()+"/file.php?file=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain())+r"/(?P<filestory>file|story).php\?(file|story)=(?P<id>\d+)$"
+        
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        url = self.url
+        logger.debug("URL: "+url)
+
+        # one-shot stories use file url instead of story.  'Luckily',
+        # we don't have to worry about one-shots becoming
+        # multi-chapter because ressec is frozen.  Still need 'story'
+        # url for metadata, however.
+        try:
+            if 'file' in url:
+                data = self._postUrlUP(url)
+                soup = bs.BeautifulSoup(data)
+                storya = soup.find('a',href=re.compile(r"^story.php\?story=\d+"))
+                url = 'http://'+self.host+'/'+storya['href'].split('&')[0]  # strip rs_session
+
+                fileas = soup.find('a',href=re.compile(r"^file.php\?file=\d+"))
+                if fileas:
+                    for filea in fileas:
+                        if 'Previous Chapter' in filea.string or 'Next Chapter' in filea.string:
+                            raise exceptions.FailedToDownload(self.getSiteDomain() +" Cannot use chapter url with multi-chapter stories on this site.")
+                    
+            logger.debug("metadata URL: "+url)
+            data = self._fetchUrl(url)
+            # print data
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+        if "Story not found" in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Story not found.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+
+        # check user/pass on a chapter for multi-chapter
+        if 'file' not in self.url:
+            self._postUrlUP('http://'+self.host+'/'+soup.find('a', href=re.compile(r"^file.php\?file=\d+"))['href'])
+
+        ## Title
+        h2 = soup.find('h2')
+
+        # Find authorid and URL from... author url.
+        a = h2.find('a')
+        ahref = a['href'].split('&')[0]  # strip rs_session
+        
+        self.story.setMetadata('authorId',ahref.split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+ahref)
+        self.story.setMetadata('author',a.string)
+
+        # title, remove byauthorname.
+        self.story.setMetadata('title',h2.text[:h2.text.index("by"+a.string)])
+
+        dates = soup.findAll('span', {'class':'date'})
+        if dates: # only for multi-chapter
+            self.story.setMetadata('datePublished', makeDate(stripHTML(dates[0]), self.dateformat))
+            self.story.setMetadata('dateUpdated', makeDate(stripHTML(dates[-1]), self.dateformat))
+
+        words = soup.findAll('span', {'class':'size'})
+        wordcount=0
+        for w in words:
+            wordcount = wordcount + int(w.string[:-6].replace(',',''))
+
+        self.story.setMetadata('numWords',"%s"%wordcount)
+
+        self.story.setMetadata('rating', soup.find('a',href=re.compile(r"^rating.php\?rating=\d+")).string)
+        
+        # other tags
+
+        labels = soup.find('table', {'class':'info'}).findAll('th')
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+            
+            if label != None:
+                
+                if 'Categories' in label:
+                    for g in stripHTML(value).split('\n'):
+                        self.story.addToList('genre',g)
+                
+                if 'Pairings' in label:
+                    for g in stripHTML(value).split('\n'):
+                        self.story.addToList('ships',g)
+                
+                if 'Summary' in label:
+                    self.setDescription(url,stripHTML(value).replace("\n"," ").replace("\r",""))
+                    value.extract() # remove summary incase it contains file URLs.
+
+                if 'Updated' in label: # one-shots only.
+                    print "value:%s"%value
+                    value.find('sup').extract() # remove 'st', 'nd', 'th' ordinals
+                    print "value:%s"%value
+                    date = makeDate(stripHTML(value), '%d %B %Y') # full month name
+                    self.story.setMetadata('datePublished', date)
+
+                if 'Length' in label: # one-shots only.
+                    self.story.setMetadata('numWords',value.string[:-6])
+                    
+        # one-shot.
+        if 'file' in self.url:
+            self.chapterUrls.append((self.story.getMetadata('title'),self.url))
+        else: # multi-chapter
+            # Find the chapters: 'library_storyview.php?chapterid=3
+            chapters=soup.findAll('a', href=re.compile(r"^file.php\?file=\d+"))
+            if len(chapters)==0:
+                raise exceptions.FailedToDownload(self.getSiteDomain() +" says: No chapters found.")
+            else:
+                for chapter in chapters:
+                    chhref = chapter['href'].split('&')[0] # strip rs_session
+                    # just in case there's tags, like <i> in chapter titles.
+                    self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chhref))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+
+
+    def _postUrlUP(self, url):
+        params = {}
+        if self.password:
+            params['username'] = self.username
+            params['password'] = self.password
+        else:
+            params['username'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['accept.x'] = 1
+        params['accept.y'] = 1
+
+        excpt=None
+        for sleeptime in [0.5, 1.5, 4, 9]:
+            time.sleep(sleeptime)	
+            try:
+                data = self._postUrl(url, params)
+                if data == "Unable to connect to the database":
+                    raise exceptions.FailedToDownload("Site reported 'Unable to connect to the database'")
+                if "I certify that I am over the age of 18 and that accessing the following story will not violate the laws of my country or local ordinances." in data:
+                    raise exceptions.FailedToLogin(url,params['username'])
+                return data
+            except exceptions.FailedToLogin, ftl:
+                # no need to retry these.
+                raise(ftl)
+            except Exception, e:
+                excpt=e
+                logger.warn("Caught an exception reading URL: %s  Exception %s."%(unicode(url),unicode(e)))
+
+        logger.error("Giving up on %s" %url)
+        logger.exception(excpt)
+        raise(excpt)
+        
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        data = self._postUrlUP(url)
+        #print("data:%s"%data)
+        
+        # some stories have html that confuses the parser.  For story
+        # text we don't care about anything before '<table id="page"'
+        # and seems to clear the issue.
+        data = data[data.index('<table id="page"'):]
+        
+        soup = bs.BeautifulSoup(data)
+        
+        div = soup.find('td',{'id':'page_content'})
+        div.name='div'
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+
+        ## Remove stuff from page_content
+
+        # Remove all tags before the first <hr> after class=info table (including hr)
+        hr = div.find('table',{'class':'info'}).findNext('hr')
+        for tag in hr.findAllPrevious():
+            tag.extract()
+        hr.extract()
+        
+        # Remove all tags after the last <hr> (including hr)
+        hr = div.findAll('hr')[-1]
+        for tag in hr.findAllNext():
+            tag.extract()
+        hr.extract()
+        
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_squidgeorgpeja.py

@@ -76,7 +76,7 @@ class SquidgeOrgPejaAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'www.squidge.org'
     
-    @classmethod # must be @staticmethod, don't remove it.
+    @classmethod # must be @classmethod, don't remove it.
     def getConfigSection(cls):
         # The config section name.  Only override if != site domain.
         return cls.getSiteDomain()+'/peja'

fanficdownloader/adapters/adapter_test1.py

@@ -101,9 +101,10 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
                 }
             self.story.setMetadata('language',langs[idnum%len(langs)])
             self.setSeries('The Great Test',idnum)
-        
+        if idnum == 0:
+            self.setSeries("A Nook Hyphen Test "+self.story.getMetadata('dateCreated'),idnum)
+            
         self.story.setMetadata('rating','Tweenie')
-        
 
         if self.story.getMetadata('storyId') == '673':
             self.story.addToList('author','Author From List')
@@ -114,6 +115,10 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
         
             self.story.addToList('authorUrl','http://author/url')
             self.story.addToList('authorUrl','http://author/url-2')
+            self.story.addToList('category','Power Rangers')
+            self.story.addToList('category','SG-1')
+            self.story.addToList('genre','Porn')
+            self.story.addToList('genre','Drama')
         else:
             self.story.setMetadata('authorId','98765')
             self.story.setMetadata('authorUrl','http://author/url')
@@ -121,21 +126,36 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
         self.story.addToList('warnings','Swearing')
         self.story.addToList('warnings','Violence')
 
-        self.story.addToList('category','Harry Potter')
-        self.story.addToList('category','Furbie')
-        self.story.addToList('category','Crossover')
-        self.story.addToList('category',u'Puella Magi Madoka Magica/魔法少女まどか★マギカ')
-        self.story.addToList('category',u'Magical Girl Lyrical Nanoha')
-
+        if self.story.getMetadata('storyId') == '80':
+            self.story.addToList('category',u'Rizzoli & Isles')
+            self.story.addToList('characters','J. Rizzoli')
+        elif self.story.getMetadata('storyId') == '81':
+            self.story.addToList('category',u'Pitch Perfect')
+            self.story.addToList('characters','Chloe B.')
+        elif self.story.getMetadata('storyId') == '83':
+            self.story.addToList('category',u'Rizzoli & Isles')
+            self.story.addToList('characters','J. Rizzoli')
+            self.story.addToList('category',u'Pitch Perfect')
+            self.story.addToList('characters','Chloe B.')
+        elif self.story.getMetadata('storyId') == '82':
+            self.story.addToList('characters','Henry (Once Upon a Time)')        
+            self.story.addToList('category',u'Once Upon a Time (TV)')
+        else:
+            self.story.addToList('category','Harry Potter')
+            self.story.addToList('category','Furbie')
+            self.story.addToList('category','Crossover')
+            self.story.addToList('category',u'Puella Magi Madoka Magica/魔法少女まどか★マギカ')
+            self.story.addToList('category',u'Magical Girl Lyrical Nanoha')
+            self.story.addToList('category',u'Once Upon a Time (TV)')
+            self.story.addToList('characters','Bob Smith')
+            self.story.addToList('characters','George Johnson')
+            self.story.addToList('characters','Fred Smythe')
+            
         self.story.addToList('genre','Fantasy')
         self.story.addToList('genre','Comedy')
         self.story.addToList('genre','Sci-Fi')
         self.story.addToList('genre','Noir')
-        
-        self.story.addToList('characters','Bob Smith')
-        self.story.addToList('characters','George Johnson')
-        self.story.addToList('characters','Fred Smythe')
-        
+                
         self.story.addToList('listX','xVal1')
         self.story.addToList('listX','xVal2')
         self.story.addToList('listX','xVal3')
@@ -161,9 +181,9 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
                             ('Chapter 3, Over Cinnabar',self.url+"&chapter=4"),
                             ('Chapter 4',self.url+"&chapter=5"),
                             ('Chapter 5',self.url+"&chapter=6"),
-                            #('Chapter 6',self.url+"&chapter=7"),
-                            #('Chapter 7',self.url+"&chapter=8"),
-                            #('Chapter 8',self.url+"&chapter=9"),
+                            ('Chapter 6',self.url+"&chapter=7"),
+                            ('Chapter 7',self.url+"&chapter=8"),
+                            ('Chapter 8',self.url+"&chapter=9"),
                             #('Chapter 9',self.url+"&chapter=0"),
                             #('Chapter 0',self.url+"&chapter=a"),
                             #('Chapter a',self.url+"&chapter=b"),
@@ -186,9 +206,6 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
 
     def getChapterText(self, url):
         logger.debug('Getting chapter text from: %s' % url)
-        if self.story.getMetadata('storyId') == '667':
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!" % url)
-
         if self.story.getMetadata('storyId').startswith('670') or \
                 self.story.getMetadata('storyId').startswith('672'):
             time.sleep(1.0)
@@ -201,20 +218,40 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
 <p>http://test1.com?sid=664 - Crazy string title</p>
 <p>http://test1.com?sid=665 - raises AdultCheckRequired</p>
 <p>http://test1.com?sid=666 - raises StoryDoesNotExist</p>
-<p>http://test1.com?sid=667 - raises FailedToDownload on chapter 1</p>
+<p>http://test1.com?sid=667 - raises FailedToDownload on chapters 2+</p>
 <p>http://test1.com?sid=668 - raises FailedToLogin unless username='Me'</p>
 <p>http://test1.com?sid=669 - Succeeds with Updated Date=now</p>
 <p>http://test1.com?sid=670 - Succeeds, but sleeps 2sec on each chapter</p>
+
+
+
+
 <p>http://test1.com?sid=671 - Succeeds, but sleeps 2sec metadata only</p>
 <p>http://test1.com?sid=672 - Succeeds, quick meta, sleeps 2sec chapters only</p>
-<p>http://test1.com?sid=673 - Succeeds, multiple authors</p>
+<p>http://test1.com?sid=673 - Succeeds, multiple authors, extra categories, genres</p>
+<p>http://test1.com?sid=0 - Succeeds, generates some text specifically for testing hyphenation problems with Nook STR/STRwG</p>
 <p>Odd sid's will be In-Progress, evens complete.  sid&lt;10 will be assigned one of four languages and included in a series.</p>
 </div>
+'''
+        elif self.story.getMetadata('storyId') == '0':
+            text=u'''
+<h3>45. Pronglet Returns to Hogwarts: Chapter 7</h3>
+<br />
+    eyes… but I’m not convinced we should automatically<br />
+<br /><br />
+<b>Thanks to the latest to recommend me: Alastor</b><br />
+<br /><br />
+    “Sure, invite her along. Does she have children?”<br />
+<br />
 '''
         else:
+            if self.story.getMetadata('storyId') == '667':
+                raise exceptions.FailedToDownload("Error downloading Chapter: %s!" % url)
+
             text=u'''
 <div>
 <h3>Chapter title from site</h3>
+<p>Timestamp:'''+datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")+'''</p>
 <p>Lorem '''+self.crazystring+u''' <i>italics</i>, <b>bold</b>, <u>underline</u> consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p>
 br breaks<br><br>
 Puella Magi Madoka Magica/魔法少女まどか★マギカ

fanficdownloader/adapters/adapter_thewriterscoffeeshopcom.py

@@ -48,7 +48,7 @@ class TheWritersCoffeeShopComSiteAdapter(BaseSiteAdapter):
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/library/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        self.dateformat = "%B %d, %Y"
+        self.dateformat = "%d %b %Y"
 
             
     @staticmethod

fanficdownloader/adapters/adapter_tthfanficorg.py

@@ -100,8 +100,8 @@ class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter):
     
         if "Stories Published" not in d : #Member Account
             logger.info("Failed to login to URL %s as %s" % (loginUrl,
-                                                              params['penname']))
-            raise exceptions.FailedToLogin(url,params['penname'])
+                                                             params['urealname']))
+            raise exceptions.FailedToLogin(self.url,params['urealname'])
             return False
         else:
             return True

fanficdownloader/adapters/base_adapter.py

@@ -255,6 +255,13 @@ class BaseSiteAdapter(Configurable):
         "Only needs to be overriden if != site domain."
         return cls.getSiteDomain()
     
+    @classmethod
+    def stripURLParameters(cls,url):
+        "Only needs to be overriden if URL contains more than one parameter"
+        ## remove any trailing '&' parameters--?sid=999 will be left.
+        ## that's all that any of the current adapters need or want.
+        return re.sub(r"&.*$","",url)
+    
     ## URL pattern validation is done *after* picking an adaptor based
     ## on domain instead of *as* the adaptor selector so we can offer
     ## the user example(s) for that particular site.

fanficdownloader/bbcodeutils/bbcode2html.py

@@ -1,325 +0,0 @@
-#!/usr/bin/python
-# -*- coding: UTF-8 -*-
-#
-# Author:        Pau Sanchez (contact@pausanchez.com)
-# Version:       v1.0
-# Last Modified: 2010/09/15
-# 
-# For the latest version check out:
-#   http://www.codigomanso.com/en/projects
-# 
-# My blog:
-#   http://www.codigomanso.com/en/  - English Version
-#   http://www.codigomanso.com/es/  - Spanish Version
-#
-
-import sys
-import os
-import re
-import urllib
-
-class bbcode2html:
-  '''
-  This class gets a parsed BBCode and transforms it to valid HTML
-
-  Useful functions of this class:
-    html
-    convertToHTML
-
-  Example:
-    > parser = bbcodeparser ()
-    > parser.parse ('[b]bold[/b]')
-    > bbcode2html (parser).html()
-    <b>bold</b>
-
-    # This is faster for huge strings but changes the parser object internally
-    > bbcode2html (parser).html(doDeepCopy = False)
-    <b>bold</b>
-  '''
-  def __init__ (self, parser):
-    self._parser = parser
-    return
-
-  def html (self, allowClassAttr = False, doDeepCopy = True, parser = None):
-    '''
-    Convert current parsed code to HTML
-
-    Example:
-      code = bbcodeparser ('[b]bold[/b]')
-      code.html() -> '<b>bold</b>'
-    '''
-    if parser is None:
-      parser = self._parser
-
-    tokens = parser
-    if instanceof (parser, bbcodeparser):
-      tokens = parser.getTokens()
-
-    return bbcode2html.convertToHTML (tokens, allowClassAttr = allowClassAttr, doDeepCopy = doDeepCopy)
-
-  @staticmethod
-  def htmlString (string):
-    toReplace = {
-      u'<' : '&lt;',
-      u'>' : '&gt;',
-      u'"' : "&quot;",
-      u'&' : "&amp;"
-    }
-    for entity in toReplace:
-      string = string.replace(entity, toReplace[entity])
-    return string
-
-  @staticmethod
-  def getValidTags ():
-    simpleTags = ['b', 'u', 'i', 'sup', 'sub', 'ul', 'ol', 'li', 'table', 'tr', 'th', 'td', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6']
-    validTags = {
-      'p'         : { 'color' : 'color', 'size' : 'size', 'font' : 'font' },
-      'color'     : { 'color' : 'color' },
-      'size'      : { 'size' : 'size' },
-      'font'      : { 'font' : 'font' },
-      'img'       : { 'alt' : 'alt', 'title' : 'title', 'width' : 'width' , 'height' : 'height', 'img' : 'img'},
-      'url'       : { 'href' : 'href', 'url' : 'href', 'link'  : 'href', 'title' : 'title' },
-      's'         : { },
-      'code'      : { },
-      'quote'     : { },
-      'list'      : { 'list' : 'type' },
-      'email'     : { 'email': 'href'},
-      'google'    : { 'google':  'google'},
-      'wikipedia' : { 'wikipedia' : 'wikipedia', 'language' : 'language', 'lang' : 'lang'}
-    }
-
-    for tag in simpleTags:
-      validTags[tag] = { }
-    return validTags
-
-  @staticmethod
-  def convertToHTML (tokens, allowClassAttr = False, validTags = None, doDeepCopy = True):
-    '''
-    Convert internally parsed BBCode to XHTML
-
-    @doDeepCopy 
-      True:  it does a deep copy of tokens so this list will remain unchanged
-      False: tokens will be modified internally, but the output will be produced like 5x faster
-             it's a good idea to use False only when this is the last operation
-    '''
-    # do a deep copy
-    if doDeepCopy:
-      import copy
-      tokens = copy.deepcopy (tokens)
-
-    # filter invalid tags and attributes
-    if validTags is None:
-      validTags = bbcode2html.getValidTags()    
-
-    bbcode2html._filterInvalidTagsAndAttributes (tokens, validTags, allowClassAttr)
-    
-    # Start to convert
-    index       = 0
-    tokenLength = len (tokens)
-
-    # use a list for the output (an order of magnitude faster than using string concatenation)
-    htmlList = []
-    lastListOpener = []
-
-    while index < tokenLength:
-
-      if isinstance (tokens [index], basestring):
-        htmlList.append (bbcode2html.htmlString (tokens [index]))
-        index += 1
-        continue
-
-      token     = tokens[index] 
-      tag       = token['tag']  # opening or closing simple tag. e.g: 'b', '/b', '/u', ...
-      tagName   = (tag[1:] if tag[0] == '/' else tag)
-      tagOpener = (u'/' if tag[0] == '/' else u'')
-      tokenArgs = (token['args'] if 'args' in token else {})
-
-      # opening or closing simple tag COLOR / SIZE
-      if (tagName in ['p', 'color', 'size', 'font']): 
-        style  = ''
-        style += ((u' color: ' + tokenArgs['color'] + u';') if ('color' in tokenArgs) else '')
-        style += ((u' font-size: ' + tokenArgs['size'] + u'pt;') if ('size' in tokenArgs) else '')
-        style += ((u' font-family: ' + tokenArgs['font'] + u';') if ('font' in tokenArgs) else '')
-        style  = style.strip()
-
-        pArgs = {}
-        if style != '':
-          pArgs ['style'] = style
-
-        if 'class' in tokenArgs:
-          pArgs ['class'] = tokenArgs['class']
-
-        if ('args' not in token) and (tagName != 'p'):
-          if (tagOpener == '/'): # if closing tag, close it
-            htmlList.append (u'</span>')
-          index += 1
-          continue
-
-        if tagName != 'p':
-          tag = tagOpener + u'span'
-
-        htmlList.append (bbcode2html.xml (tag, pArgs))
-
-      # IMG tag
-      elif tag == 'img' and (index+2 < tokenLength):
-        if 'img' in tokenArgs:
-          # has the form of <width>x<height> ?
-          sizeMatch = re.match (u'^\s*(\d+)[xX](\d+)\s*$', tokenArgs['img'])
-          if sizeMatch is not None:
-            tokenArgs['width']  = sizeMatch.group(1)
-            tokenArgs['height'] = sizeMatch.group(2)
-          # then assume is the alternative text
-          else:
-            tokenArgs['alt'] = tokenArgs['img']
-          del tokenArgs['img']
-
-        # add the source of the image
-        tokenArgs ['src'] = tokens[index+1]
-
-        # [img]http://www.whatever.com/pic.jpg[/img]
-        htmlList.append (
-          bbcode2html.xml ('img', tokenArgs, soloTag=True)
-        )
-        index += 2 # skip next token and closing tag
-
-      # URL tag
-      elif tag == 'url':
-        if ('args' not in token) and (index+2 < tokenLength):
-          # [url]http://www.google.com[/url]
-          htmlList.append (bbcode2html.xml ('a', { 'href' : tokens[index+1] }))
-        else:
-          # [url=http://www.google.com]Google[/url]
-          # [url link=http://www.google.com title="This is Google"]Google[/url]
-          htmlList.append (bbcode2html.xml ('a', tokenArgs))
-  
-      # URL closing tag (sometimes needed)
-      elif (tag == '/url') or (tag == '/email'):
-        htmlList.append (u'</a>')
-
-      # Email tag
-      elif tag == 'email':
-        if ('args' not in token) and (index+2 < tokenLength):
-          # [email]asdf@asdf.com]
-          htmlList.append (bbcode2html.xml ('a', { 'href' : u'mailto:' + tokens[index+1].strip() }))
-        else:
-          # [email=asdf@asfd.com]john smith[/email]
-          if 'href' in tokenArgs:
-            tokenArgs['href'] = u'mailto:' + tokenArgs['href']
-          htmlList.append (bbcode2html.xml ('a', tokenArgs))
-  
-      elif tagName == 'list':
-        if tagOpener == '/':
-          htmlList.append (bbcode2html.xml (u'/' + lastListOpener.pop()))
-        else:
-          if ('type' not in tokenArgs):
-            htmlList.append (bbcode2html.xml (tagOpener + u'ul', tokenArgs))
-            lastListOpener.append ('ul')
-          else:
-            htmlList.append (bbcode2html.xml (tagOpener + u'ol', tokenArgs))
-            lastListOpener.append ('ol')
-
-      elif tagName == '*':
-        htmlList.append (bbcode2html.xml (tagOpener + u'li', tokenArgs))
-
-      elif (tagName == 's'):
-        tokenArgs['style'] =  'text-decoration: line-through;'
-        htmlList.append (bbcode2html.xml (tagOpener + u'span', tokenArgs))
-
-      elif (tagName == 'code'):
-        htmlList.append (bbcode2html.xml (tagOpener + u'pre', tokenArgs))
-
-      elif (tagName == 'quote'):
-        htmlList.append (bbcode2html.xml (tagOpener + u'blockquote', tokenArgs))
-
-      elif (tagName == 'google'):
-        htmlList.append (
-          bbcode2html.xml (
-            tagOpener + u'a', 
-            {'href' : 'http://www.google.com/search?q=' + urllib.quote_plus (tokens[index+1])},
-            tokens[index+1]
-          )
-        )
-        index += 2
-
-      elif (tagName == 'wikipedia'):
-        subdomain = 'www'
-        for arg  in ['lang', 'language', 'wikipedia']:
-          if arg in tokenArgs:
-            subdomain = tokenArgs[arg] 
-
-        htmlList.append (
-          bbcode2html.xml (
-            tagOpener + u'a', 
-            {'href' : 'http://' + subdomain + '.wikipedia.org/wiki/' + tokens[index+1].replace (' ', '_')},
-            tokens[index+1]
-          )
-        )
-        index += 2
-
-      elif (tagName in validTags):
-        htmlList.append (
-          bbcode2html.xml (tag, tokenArgs)
-        )
-
-      else:
-        # ignore this tag
-        pass 
-
-      index += 1
-
-    return ''.join (htmlList)
-
-  @staticmethod
-  def _filterInvalidTagsAndAttributes (tokens, validTags, allowClassAttr):
-    '''
-    Helper function to filter out invalid attributes from the tokens list
-    '''
-    # add 'class' attribute as valid (mapping 'class' itself)
-    if allowClassAttr:
-      for attr in validTags:
-        validTags[attr]['class'] = 'class'
-
-    # remove invalid attributes from tokens
-    for tindex in range(0, len(tokens)):
-      if isinstance (tokens[tindex], dict) and ('args' in tokens[tindex]) and (tokens[tindex]['tag'] in validTags):
-        validList = validTags[tokens[tindex]['tag']]
-
-        filteredArgs = {}
-        for arg in tokens[tindex]['args']:
-          if arg in validList:
-            # rename the argument
-            filteredArgs[validList[arg]] = tokens[tindex]['args'][arg]
-          else:
-            pass # do not include this arg in the filteredArgs
-
-        tokens[tindex]['args'] = filteredArgs
-
-    return
-
-  @staticmethod
-  def xml (tag, attrs = {}, text = None, soloTag = False):
-    '''
-    Helper function to produce valid XML output
-    '''
-    xml = u'<' + tag.lower()
-
-    # make sure we sort attributes alphabetically (for deterministic output)
-    # Faster but non-deterministic:
-    #  for (key, value) in attrs.iteritems():
-    #    xml += u' ' + key + u'="' + value + u'"'
-    for key in sorted (attrs.keys()):
-      xml += u' ' + key + u'="' + attrs[key] + u'"'
-      
-    # close tag
-    if text is None:
-      if soloTag:
-        xml += u' />'
-      else:
-        xml += u'>'
-    else:
-      xml += u'>' + text + u'</' + tag.lower() + '>'
-
-    return xml
-
-
-

fanficdownloader/bbcodeutils/bbcodebuilder.py

@@ -1,77 +0,0 @@
-#!/usr/bin/python
-# -*- coding: UTF-8 -*-
-#
-# Author:        Pau Sanchez (contact@pausanchez.com)
-# Version:       v1.0
-# Last Modified: 2010/09/15
-# 
-# For the latest version check out:
-#   http://www.codigomanso.com/en/projects
-# 
-# My blog:
-#   http://www.codigomanso.com/en/  - English Version
-#   http://www.codigomanso.com/es/  - Spanish Version
-#
-
-import sys
-import os
-import re
-import hashlib
-
-class bbcodebuilder:
-  '''
-  This class helps to build BBCode programmatically.
-
-  The function names are used as the tag name, then the first parameter
-  is the string that goes inside the tags and any extra parameter is 
-  appended as a parameter to the tag
-
-  Examples:
-    > bbcode = bbcodebuilder()  # create a instance!
-
-    > print bbcode.b ('bold')
-    [b]bold[/b]
-
-    > print bbcode.color ('this goes in red', 'red')
-    [color=red]this goes in red[/color]
-
-    > print bbcode.url ('Google', 'http://www.google.com')
-    [url=http://www.google.com]Google[/url]
-    
-    > print bbcode.alist('item 1', 'item 2')
-    [list=a]
-      [*]item 1
-      [*]item 2
-    [/list]
-
-
-  This solution is based on the recipe found on:
-    http://code.activestate.com/recipes/576831-simple-bbcode-support/
-  '''
-
-  def __getattr__(self, name):
-    '''
-    This is a generic getter that returns a function which gets the first parameter
-    as the string that goes between the tags, and extra parameters as tag parameters.
-
-    The name of the attribute is used as the tag name
-    '''
-    class bbcodebuilder_helper:
-      def __init__(self, name):
-        self._name = name
-
-      def __call__(self, string, *args):
-        return u'[{0}{1}]{2}[/{0}]'.format(self._name, (u'=' + u','.join(map(str, args))) if args else u'', string)
-    
-    return bbcodebuilder_helper (name)
-
-  def list(self, *items):
-    return u'[list]' + u''.join(map(lambda item: u"\n  [*]" + item, items)) + u"\n[/list]"
-
-  def nlist(self, *items):
-    return u'[list=1]' + u''.join(map(lambda item: u"\n  [*]" + item, items)) + u"\n[/list]"
-
-  def alist(self, *items):
-    return u'[list=a]' + u''.join(map(lambda item: u"\n  [*]" + item, items)) + u"\n[/list]"
-
-

fanficdownloader/bbcodeutils/bbcodeparser.py

@@ -1,258 +0,0 @@
-#!/usr/bin/python
-# -*- coding: utf-8 -*-
-#
-# Author:        Pau Sanchez (contact@pausanchez.com)
-# Version:       v1.0
-# Last Modified: 2010/09/15
-# 
-# For the latest version check out:
-#   http://www.codigomanso.com/en/projects
-# 
-# My blog:
-#   http://www.codigomanso.com/en/  - English Version
-#   http://www.codigomanso.com/es/  - Spanish Version
-#
-
-import sys
-import os
-import re
-import hashlib
-
-class bbcodeparser:
-  '''
-  This class parses BBCode into a internal structure to allow later processing and
-  conversion to HTML.
-
-  The parser tries to fix invalid code (like unclosed tags)
-
-  Useful URLs:
-    http://en.wikipedia.org/wiki/BBCode
-    http://www.bbcode.org/reference.php
-
-  Example:
-    > bbcode = bbcodeparser ()
-    > bbcode.parse ('[b]text in bold[/b]').html()
-    <b>text in bold</b>
-
-    # dump HTML
-    > bbcode.parse ('[p][color=red]text in red').html()
-    <p><span style="color:red;">text in red</span></p>
-
-    # dump fixed BBCode
-    > bbcode.parse ('[p][color=red]text in red').bbcode()
-    [p][color=red]text in red[/color][/p]
-
-    > bbcode.parse ('This [b][i]code[/b] will be fixed[/invalid]').bbcode()
-    This [b][i]code[/i][/b] will be fixed
-
-    # dump fixed bbcode
-    > str (bbcodeparse ('This [b][i]code[/b] will be fixed[/invalid]'))
-    This [b][i]code[/i][/b] will be fixed
-  '''
-  _bbcode = ''
-  _tokens = []
-
-  def __init__ (self, bbcode = '', fixInvalidCode = True):
-    ''' Initialize and parse bbcode string (if any is given)
-    '''
-    self.parse (bbcode, fixInvalidCode)
-    return
-
-  def __str__ (self):
-    return self.bbcode()
-
-  def parse (self, bbcode = None, fixInvalidCode = True):
-    '''
-    It will parse and return the token list, trying to fix tags if
-    fixInvalidCode is True
-
-    It will return the current object to allow chaining
-
-    Example:
-      code = bbcode()
-      code.parse ('<b>bold</b>', True) -> 
-      code.parse ('<b>bold<i>italics</b>', True) -> internally will add the missing '</i>'
-    '''
-    if bbcode is not None:
-      self._bbcode = bbcode
-      self._tokens = self.tokenize (bbcode)
-      if fixInvalidCode:
-        self._tokens = self.fixWrongTags (self._tokens)
-
-    return self
-
-  # return ALL tokens
-  def getTokens (self):
-    return self._tokens
-
-  def bbcode (self):
-    '''
-    Dump BBCode again. This is useful for dumping valid BBCode
-    '''
-    bbcode = []
-    for token in self._tokens:
-      if token is None:
-        continue
-
-      if isinstance (token, basestring):
-        bbcode.append (token.replace (u'[', u'\[').replace (u']', u'\]'))
-        continue
-      
-      tag       = token['tag']  # opening or closing simple tag. e.g: 'b', '/b', '/u', ...
-      tagOpener = (u'/' if tag[0] == u'/' else u'')
-
-      if (tagOpener == '/') or ('args' not in token):
-        bbcode.append (u'[' + tag + u']')
-      else:
-        # process args
-        argstr = ''
-
-        # the arg with the same name as the tag repersents the '=whatever'
-        if tag in token['args']:
-          if re.match ('\s|"', token['args'][tag]) is None:
-            argstr = u'=' + token['args'][tag]
-          else:
-            argstr = u'="' + token['args'][tag].replace (u'"', u'\"') + u'"'
-
-        for (k,v) in token['args'].iteritems():
-          if k == tag: # already processed
-            continue
-          argstr += ' ' + k + u'="' + v.replace (u'"', u'\"') + u'"'
-          
-        bbcode.append (u'[' + tag + argstr + ']')
-
-    return u''.join (bbcode)
-
-  def html (self, allowClassAttr = False, doDeepCopy = True):
-    '''
-    Convert current parsed code to HTML
-
-    @allowClassAttr
-      Is something like [b class="asdf"] allowed?
-
-    @doDeepCopy 
-      True:  it does a deep copy of tokens so this list will remain unchanged
-      False: tokens will be modified internally, but the output will be produced like 5x faster
-             it's a good idea to use False when the string parsed is huge and this is the
-             last operation on the string
-
-    Example:
-      code = bbcode ('[b]bold[/b]')
-      code.html() -> '<b>bold</b>'
-    '''
-    from bbcode2html import bbcode2html
-    return bbcode2html.convertToHTML (self._tokens, allowClassAttr = allowClassAttr, doDeepCopy = doDeepCopy)
-
-
-  @staticmethod
-  def fixWrongTags (inTokenList):
-    ''' Add missing tokens that have not been closed properly and try to fix some scenarios
-    '''
-    opened       = []
-    outTokenList = []
-    for token in inTokenList:
-      # normal string... do nothing
-      if isinstance(token, basestring):
-        outTokenList.append (token)
-      else:
-        # if starts with '/' is closing a tag
-        if token['tag'][0] == '/':
-          while (len (opened) > 0) and (opened[-1] != token['tag'][1:]):
-            outTokenList.append ({'tag' : '/' + opened[-1] })
-            del opened[-1]
-
-          if len(opened):
-            del opened[-1]
-            outTokenList.append (token)
-
-        # opening tag
-        else:
-          # if I open the same tag I opened before, close it, and open it again
-          if (len(opened) > 0) and (token['tag'] == opened[-1]):
-            outTokenList.append ({'tag' : '/' + opened[-1] })
-          else:
-            opened.append (token['tag'])
-          outTokenList.append (token)
-
-    # close all elements that have not been closed
-    while len(opened):
-      outTokenList.append ({'tag' : '/' + opened[-1] })
-      del opened[-1]
-
-    return outTokenList
-
-  @staticmethod
-  def tokenize(code):
-    '''
-    Tokenize BBCode tags and parameters
-
-    Return the token list using a internal format. See the example:
-      [
-        { 'tag' : 'p', 'args' : { 'font' : 'arial' } },
-        'This is ',
-        { 'tag' : 'url', 'args' : {'url' : 'http://www.google.com'} },
-        'a link to google',
-        { 'tag' : '/url' },
-        { 'tag' : '/p' }
-      ]
-    '''
-    re_tags    = re.compile (r'(\[[^]]+\])', re.DOTALL | re.UNICODE)
-    re_tagName = re.compile (r'\[([^]=\s]+)([^]]*)\]',  re.DOTALL | re.UNICODE)
-    #re_tagArgs = re.compile (r'\s*([^=]*)=(("([^"]+)")|([^\s]+))',  re.DOTALL | re.UNICODE)
-    re_tagArgs = re.compile (r'\s*([\w]*)=(("([^"]+)")|([^\s]+))',  re.DOTALL | re.UNICODE)
-
-    # get a unique name and replace escaped braces encode utf8 to
-    # prevent CLI/Web from barfing on unicode chars.  Not sure why
-    # this even needs to be 'unique' like this, but that's the way
-    # they wrote it.
-    unique = hashlib.md5(code.encode('utf8')).hexdigest()
-    code   = code.replace ('\[', unique+'_OPEN_BRACE')
-    code   = code.replace ('\]', unique+'_CLOSE_BRACE')
-
-    splitted = re_tags.split(code)
-
-    outTokenList = []
-    for token in splitted:
-      if len(token) == 0:
-        continue
-      
-      if token[0] == '[':
-        match   = re_tagName.match (token)
-        if match:
-          tagName = match.group(1)
-          tagArgs = match.group(2)
-          
-          tagToken = { 'tag' : tagName.lower() }
-  
-          # parse arguments (if any)
-          if len(tagArgs) > 0:
-            allArgs = re_tagArgs.findall(tagArgs)
-  
-            tagArgs = {}
-            for arg in allArgs:
-              # if the argument has no name, use the tagName itself
-              argName  = (arg[0] if arg[0] != '' else tagName)
-              argValue = (arg[3] if (arg[1][0] == '"') else arg[4])
-  
-              tagArgs[argName.lower()] = argValue.replace ('\"', '"')
-  
-            tagToken['args'] = tagArgs
-  
-          outTokenList.append (tagToken)
-
-        # no match, append the text as it is
-        else:
-          outTokenList.append (token)
-      # append the text as it is
-      else:
-        outTokenList.append (token)
-
-    # restore escaped braces back (once code is parsed)
-    restoredTokenList = []
-    for token in outTokenList:
-      if isinstance (token, basestring):
-        token = token.replace (unique+'_OPEN_BRACE', '[').replace (unique+'_CLOSE_BRACE', ']')
-      restoredTokenList.append (token)
-
-    return restoredTokenList
-

fanficdownloader/bbcodeutils/readme.txt

@@ -1,81 +0,0 @@
-AUTHOR
-  Pau Sanchez
-  http://www.codigomanso.com/
-
-VERSION:
-  bbcodeutils v1.0
-  
-LICENSE
-  This code is licensed under Creative Commons Attribution 3.0
-  http://creativecommons.org/licenses/by/3.0/
-
-  You can use this python module or any part of the code you want as long as you add
-  my name as a contributor to your project.
-  
-DESCRIPTION
-  This module can be used to produce HTML from BBCode, to generate BBCode or to fix invalid BBCode.
-
-  The classes are:
-    - bbcodeparser
-    - bbcodebuilder
-    - bbcode2html
-  
-  You can use bbcodeparser to parse BBCode and to produce output in any format you want.
-
-  Open the python file to find more information and examples of use of each class. It can
-  be a good idea to check the test.py for examples
-
-  To run the unit tests:
-    > python test.py
-  
-  To run the performance test:
-    > python test.py BBCodeTests.performanceTest  
-
-
-EXAMPLES OF BBCode:
-
-  [b] -> bold
-  [u] -> underline
-  [i] -> italic
-
-  [center] -> center the text inside
-  [color=XXX] -> change color of text
-  [size=XXX] -> change size of text
-
-  Lists:
-  [ul] -> unordered list
-  [ol] -> ordered list
-  [li] -> list item
-
-  [list] -> start unordered list
-  [*]    -> list item
-  [list=1] -> start a list of numbers
-  [list=a] -> start a list of alphabetic characters
-
-  Advanced:
-  [url] -> link to url
-  [url=http://link/url/]text[/url]  
-  [url link=http://link/url/ title="This is the title"]text[/url]  
-
-  [img]http://to/image[/img]
-  [img=230x330]http://to/image[/img]
-  [img="Alt text here"]http://to/image[/img]
-  [img="Alt text here" width=320 height=240]http://to/image[/img]
-
-  [email]asdf@asdf.com[/email]
-  [email=john@asdf.com]John Smith[/email]
-
-  [google]search this[/google]
-  [wikipedia]Tom Hanks[/wikipedia]
-  [wikipedia lang=es]Tom Hanks[/wikipedia]
-
-  Tables:
-  [table]
-  [tr]
-  [th]
-  [td]
-
-  Advanced:
-  [google]
-  [wikipedia]
-

fanficdownloader/bbcodeutils/test.py

@@ -1,420 +0,0 @@
-#!/usr/bin/python
-# -*- coding: UTF-8 -*-
-#
-# Author:        Pau Sanchez (contact@pausanchez.com)
-# Version:       v1.0
-# Last Modified: 2010/09/15
-# 
-# For the latest version check out:
-#   http://www.codigomanso.com/en/projects
-# 
-# My blog:
-#   http://www.codigomanso.com/en/  - English Version
-#   http://www.codigomanso.com/es/  - Spanish Version
-#
-
-from bbcodeparser import bbcodeparser
-from bbcodebuilder import bbcodebuilder
-
-import random
-import unittest
-
-class BBCodeTests(unittest.TestCase):
-  def setUp (self):
-    self.bbcode = bbcodeparser()
-    return
-
-  def testConstructor (self):
-    self.assertEqual (bbcodeparser ('whatever').html(), 'whatever')
-    self.assertEqual (bbcodeparser ('[b]bold[/b]').html(), '<b>bold</b>')
-    self.assertEqual (str (bbcodeparser ('[b]bold[/b]')), '[b]bold[/b]')
-    return
-
-  def testBold (self):
-    self.assertEqual (self.bbcode.parse ('whatever').html(), 'whatever')
-    self.assertEqual (self.bbcode.parse ('[b]bold[/b]').html(), '<b>bold</b>')
-    self.assertEqual (self.bbcode.parse ('[B]bold[/b]').html(), '<b>bold</b>')
-    self.assertEqual (self.bbcode.parse ('this is [B]bold[/B]').html(), 'this is <b>bold</b>')
-    return
-
-  def testItalic (self):
-    self.assertEqual (self.bbcode.parse ('[i]italic[/i]').html(), '<i>italic</i>')
-    return
-
-  def testUnderline (self):
-    self.assertEqual (self.bbcode.parse ('[u]italic[/u]').html(), '<u>italic</u>')
-    return
-
-  def testURLs (self):
-    self.assertEqual (
-      self.bbcode.parse ('[url]http://www.google.com[/url]').html(),
-      '<a href="http://www.google.com">http://www.google.com</a>'
-    )
-    self.assertEqual (
-      self.bbcode.parse ('[url=http://www.google.com]Google[/url]').html(),
-      '<a href="http://www.google.com">Google</a>'
-    )
-    self.assertEqual (
-      self.bbcode.parse ('[url="http://www.google.com"]Google[/url]').html(),
-      '<a href="http://www.google.com">Google</a>'
-    )
-    self.assertEqual (
-      self.bbcode.parse ('[url="http://www.google.com" title="Search Engine"]Google[/url]').html(),
-      '<a href="http://www.google.com" title="Search Engine">Google</a>'
-    )
-    self.assertEqual (
-      self.bbcode.parse ('[url link="http://www.google.com"]Google[/url]').html(),
-      '<a href="http://www.google.com">Google</a>'
-    )
-    return
-
-  def testPTag (self):
-    self.assertEqual (
-      self.bbcode.parse ('[p color=#0000ff]blue[/p]').html(),
-      u'<p style="color: #0000ff;">blue</p>'
-    )
-    self.assertEqual (
-      self.bbcode.parse ('[p size=12]12pt font[/p]').html(),
-      u'<p style="font-size: 12pt;">12pt font</p>'
-    )
-
-    self.assertEqual (
-      self.bbcode.parse ('[p font=arial]arial[/p]').html(),
-      u'<p style="font-family: arial;">arial</p>'
-    )
-
-    self.assertEqual (
-      self.bbcode.parse ('[p font=arial color=blue size=14]blue 14pt arial').html(),
-      u'<p style="color: blue; font-size: 14pt; font-family: arial;">blue 14pt arial</p>' 
-    )
-
-    self.assertEqual (
-      self.bbcode.parse ('[p class=whatever]text[/p]').html(),
-      u'<p>text</p>' 
-    )
-    return
-
-  def testColorTag (self):
-    self.assertEqual (
-      self.bbcode.parse ('[color=#0000ff]blue[/color]').html(),
-      u'<span style="color: #0000ff;">blue</span>'
-    )
-    return
-
-  def testSizeTag (self):
-    self.assertEqual (
-      self.bbcode.parse ('[size=12]12pt font[/size]').html(),
-      u'<span style="font-size: 12pt;">12pt font</span>'
-    )
-    return
-
-  def testEmail(self):
-    self.assertEqual (
-      self.bbcode.parse ('[email]asdf@asdf.com[/email]').html(),
-      u'<a href="mailto:asdf@asdf.com">asdf@asdf.com</a>'
-    )
-
-    self.assertEqual (
-      self.bbcode.parse ('[email=john@smith.com]John Smith[/email]').html(),
-      u'<a href="mailto:john@smith.com">John Smith</a>'
-    )
-    return
-
-  def testImgTag (self):
-    self.assertEqual (
-      self.bbcode.parse ('[img]http://www.codigomanso.com/image.jpg[/img]').html(),
-      u'<img src="http://www.codigomanso.com/image.jpg" />'
-    )
-
-    self.assertEqual (
-      self.bbcode.parse ('[img="This is the ALT of the image"]http://www.codigomanso.com/image.jpg[/img]').html(),
-      u'<img alt="This is the ALT of the image" src="http://www.codigomanso.com/image.jpg" />'
-    )
-
-    self.assertEqual (
-      self.bbcode.parse ('[img=320x200]http://www.codigomanso.com/image.jpg[/img]').html(),
-      u'<img height="200" src="http://www.codigomanso.com/image.jpg" width="320" />'
-    )
-
-    self.assertEqual (
-      self.bbcode.parse ('[img=320x200 title="Image Test"]http://www.codigomanso.com/image.jpg[/img]').html(),
-      u'<img height="200" src="http://www.codigomanso.com/image.jpg" title="Image Test" width="320" />'
-    )
-
-    self.assertEqual (
-      self.bbcode.parse ('[img="whatever" width=320 height="212" title="Image Test"]http://www.codigomanso.com/image.jpg[/img]').html(),
-      u'<img alt="whatever" height="212" src="http://www.codigomanso.com/image.jpg" title="Image Test" width="320" />'
-    )
-    return
-
-  def testGoogleURL (self):
-    self.assertEqual (
-      self.bbcode.parse ('[google]asdf[/google]').html(),
-      u'<a href="http://www.google.com/search?q=asdf">asdf</a>'
-    )
-    self.assertEqual (
-      self.bbcode.parse ('[google]Tom Hanks[/google]').html(),
-      u'<a href="http://www.google.com/search?q=Tom+Hanks">Tom Hanks</a>'
-    )
-    return
-    
-  def testWikipediaURL (self):
-    self.assertEqual (
-      self.bbcode.parse ('[wikipedia]Tom Hanks[/wikipedia]').html(),
-      u'<a href="http://www.wikipedia.org/wiki/Tom_Hanks">Tom Hanks</a>'
-    )
-
-    self.assertEqual (
-      self.bbcode.parse ('[wikipedia language=en]Tom Hanks[/wikipedia]').html(),
-      u'<a href="http://en.wikipedia.org/wiki/Tom_Hanks">Tom Hanks</a>'
-    )
-    
-    self.assertEqual (
-      self.bbcode.parse ('[wikipedia lang=es]Tom Hanks[/wikipedia]').html(),
-      u'<a href="http://es.wikipedia.org/wiki/Tom_Hanks">Tom Hanks</a>'
-    )
-    
-    self.assertEqual (
-      self.bbcode.parse ('[wikipedia=es]Tom Hanks[/wikipedia]').html(),
-      u'<a href="http://es.wikipedia.org/wiki/Tom_Hanks">Tom Hanks</a>'
-    )
-    return
-    
-  def testListTags (self):
-    self.assertEqual (
-      self.bbcode.parse ('[ul][li]item 1[/li][li]item 2[/li][/ul]').html(),
-      u'<ul><li>item 1</li><li>item 2</li></ul>'
-    )
-    
-    self.assertEqual (
-      self.bbcode.parse ('[ol][li]item 1[/li][li]item 2[/li][/ol]').html(),
-      u'<ol><li>item 1</li><li>item 2</li></ol>'
-    )
-
-    self.assertEqual (
-      self.bbcode.parse ('[list][li]item 1[/li][li]item 2[/li][/list]').html(),
-      u'<ul><li>item 1</li><li>item 2</li></ul>'
-    )
-
-    self.assertEqual (
-      self.bbcode.parse ('[list][*]item 1[*]item 2[/list]').html(),
-      u'<ul><li>item 1</li><li>item 2</li></ul>'
-    )
-
-    self.assertEqual (
-      self.bbcode.parse ('[list=1][li]item 1[/li][li]item 2[/li][/list]').html(),
-      u'<ol type="1"><li>item 1</li><li>item 2</li></ol>'
-    )
-    return
-
-  def testInvalidCode (self):
-    self.assertEqual (self.bbcode.parse ('[invalid]valid text[/invalid]').html(), 'valid text')
-    self.assertEqual (
-      self.bbcode.parse ('[b]bold and [i]italics[/b]').html(),
-      '<b>bold and <i>italics</i></b>'
-    )
-    self.assertEqual (
-      self.bbcode.parse ('[/b]invalid[/b][/p]').html(),
-      'invalid'
-    )
-    self.assertEqual (
-      self.bbcode.parse ('[p][b]bold').html(),
-      '<p><b>bold</b></p>'
-    )
-    self.assertEqual (
-      self.bbcode.parse ('[p][b]a <b>').html(),
-      '<p><b>a &lt;b&gt;</b></p>'
-    )
-
-    self.assertEqual (
-      self.bbcode.parse ('[ol][li]item 1[li]item 2[/li][/ol]').html(),
-      u'<ol><li>item 1</li><li>item 2</li></ol>'
-    )
-
-    self.assertEqual (
-      self.bbcode.parse ('[b]\[b\] stands for [b]bold[/b]').html(),
-      u'<b>[b] stands for </b><b>bold</b>'
-    )
-    return
-
-  def testEscapedBrackets (self):
-    self.assertEqual (
-      self.bbcode.parse ('\[b\]not bold\[/b\]').html(),
-      u'[b]not bold[/b]'
-    )
-
-    self.assertEqual (
-      self.bbcode.parse ('[b]\[b\] stands for bold[/b]').html(),
-      u'<b>[b] stands for bold</b>'
-    )
-
-    self.assertEqual (
-      self.bbcode.parse ('\[b\][b]stands for bold[/b]').html(),
-      u'[b]<b>stands for bold</b>'
-    )
-
-    self.assertEqual (
-      self.bbcode.parse ('\[b\][b]stands for bold[/b] just like <b> in HTML').html(),
-      u'[b]<b>stands for bold</b> just like &lt;b&gt; in HTML'
-    )
-
-  def testBigExample (self):
-    inputText = """check this out
-
-    [h1 class=circle]heading[/h1]
-
-    [p size=14 color=blue font="verdana, Times New Roman"]This is [b] bold [/b] and this [i]italic[/i] and this is [color=red]red[/color] and this is [color="red"]also red[/color].
-    [/p]
-
-    fix [b][i]bold [font=verdana][size=12]and[/size][/font] italic[/b]
-    [img]http://www.codigomanso.com/b.jpg[/img]
-    [url]http://www.codigomanso.com/[/url]
-    [url=http://www.codigomanso.com/]Codigo Manso[/url]
-    [uRl link=http://www.codigomanso.com title="Codigo Manso Blog"]Codigo Manso[/url]
-
-    [ul]
-     [Li]item 1[/Li]
-     [li]item 2[/LI]
-    [/UL]
-
-    [list=1 ]
-     [*]item 1
-     [*]item 2
-    [/list]
-
-    [table class="big"]
-      [tr]
-        [th]big[/th]
-      [/tr]
-    [/table]
-    [invalid class="extra"]whatever[/invalid]"""
-
-    out = self.bbcode.parse (inputText).html(allowClassAttr = True)
-    self.assertEquals (out, '''check this out
-
-    <h1 class="circle">heading</h1>
-
-    <p style="color: blue; font-size: 14pt; font-family: verdana, Times New Roman;">This is <b> bold </b> and this <i>italic</i> and this is <span style="color: red;">red</span> and this is <span style="color: red;">also red</span>.
-    </p>
-
-    fix <b><i>bold <span style="font-family: verdana;"><span style="font-size: 12pt;">and</span></span> italic</i></b>
-    <img src="http://www.codigomanso.com/b.jpg" />
-    <a href="http://www.codigomanso.com/">http://www.codigomanso.com/</a>
-    <a href="http://www.codigomanso.com/">Codigo Manso</a>
-    <a href="http://www.codigomanso.com" title="Codigo Manso Blog">Codigo Manso</a>
-
-    <ul>
-     <li>item 1</li>
-     <li>item 2</li>
-    </ul>
-
-    <ol type="1">
-     <li>item 1
-     </li><li>item 2
-    </li></ol>
-
-    <table class="big">
-      <tr>
-        <th>big</th>
-      </tr>
-    </table>
-    whatever''')
-
-
-  def testBBCodeDumper (self):
-    self.assertEquals (
-      self.bbcode.parse ('[b]bold[/b]').bbcode(),
-      '[b]bold[/b]'
-    )
-
-    self.assertEquals (
-      self.bbcode.parse ('[color=red]text in red[/color]').bbcode(),
-      '[color=red]text in red[/color]'
-    )
-    self.assertEquals (
-      self.bbcode.parse ('[p][color=red]text in red').bbcode(),
-      '[p][color=red]text in red[/color][/p]'
-    )
-
-    self.assertEquals (
-      self.bbcode.parse ('This [b][i]code[/b] will be fixed[/invalid]').bbcode(),
-      'This [b][i]code[/i][/b] will be fixed'
-    )
-
-    self.assertEquals (
-      self.bbcode.parse ('\[[url]http://www.codigomanso.com/en[/url]\]').bbcode(),
-      "\[[url]http://www.codigomanso.com/en[/url]\]"
-    )
-
-  def performanceTest(self):
-    '''
-    This test checks the performance of parse and html operations
-
-    To run this test type:
-      > python test.py BBCodeTests.performanceTest
-    '''
-    inputText = """check this out
-
-    [h1 class=circle]heading[/h1]
-
-    [p size=14 color=blue font="verdana, Times New Roman"]This is [b] bold [/b] and this [i]italic[/i] and this is [color=red]red[/color] and this is [color="red"]also red[/color].
-    [/p]
-
-    fix [b][i]bold [font=verdana][size=12]and[/size][/font] italic[/b]
-    [img]http://www.codigomanso.com/b.jpg[/img]
-    [url]http://www.codigomanso.com/[/url]
-    [url=http://www.codigomanso.com/]Codigo Manso[/url]
-    [uRl link=http://www.codigomanso.com title="Codigo Manso Blog"]Codigo Manso[/url]
-
-    [ul]
-     [Li]item 1[/Li]
-     [li]item 2[/LI]
-    [/UL]
-
-    [list=1 ]
-     [*]item 1
-     [*]item 2
-    [/list]
-
-    [table class="big"]
-      [tr]
-        [th]big[/th]
-      [/tr]
-    [/table]
-    [invalid class="extra"]whatever[/invalid]"""
-
-    import time
-    start = time.time()
-
-    for i in range(0, 12):
-      inputText += inputText
-
-    print "len(inputText) = %.2f MB  (took %.2f seconds)" % (len(inputText)/(1024.0*1024.0), time.time() - start)
-    
-    bbcode = bbcodeparser()
-    start = time.time()
-    bbcode.parse (inputText)
-    total = (time.time() - start)
-    print "time (bbcode.parse()) = %f" % total
-    print "  >> %.2f chars/second" % (len(inputText) / total)
-
-    start = time.time()
-    bbcode.html(doDeepCopy = False)
-    total = (time.time() - start)
-    print "time (bbcode.html()) = %f" % total
-    print "  >> %.2f chars/second" % (len(inputText) / total)
-    return
-    
-  def testCodeBuilder (self):
-    bbcode = bbcodebuilder ()
-    self.assertEquals (bbcode.b ('bold'), u'[b]bold[/b]')
-    self.assertEquals (bbcode.color ('this goes in red', 'red'), u'[color=red]this goes in red[/color]')
-    self.assertEquals (bbcode.url ('Google', 'http://www.google.com'), u'[url=http://www.google.com]Google[/url]')
-    self.assertEquals (bbcode.alist('item 1', 'item 2'), u"[list=a]\n  [*]item 1\n  [*]item 2\n[/list]")
-    return
-
-if __name__ == '__main__':
-    unittest.main()
-
-
-

fanficdownloader/configurable.py

@@ -74,7 +74,8 @@ class Configuration(ConfigParser.SafeConfigParser):
             # internal stuff.
             'langcode',
             'output_css',
-            'authorHTML'
+            'authorHTML',
+            'lastupdate'
             ]
 
     def addConfigSection(self,section):
@@ -93,7 +94,12 @@ class Configuration(ConfigParser.SafeConfigParser):
                 #print("found %s in section [%s]"%(key,section))
                 return True
             except:
-                pass
+                try:
+                    self.get(section,"add_to_"+key)
+                    #print("found add_to_%s in section [%s]"%(key,section))
+                    return True
+                except:
+                    pass
 
         return False
         
@@ -105,16 +111,24 @@ class Configuration(ConfigParser.SafeConfigParser):
                 if val and val.lower() == "false":
                     val = False
                 #print "getConfig(%s)=[%s]%s" % (key,section,val)
-                return val
+                break
             except (ConfigParser.NoOptionError, ConfigParser.NoSectionError), e:
                 pass
 
+        for section in self.sectionslist[::-1]:
+            # 'martian smiley' [::-1] reverses list by slicing whole list with -1 step.
+            try:
+                val = val + self.get(section,"add_to_"+key)
+                #print "getConfig(add_to_%s)=[%s]%s" % (key,section,val)
+            except (ConfigParser.NoOptionError, ConfigParser.NoSectionError), e:
+                pass
+            
         return val
 
     # split and strip each.
     def getConfigList(self, key):
         vlist = self.getConfig(key).split(',')
-        vlist = [ v.strip() for v in vlist ]
+        vlist = filter( lambda x : x !='', [ v.strip() for v in vlist ])
         #print "vlist("+key+"):"+str(vlist)
         return vlist        
 

fanficdownloader/epubutils.py

@@ -22,7 +22,7 @@ def get_dcsource_chaptercount(inputio):
 def get_update_data(inputio,
                     getfilecount=True,
                     getsoups=True):
-    epub = ZipFile(inputio, 'r')
+    epub = ZipFile(inputio, 'r') # works equally well with inputio as a path or a blob
 
     ## Find the .opf file.
     container = epub.read("META-INF/container.xml")
@@ -153,7 +153,7 @@ def get_path_part(n):
 def get_story_url_from_html(inputio,_is_good_url=None):
 
     #print("get_story_url_from_html called")
-    epub = ZipFile(inputio, 'r')
+    epub = ZipFile(inputio, 'r') # works equally well with inputio as a path or a blob
 
     ## Find the .opf file.
     container = epub.read("META-INF/container.xml")

fanficdownloader/geturls.py

@@ -27,8 +27,6 @@ from configurable import Configuration
 
 def get_urls_from_page(url,configuration=None):
 
-    normalized = set() # normalized url
-    retlist = [] # orig urls.
     if not configuration:
         configuration = Configuration("test1.com","EPUB")
 
@@ -56,6 +54,16 @@ def get_urls_from_page(url,configuration=None):
         opener = u2.build_opener(u2.HTTPCookieProcessor(),GZipProcessor())
         data = opener.open(url).read()
 
+    return get_urls_from_html(data,url)
+
+def get_urls_from_html(data,url=None,configuration=None):
+
+    normalized = set() # normalized url
+    retlist = [] # orig urls.
+    
+    if not configuration:
+        configuration = Configuration("test1.com","EPUB")
+
     soup = BeautifulSoup(data)
     
     for a in soup.findAll('a'):
@@ -81,6 +89,33 @@ def get_urls_from_page(url,configuration=None):
 
     return retlist
 
+def get_urls_from_text(data,configuration=None):
+
+    normalized = set() # normalized url
+    retlist = [] # orig urls.
+    
+    if not configuration:
+        configuration = Configuration("test1.com","EPUB")
+    
+    for href in re.findall('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+', data):
+        # this (should) catch normal story links, some javascript
+        # 'are you old enough' links, and 'Report This' links.
+        # The 'normalized' set prevents duplicates.
+        if 'story.php' in href:
+            m = re.search(r"(?P<sid>(view)?story\.php\?(sid|psid|no|story|stid)=\d+)",a['href'])
+            if m != None:
+                href = form_url(None,m.group('sid'))
+        try:
+            href = href.replace('&index=1','')
+            adapter = adapters.getAdapter(configuration,href)
+            if adapter.story.getMetadata('storyUrl') not in normalized:
+                normalized.add(adapter.story.getMetadata('storyUrl'))
+                retlist.append(href)
+        except:
+            pass
+
+    return retlist
+
 def form_url(parenturl,url):
      url = url.strip() # ran across an image with a space in the
                        # src. Browser handled it, so we'd better, too.

fanficdownloader/story.py

@@ -21,6 +21,7 @@ import string
 from math import floor
 from functools import partial
 import logging
+logger = logging.getLogger(__name__)
 import urlparse as up
 
 import exceptions
@@ -29,10 +30,21 @@ from configurable import Configurable
 
 # Create convert_image method depending on which graphics lib we can
 # load.  Preferred: calibre, PIL, none
+        
+imagetypes = {
+    'jpg':'image/jpeg',
+    'jpeg':'image/jpeg',
+    'png':'image/png',
+    'gif':'image/gif',
+    'svg':'image/svg+xml',
+    }
+
 try:
     from calibre.utils.magick import Image
+    convtype = {'jpg':'JPG', 'png':'PNG'}
 
-    def convert_image(url,data,sizes,grayscale):
+    def convert_image(url,data,sizes,grayscale,
+                      removetrans,imgtype="jpg",background='#ffffff'):
         export = False
         img = Image()
         img.load(data)
@@ -44,18 +56,25 @@ try:
             img.size = (nwidth, nheight)
             export = True
             
+        if normalize_format_name(img.format) != imgtype:
+            export = True
+
+        if removetrans and img.has_transparent_pixels():
+            canvas = Image()
+            canvas.create_canvas(int(img.size[0]), int(img.size[1]), str(background))
+            canvas.compose(img)
+            img = canvas
+            export = True
+            
         if grayscale and img.type != "GrayscaleType":
             img.type = "GrayscaleType"
             export = True
 
-        if normalize_format_name(img.format) != "jpg":
-            export = True
-
         if export:
-            return (img.export('JPG'),'jpg','image/jpeg')
+            return (img.export(convtype[imgtype]),imgtype,imagetypes[imgtype])
         else:
-            logging.debug("image used unchanged")
-            return (data,'jpg','image/jpeg')
+            logger.debug("image used unchanged")
+            return (data,imgtype,imagetypes[imgtype])
         
 except:
 
@@ -63,8 +82,9 @@ except:
     try:
         import Image
         from StringIO import StringIO
-        def convert_image(url,data,sizes,grayscale):
-            
+        convtype = {'jpg':'JPEG', 'png':'PNG'}
+        def convert_image(url,data,sizes,grayscale,
+                          removetrans,imgtype="jpg",background='#ffffff'):
             export = False
             img = Image.open(StringIO(data))
             
@@ -75,36 +95,36 @@ except:
                 img = img.resize((nwidth, nheight),Image.ANTIALIAS)
                 export = True
                 
-            if grayscale and img.mode != "L":
-                img = img.convert("L")
-                export = True
-
-            if normalize_format_name(img.format) != "jpg":
+            if normalize_format_name(img.format) != imgtype:
                 if img.mode == "P":
                     # convert pallete gifs to RGB so jpg save doesn't fail.
                     img = img.convert("RGB")
                 export = True
 
+            if removetrans and img.mode == "RGBA":
+                background = Image.new('RGBA', img.size, background)
+                # Paste the image on top of the background
+                background.paste(img, img)
+                img = background.convert('RGB')
+                export = True
+
+            if grayscale and img.mode != "L":
+                img = img.convert("L")
+                export = True
+
             if export:
                 outsio = StringIO()
-                img.save(outsio,'JPEG')
-                return (outsio.getvalue(),'jpg','image/jpeg')
+                img.save(outsio,convtype[imgtype])
+                return (outsio.getvalue(),imgtype,imagetypes[imgtype])
             else:
-                logging.debug("image used unchanged")
-                return (data,'jpg','image/jpeg')
+                logger.debug("image used unchanged")
+                return (data,imgtype,imagetypes[imgtype])
         
     except:
         # No calibre or PIL, simple pass through with mimetype.
-        def convert_image(url,data,sizes,grayscale):
+        def convert_image(url,data,sizes,grayscale,
+                          removetrans,imgtype="jpg",background='#ffffff'):
             return no_convert_image(url,data)
-        
-imagetypes = {
-    'jpg':'image/jpeg',
-    'jpeg':'image/jpeg',
-    'png':'image/png',
-    'gif':'image/gif',
-    'svg':'image/svg+xml',
-    }
 
 ## also used for explicit no image processing.
 def no_convert_image(url,data):
@@ -113,7 +133,7 @@ def no_convert_image(url,data):
     ext=parsedUrl.path[parsedUrl.path.rfind('.')+1:].lower()
     
     if ext not in imagetypes:
-        logging.debug("no_convert_image url:%s - no known extension"%url)
+        logger.debug("no_convert_image url:%s - no known extension"%url)
         # doesn't have extension? use jpg.
         ext='jpg'
         
@@ -240,7 +260,8 @@ class Story(Configurable):
             self.metadata[key]=value
         if key == "language":
             try:
-                self.metadata['langcode'] = langs[self.metadata[key]]
+                # getMetadata not just self.metadata[] to do replace_metadata.
+                self.metadata['langcode'] = langs[self.getMetadata(key)]
             except:
                 self.metadata['langcode'] = 'en'
         if key == 'dateUpdated':
@@ -394,11 +415,13 @@ class Story(Configurable):
         # includelist prevents infinite recursion of include_in_'s
         if self.hasConfig("include_in_"+listname) and listname not in includelist:
             for k in self.getConfigList("include_in_"+listname):
-                retlist.extend(self.getList(k,removeallentities,doreplacements,includelist=includelist+[listname]))
+                retlist.extend(self.getList(k,removeallentities=False,
+                                            doreplacements=doreplacements,includelist=includelist+[listname]))
         else:
         
             if not self.isList(listname):
-                retlist = [self.getMetadata(listname,removeallentities, doreplacements)]
+                retlist = [self.getMetadata(listname,removeallentities=False,
+                                            doreplacements=doreplacements)]
             else:
                 retlist = self.getMetadataRaw(listname)
 
@@ -411,8 +434,13 @@ class Story(Configurable):
                                   map(removeAllEntities,retlist) )
 
         if retlist:
-            # remove dups and sort.
-            return sorted(list(set(retlist)))
+            if listname in ('author','authorUrl'):
+                # need to retain order for author & authorUrl so the
+                # two match up.
+                return retlist
+            else:
+                # remove dups and sort.
+                return sorted(list(set(retlist)))
         else:
             return []
 
@@ -518,18 +546,27 @@ class Story(Configurable):
             try:
                 if self.getConfig('no_image_processing'):
                     (data,ext,mime) = no_convert_image(imgurl,
-                                                   fetch(imgurl))
+                                                       fetch(imgurl))
                 else:
                     try:
                         sizes = [ int(x) for x in self.getConfigList('image_max_size') ]
                     except Exception, e:
                         raise exceptions.FailedToDownload("Failed to parse image_max_size from personal.ini:%s\nException: %s"%(self.getConfigList('image_max_size'),e))
+                    grayscale = self.getConfig('grayscale_images')
+                    imgtype = self.getConfig('convert_images_to')
+                    if not imgtype:
+                        imgtype = "jpg"
+                    removetrans = self.getConfig('remove_transparency')
+                    removetrans = removetrans or grayscale or imgtype=="jpg"
                     (data,ext,mime) = convert_image(imgurl,
                                                     fetch(imgurl),
                                                     sizes,
-                                                    self.getConfig('grayscale_images'))
+                                                    grayscale,
+                                                    removetrans,
+                                                    imgtype,
+                                                    background="#"+self.getConfig('background_color'))
             except Exception, e:
-                logging.info("Failed to load or convert image, skipping:\n%s\nException: %s"%(imgurl,e))
+                logger.info("Failed to load or convert image, skipping:\n%s\nException: %s"%(imgurl,e))
                 return "failedtoload"
             
             # explicit cover, make the first image.
@@ -564,7 +601,7 @@ class Story(Configurable):
                     ext)
                 self.imgtuples.append({'newsrc':newsrc,'mime':mime,'data':data})
                 
-            logging.debug("\nimgurl:%s\nnewsrc:%s\nimage size:%d\n"%(imgurl,newsrc,len(data)))
+            logger.debug("\nimgurl:%s\nnewsrc:%s\nimage size:%d\n"%(imgurl,newsrc,len(data)))
         else:
             newsrc = self.imgtuples[self.imgurls.index(imgurl)]['newsrc']
             

fanficdownloader/writers/writer_epub.py

@@ -21,6 +21,7 @@ import StringIO
 import zipfile
 from zipfile import ZipFile, ZIP_STORED, ZIP_DEFLATED
 import urllib
+import re
 
 ## XML isn't as forgiving as HTML, so rather than generate as strings,
 ## use DOM to generate the XML files.
@@ -657,7 +658,12 @@ div { margin: 0pt; padding: 0pt; }
                 # as one line.  This causes problems for nook(at
                 # least) when the chapter size starts getting big
                 # (200k+)
-                fullhtml = fullhtml.replace('</p>','</p>\n').replace('<br />','<br />\n')
+                #fullhtml = fullhtml.replace('</p>','</p>\n').replace('<br />','<br />\n')
+                # The replaces above added tons of extra newlines
+                # during *each* epub update.  The regexp version adds
+                # only one and removes any extra.
+                fullhtml = re.sub(r'(</p>|<br />)\n*',r'\1\n',fullhtml)
+
                 outputepub.writestr("OEBPS/file%04d.xhtml"%(index+1),fullhtml.encode('utf-8'))
                 del fullhtml
 

index.html

@@ -54,9 +54,12 @@
 	      much easier. </p>
 	  </div>
 	  <!-- put announcements here, h3 is a good title size. -->
-	  <h3>Fixes:</h3>
+	  <h3>Changes:</h3>
 	  <p>
-	    Minor fixes for efpfanfic.net, allow 'On Hiatus' status for fimfiction.net.
+	    <ul>
+	      <li>Yet more fixes for yet more fimfiction.net changes.</li>
+	      <li>Add "add_to_" feature to ini config.  Allow higher priority sections to *add* to any ini param rather than replace it.</li>
+	    </ul>
 	  </p>
 	  <p>
 	    Questions?  Check out our 
@@ -66,7 +69,7 @@
 	    If you have any problems with this application, please
 	    report them in
 	    the <a href="http://groups.google.com/group/fanfic-downloader">FanFictionDownLoader Google Group</a>.  The
-	    <a href="http://4-4-33.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
+	    <a href="http://4-4-46.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
 	  </p>
 	  <div id='error'>
 	    {{ error_message }}
@@ -290,6 +293,8 @@
 	  <dd>
 	    Use the URL of the story's summary, such as
 	    <br /><a href="http://archive.skyehawke.com/story.php?no=17466">http://archive.skyehawke.com/story.php?no=17466</a>.
+	    <br /><a href="http://www.skyehawke.com/archive/story.php?no=17466">http://www.skyehawke.com/archive/story.php?no=17466</a>.
+	    <br /><a href="http://skyehawke.com/archive/story.php?no=17466">http://skyehawke.com/archive/story.php?no=17466</a>.
 	  </dd>
 	  <dt>www.libraryofmoria.com</dt>
 	  <dd>
@@ -389,11 +394,6 @@
 	    Use the URL of the story's first chapter, such as
 	    <br /><a href="http://samdean.archive.nu/viewstory.php?sid=1234">http://samdean.archive.nu/viewstory.php?sid=1234</a>
 	  </dd>
-	  <dt>www.yourfanfiction.com</dt>
-	  <dd>
-	    Use the URL of the story's first chapter, such as
-	    <br /><a href="http://www.yourfanfiction.com/viewstory.php?sid=1234">http://www.yourfanfiction.com/viewstory.php?sid=1234</a>
-	  </dd>
 	  <dt>www.destinysgateway.com</dt>
 	  <dd>
 	    Use the URL of the story's first chapter, such as
@@ -563,6 +563,56 @@
 	    Use the URL of any story chapter, such as
 	    <br /><a href="http://www.efpfanfic.net/viewstory.php?sid=12345">http://www.efpfanfic.net/viewstory.php?sid=12345</a>
 	  </dd>
+	  <dt>www.potterfics.com</dt>
+	  <dd>
+	    Use the URL of the story's chapter list, such as
+	    <br /><a href="http://www.potterfics.com/historias/127583">http://www.potterfics.com/historias/127583</a>
+	  </dd>
+	  <dt>www.dotmoon.net</dt>
+	  <dd>
+	    Use the URL of the story's chapter list, such as
+	    <br /><a href="http://www.dotmoon.net/library_view.php?storyid=1234">http://www.dotmoon.net/library_view.php?storyid=1234</a>
+	  </dd>
+	  <dt>efiction.esteliel.de</dt>
+	  <dd>
+	    Use the URL of the story's chapter list, such as
+	    <br /><a href="http://efiction.esteliel.de/viewstory.php?sid=1234">http://efiction.esteliel.de/viewstory.php?sid=1234</a>
+	  </dd>
+	  <dt>pommedesang.com</dt>
+	  <dd>
+	    Use the URL of the story's chapter list, such as
+	    <br /><a href="http://pommedesang.com/efiction/viewstory.php?sid=1234">http://pommedesang.com/efiction/viewstory.php?sid=1234</a>
+	    <br /><a href="http://pommedesang.com/sds/viewstory.php?sid=1234">http://pommedesang.com/sds/viewstory.php?sid=1234</a>
+	  </dd>
+	  <dt>www.restrictedsection.org</dt>
+	  <dd>
+	    Use the URL of the story's chapter list, such as
+	    <br /><a href="http://www.restrictedsection.org/story.php?story=1234">http://www.restrictedsection.org/story.php?story=1234</a>
+	    <br />Or the story URL for one-shots, such as
+	    <br /><a href="http://www.restrictedsection.org/file.php?file=1234">http://www.restrictedsection.org/file.php?file=1234</a>
+	  </dd>
+	  <dt>imagine.e-fic.com</dt>
+	  <dd>
+	    Use the URL of the story's chapter list, such as
+	    <br /><a href="http://imagine.e-fic.com/viewstory.php?sid=1234">http://imagine.e-fic.com/viewstory.php?sid=1234</a>
+	  </dd>
+	  <dt>buffynfaith.net</dt>
+	  <dd>
+	    Use the URL of the story's chapter list, such as
+	    <br /><a href="http://buffynfaith.net/fanfictions/index.php?act=vie&id=1234">http://buffynfaith.net/fanfictions/index.php?act=vie&amp;id=1234</a>
+	    <br />Or, use the URL of any story chapter, such as
+	    <br /><a href="http://buffynfaith.net/fanfictions/index.php?act=vie&id=1234&ch=2">http://buffynfaith.net/fanfictions/index.php?act=vie&amp;id=1234&amp;ch=2</a>
+	  </dd>
+	  <dt>www.henneth-annun.net</dt>
+	  <dd>
+	    Use the URL of the story's chapter list, such as
+	    <br /><a href="http://www.henneth-annun.net/stories/chapter.cfm?stid=1234">http://www.henneth-annun.net/stories/chapter.cfm?stid=1234</a>
+	  </dd>
+	  <dt>http://www.psychfic.com</dt>
+	  <dd>
+	    Use the URL of the story's chapter list, such as
+	    <br /><a href="http://www.psychfic.com/viewstory.php?sid=1234">http://www.psychfic.com/viewstory.php?sid=1234</a>
+	  </dd>
 	</dl>
 
 	<p>

makeplugin.py

@@ -27,7 +27,7 @@ if __name__=="__main__":
     exclude=['*.pyc','*~','*.xcf','*[0-9].png']
     # from top dir. 'w' for overwrite
     createZipFile(filename,"w",
-                  ['plugin-defaults.ini','plugin-example.ini','epubmerge.py','fanficdownloader'],
+                  ['plugin-defaults.ini','plugin-example.ini','fanficdownloader','downloader.py','defaults.ini'],
                   exclude=exclude)
     #from calibre-plugin dir. 'a' for append
     os.chdir('calibre-plugin')

plugin-defaults.ini

@@ -157,7 +157,7 @@ extratags: FanFiction
 ## metadata part(s) to look at, 2) a regular expression to match the
 ## template, and 3) the name of the GC setting to use, which must
 ## match exactly.  Use this parameter in [defaults], or by site eg,
-## [www.ficwad.com]
+## [ficwad.com]
 ## Make sure to keep at least one space at the start of each line and
 ## to escape % to %%, if used.
 ## template => regexp to match => GC Setting to use.
@@ -269,6 +269,8 @@ include_tocpage: false
 ## dateUpdated,numChapters,numWords at a minimum) will be shown.
 ## Great for tracking when chapters came out and when the description,
 ## etc changed.
+## Plugin will now preserve the log page when the epub is overwritten,
+## too.
 include_logpage: false
 ## If set to 'smart', logpage will only be included if the story is
 ## status:In-Progress or already had a logpage.  That way you don't
@@ -296,12 +298,13 @@ background_color: ffffff
 ## Allow customization of CSS.  Make sure to keep at least one space
 ## at the start of each line and to escape % to %%.  Also need
 ## background_color to be in the same section, if included in CSS.
-## 'adobe-text-layout: optimizeSpeed;' prevents hyphenation on newer Nooks
+## 'adobe-hyphenate: none;' prevents hyphenation on newer Nooks
+## STR(wG) (1.2.1+ for sure)
 output_css: 
  body { background-color: #%(background_color)s;
         text-align: justify;
         margin: 2%%;
-	adobe-text-layout: optimizeSpeed; }
+	adobe-hyphenate: none; }
  pre { font-size: x-small; }
  sml { font-size: small; }
  h1 { text-align: center; }
@@ -343,7 +346,7 @@ output_css:
 ## It can be either a 'file:' or 'http:' url.
 ## Note that if you enable make_firstimage_cover in [epub], but want
 ## to use default_cover_image for a specific site, use the site:format
-## section, for example: [www.ficwad.com:epub]
+## section, for example: [ficwad.com:epub]
 ## default_cover_image is a python string Template string with
 ## ${title}, ${author} etc, same as titlepage_entries. Unless
 ## allow_unsafe_filename is true, invalid filename chars will be
@@ -433,20 +436,26 @@ extratags: FanFiction,Testing,HTML
 #is_adult:true
 
 ## AO3 adapter defines a few extra metadata entries.
-extra_valid_entries:fandoms,freefromtags,ao3categories,comments,kudos,hits,bookmarks
+extra_valid_entries:fandoms,freeformtags,freefromtags,ao3categories,comments,kudos,hits,bookmarks,collections
 fandoms_label:Fandoms
+freeformtags_label:Freeform Tags
 freefromtags_label:Freeform Tags
 ao3categories_label:AO3 Categories
 comments_label:Comments
 kudos_label:Kudos
 hits_label:Hits
-bookmarks:Bookmarks
+collections_label:Collections
+bookmarks_label:Bookmarks
+
+## freeformtags was previously typo'ed as freefromtags.  This way,
+## freefromtags will still work for people who've used it.
+include_in_freefromtags:freeformtags
 
 ## adds to titlepage_entries instead of replacing it.
-#extra_titlepage_entries: fandoms,freefromtags,ao3categories,comments,kudos,hits,bookmarks
+#extra_titlepage_entries: fandoms,freeformtags,ao3categories,comments,kudos,hits,bookmarks
 
 ## adds to include_subject_tags instead of replacing it.
-#extra_subject_tags:fandoms,freefromtags,ao3categories
+#extra_subject_tags:fandoms,freeformtags,ao3categories
 
 [ashwinder.sycophanthex.com]
 ## Site dedicated to these categories/characters/ships
@@ -477,6 +486,15 @@ extracategories:Blood Ties
 ## personal.ini, not defaults.ini.
 #is_adult:true
 
+[buffynfaith.net]
+## Site dedicated to these categories/characters/ships
+extracategories:Buffy: The Vampire Slayer
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
 [castlefans.org]
 ## Site dedicated to these categories/characters/ships
 extracategories:Castle
@@ -534,6 +552,10 @@ extraships:Draco Malfoy/Hermione Granger
 ## personal.ini, not defaults.ini.
 #is_adult:true
 
+## some sites include images that we don't ever want becoming the
+## cover image.  This lets you exclude them.
+cover_exclusion_regexp:/images/.*?ribbon.gif
+
 ## Some adapters collect additional meta information beyond the
 ## standard ones.  They need to be defined in extra_valid_entries to
 ## tell the rest of the FFDL system about them.  They can be used in
@@ -583,6 +605,10 @@ cliches_label:Character Cliches
 # themes=>#bcolumn,a
 # timeline=>#ccolumn,n
 
+[efiction.esteliel.de]
+## Site dedicated to these categories/characters/ships
+extracategories:Lord of the Rings
+
 [erosnsappho.sycophanthex.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter
@@ -659,6 +685,19 @@ extracharacters:Hermione Granger
 ## Site dedicated to these categories/characters/ships
 extracategories:Highlander
 
+[imagine.e-fic.com]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
 [indeath.net]
 ## Site dedicated to these categories/characters/ships
 extracategories:In Death
@@ -744,6 +783,22 @@ extracategories:One Direction
 ## personal.ini, not defaults.ini.
 #is_adult:true
 
+[pommedesang.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Anita Blake Vampire Hunter 
+
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
 [ponyfictionarchive.net]
 ## Site dedicated to these categories/characters/ships
 extracategories:My Little Pony: Friendship is Magic
@@ -896,6 +951,14 @@ extracategories:InuYasha
 extracharacters:Sesshoumaru,Kagome
 extraships:Sesshoumaru/Kagome
 
+[www.dotmoon.net]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
 [www.efpfanfic.net]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -947,7 +1010,7 @@ extratags:
 ## for examples of how to use them.
 extra_valid_entries:reviews,favs,follows
 
-[www.ficwad.com]
+[ficwad.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -987,6 +1050,10 @@ extracategories:Harry Potter
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+[www.henneth-annun.net]
+## Site dedicated to these categories/characters/ships
+extracategories:The Hobbit
+
 [www.hpfandom.net]
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter
@@ -1066,10 +1133,23 @@ extraships:Harry Potter/Ginny Weasley
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter
 
+[www.potterfics.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
 [www.prisonbreakfic.net]
 ## Site dedicated to these categories/characters/ships
 extracategories:Prison Break
 
+[www.psychfic.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Psych
+
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
 [www.qaf-fic.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:Queer as Folk
@@ -1079,6 +1159,16 @@ extracategories:Queer as Folk
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+[www.restrictedsection.org]
+extracategories:Harry Potter
+extragenres:Erotica
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
 [www.scarvesandcoffee.net]
 ## Site dedicated to these categories/characters/ships
 extracategories:Glee
@@ -1228,12 +1318,6 @@ extracategories:Stargate: Atlantis
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
-[www.yourfanfiction.com]
-## Some sites do not require a login, but do require the user to
-## confirm they are adult for adult content.  In commandline version,
-## this should go in your personal.ini, not defaults.ini.
-#is_adult:true
-
 [overrides]
 ## It may sometimes be useful to override all of the specific format,
 ## site and site:format sections in your private configuration.  For

plugin-example.ini

@@ -51,7 +51,7 @@
 ## default is false
 #collect_series: true
 
-[www.ficwad.com]
+[ficwad.com]
 #username:YourUsername
 #password:YourPassword