Integrate merlinfic.dtwins.co.uk, bump versions.

Default keep_summary_html:true to now,
Changes to the AO3, ficbook.net, ficwad.com adapters for summary HTML.
Add options to strip and add chapter numbers to chapter titles.

app.yaml

@@ -1,6 +1,6 @@
 # ffd-retief-hrd fanfictiondownloader
 application: fanfictiondownloader
-version: 4-4-18
+version: 4-4-25
 runtime: python27
 api_version: 1
 threadsafe: true

calibre-plugin/__init__.py

@@ -27,8 +27,8 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
     description         = 'UI plugin to download FanFiction stories from various sites.'
     supported_platforms = ['windows', 'osx', 'linux']
     author              = 'Jim Miller'
-    version             = (1, 5, 42)
-    minimum_calibre_version = (0, 8, 30)
+    version             = (1, 6, 9)
+    minimum_calibre_version = (0, 8, 57)
 
     #: This field defines the GUI plugin class that contains all the code
     #: that actually does something. Its format is module_path:class_name

calibre-plugin/about.txt

@@ -4,7 +4,7 @@
 '<a href="http://www.mobileread.com/forums/showthread.php?t=134856">Reading List</a>',
 '<a href="http://www.mobileread.com/forums/showthread.php?t=126727">Extract ISBN</a>' and
 '<a href="http://www.mobileread.com/forums/showthread.php?t=134000">Count Pages</a>'
-plugins.</p>
+plugins.  bbcodeutils code contributed by Pau Sanchez.</p>
 
 <p>
 Calibre officially distributes plugins from the mobileread.com forum site.

calibre-plugin/common_utils.py

@@ -11,9 +11,11 @@ import os
 from PyQt4 import QtGui
 from PyQt4.Qt import (Qt, QIcon, QPixmap, QLabel, QDialog, QHBoxLayout,
                       QTableWidgetItem, QFont, QLineEdit, QComboBox,
-                      QVBoxLayout, QDialogButtonBox, QStyledItemDelegate, QDateTime)
+                      QVBoxLayout, QDialogButtonBox, QStyledItemDelegate, QDateTime,
+                      QTextEdit,
+                      QListWidget, QAbstractItemView)
 from calibre.constants import iswindows
-from calibre.gui2 import gprefs, error_dialog, UNDEFINED_QDATETIME
+from calibre.gui2 import gprefs, error_dialog, UNDEFINED_QDATETIME, info_dialog
 from calibre.gui2.actions import menu_action_unique_name
 from calibre.gui2.keyboard import ShortcutConfig
 from calibre.utils.config import config_dir
@@ -445,3 +447,97 @@ class DateDelegate(QStyledItemDelegate):
             model.setData(index, UNDEFINED_QDATETIME, Qt.EditRole)
         else:
             model.setData(index, QDateTime(val), Qt.EditRole)
+            
+class PrefsViewerDialog(SizePersistedDialog):
+
+    def __init__(self, gui, namespace):
+        SizePersistedDialog.__init__(self, gui, 'Prefs Viewer dialog')
+        self.setWindowTitle('Preferences for: '+namespace)
+
+        self.gui = gui
+        self.db = gui.current_db
+        self.namespace = namespace
+        self._init_controls()
+        self.resize_dialog()
+
+        self._populate_settings()
+
+        if self.keys_list.count():
+            self.keys_list.setCurrentRow(0)
+
+    def _init_controls(self):
+        layout = QVBoxLayout(self)
+        self.setLayout(layout)
+
+        ml = QHBoxLayout()
+        layout.addLayout(ml, 1)
+
+        self.keys_list = QListWidget(self)
+        self.keys_list.setSelectionMode(QAbstractItemView.SingleSelection)
+        self.keys_list.setFixedWidth(150)
+        self.keys_list.setAlternatingRowColors(True)
+        ml.addWidget(self.keys_list)
+        self.value_text = QTextEdit(self)
+        self.value_text.setTabStopWidth(24)
+        self.value_text.setReadOnly(True)
+        ml.addWidget(self.value_text, 1)
+
+        button_box = QDialogButtonBox(QDialogButtonBox.Ok)
+        button_box.accepted.connect(self.accept)
+        self.clear_button = button_box.addButton('Clear', QDialogButtonBox.ResetRole)
+        self.clear_button.setIcon(get_icon('trash.png'))
+        self.clear_button.setToolTip('Clear all settings for this plugin')
+        self.clear_button.clicked.connect(self._clear_settings)
+        layout.addWidget(button_box)
+
+    def _populate_settings(self):
+        self.keys_list.clear()
+        ns_prefix = self._get_ns_prefix()
+        keys = sorted([k[len(ns_prefix):] for k in self.db.prefs.iterkeys()
+                       if k.startswith(ns_prefix)])
+        for key in keys:
+            self.keys_list.addItem(key)
+        self.keys_list.setMinimumWidth(self.keys_list.sizeHintForColumn(0))
+        self.keys_list.currentRowChanged[int].connect(self._current_row_changed)
+
+    def _current_row_changed(self, new_row):
+        if new_row < 0:
+            self.value_text.clear()
+            return
+        key = unicode(self.keys_list.currentItem().text())
+        val = self.db.prefs.get_namespaced(self.namespace, key, '')
+        self.value_text.setPlainText(self.db.prefs.to_raw(val))
+        
+    def _get_ns_prefix(self):
+        return 'namespaced:%s:'% self.namespace
+        
+    def _clear_settings(self):
+        from calibre.gui2.dialogs.confirm_delete import confirm
+        message = '<p>Are you sure you want to clear your settings in this library for this plugin?</p>' \
+                  '<p>Any settings in other libraries or stored in a JSON file in your calibre plugins ' \
+                  'folder will not be touched.</p>' \
+                  '<p>You must restart calibre afterwards.</p>'
+        if not confirm(message, self.namespace+'_clear_settings', self):
+            return
+        ns_prefix = self._get_ns_prefix()
+        keys = [k for k in self.db.prefs.iterkeys() if k.startswith(ns_prefix)]
+        for k in keys:
+            del self.db.prefs[k]
+        self._populate_settings()
+        d = info_dialog(self, 'Settings deleted', 
+                        '<p>All settings for this plugin in this library have been cleared.</p>'
+                        '<p>Please restart calibre now.</p>',
+                        show_copy_button=False)
+        b = d.bb.addButton(_('Restart calibre now'), d.bb.AcceptRole)
+        b.setIcon(QIcon(I('lt.png')))
+        d.do_restart = False
+        def rf():
+            d.do_restart = True
+        b.clicked.connect(rf)
+        d.set_details('')
+        d.exec_()
+        b.clicked.disconnect()
+        self.close()
+        if d.do_restart:
+            self.gui.quit(restart=True)
+        

calibre-plugin/config.py

@@ -4,10 +4,11 @@ from __future__ import (unicode_literals, division, absolute_import,
                         print_function)
 
 __license__   = 'GPL v3'
-__copyright__ = '2011, Jim Miller'
+__copyright__ = '2012, Jim Miller'
 __docformat__ = 'restructuredtext en'
 
 import traceback, copy
+from collections import OrderedDict
 
 from PyQt4.Qt import (QDialog, QWidget, QVBoxLayout, QHBoxLayout, QLabel, QLineEdit, QFont, QWidget,
                       QTextEdit, QComboBox, QCheckBox, QPushButton, QTabWidget, QVariant, QScrollArea)
@@ -22,116 +23,121 @@ from calibre_plugins.fanfictiondownloader_plugin.dialogs \
 from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.adapters import getConfigSections
 
 from calibre_plugins.fanfictiondownloader_plugin.common_utils \
-    import ( get_library_uuid, KeyboardConfigDialog )
+    import ( get_library_uuid, KeyboardConfigDialog, PrefsViewerDialog )
 
 from calibre.gui2.complete import MultiCompleteLineEdit
 
-# This is where all preferences for this plugin will be stored
+PREFS_NAMESPACE = 'FanFictionDownLoaderPlugin'
+PREFS_KEY_SETTINGS = 'settings'
+
+# Set defaults used by all.  Library specific settings continue to
+# take from here.
+default_prefs = {}
+default_prefs['personal.ini'] = get_resources('plugin-example.ini')
+
+default_prefs['updatemeta'] = True
+default_prefs['updatecover'] = False
+default_prefs['updateepubcover'] = False
+default_prefs['keeptags'] = False
+default_prefs['urlsfromclip'] = True
+default_prefs['updatedefault'] = True
+default_prefs['fileform'] = 'epub'
+default_prefs['collision'] = OVERWRITE
+default_prefs['deleteotherforms'] = False
+default_prefs['adddialogstaysontop'] = False
+default_prefs['includeimages'] = False
+default_prefs['lookforurlinhtml'] = False
+default_prefs['injectseries'] = False
+
+default_prefs['send_lists'] = ''
+default_prefs['read_lists'] = ''
+default_prefs['addtolists'] = False
+default_prefs['addtoreadlists'] = False
+default_prefs['addtolistsonread'] = False
+
+default_prefs['gcnewonly'] = False
+default_prefs['gc_site_settings'] = {}
+default_prefs['allow_gc_from_ini'] = True
+
+default_prefs['countpagesstats'] = []
+
+default_prefs['errorcol'] = ''
+default_prefs['custom_cols'] = {}
+default_prefs['custom_cols_newonly'] = {}
+
+default_prefs['std_cols_newonly'] = {}
+
+def set_library_config(library_config):
+    get_gui().current_db.prefs.set_namespaced(PREFS_NAMESPACE,
+                                              PREFS_KEY_SETTINGS,
+                                              library_config)
+    
+def get_library_config():
+    db = get_gui().current_db
+    library_id = get_library_uuid(db)
+    library_config = None
+    # Check whether this is a configuration needing to be migrated
+    # from json into database.  If so: get it, set it, rename it in json.
+    if library_id in old_prefs:
+        #print("get prefs from old_prefs")
+        library_config = old_prefs[library_id]
+        set_library_config(library_config)
+        old_prefs["migrated to library db %s"%library_id] = old_prefs[library_id]
+        del old_prefs[library_id]
+
+    if library_config is None:
+        #print("get prefs from db")
+        library_config = db.prefs.get_namespaced(PREFS_NAMESPACE, PREFS_KEY_SETTINGS,
+                                                 copy.deepcopy(default_prefs))
+    return library_config
+
+# This is where all preferences for this plugin *were* stored
 # Remember that this name (i.e. plugins/fanfictiondownloader_plugin) is also
 # in a global namespace, so make it as unique as possible.
 # You should always prefix your config file name with plugins/,
 # so as to ensure you dont accidentally clobber a calibre config file
-all_prefs = JSONConfig('plugins/fanfictiondownloader_plugin')
-
-# Set defaults used by all.  Library specific settings continue to
-# take from here.
-all_prefs.defaults['personal.ini'] = get_resources('plugin-example.ini')
-
-all_prefs.defaults['updatemeta'] = True
-all_prefs.defaults['updatecover'] = False
-all_prefs.defaults['updateepubcover'] = False
-all_prefs.defaults['keeptags'] = False
-all_prefs.defaults['urlsfromclip'] = True
-all_prefs.defaults['updatedefault'] = True
-all_prefs.defaults['fileform'] = 'epub'
-all_prefs.defaults['collision'] = OVERWRITE
-all_prefs.defaults['deleteotherforms'] = False
-all_prefs.defaults['adddialogstaysontop'] = False
-all_prefs.defaults['includeimages'] = False
-all_prefs.defaults['lookforurlinhtml'] = False
-all_prefs.defaults['injectseries'] = False
-
-all_prefs.defaults['send_lists'] = ''
-all_prefs.defaults['read_lists'] = ''
-all_prefs.defaults['addtolists'] = False
-all_prefs.defaults['addtoreadlists'] = False
-all_prefs.defaults['addtolistsonread'] = False
-
-all_prefs.defaults['gcnewonly'] = False
-all_prefs.defaults['gc_site_settings'] = {}
-all_prefs.defaults['allow_gc_from_ini'] = True
-
-all_prefs.defaults['errorcol'] = ''
-all_prefs.defaults['custom_cols'] = {}
-
-# The list of settings to copy from all_prefs or the previous library
-# when config is called for the first time on a library.
-copylist = ['personal.ini',
-            'updatemeta',
-            'updatecover',
-            'updateepubcover',
-            'keeptags',
-            'urlsfromclip',
-            'updatedefault',
-            'fileform',
-            'collision',
-            'deleteotherforms',
-            'adddialogstaysontop',
-            'includeimages',
-            'lookforurlinhtml',
-            'injectseries',
-            'gcnewonly',
-            'gc_site_settings',
-            'allow_gc_from_ini']
+old_prefs = JSONConfig('plugins/fanfictiondownloader_plugin')
 
 # fake out so I don't have to change the prefs calls anywhere.  The
 # Java programmer in me is offended by op-overloading, but it's very
 # tidy.
 class PrefsFacade():
-    def __init__(self,all_prefs):
-        self.all_prefs = all_prefs
-        self.lastlibid = None
-
-    def _get_copylist_prefs(self,frompref):
-        return filter( lambda x : x[0] in copylist, frompref.items() )
+    def __init__(self,default_prefs):
+        self.default_prefs = default_prefs
+        self.libraryid = None
+        self.current_prefs = None
         
     def _get_prefs(self):
         libraryid = get_library_uuid(get_gui().current_db)
-        if libraryid not in self.all_prefs:
-            if self.lastlibid == None:
-                self.all_prefs[libraryid] = dict(self._get_copylist_prefs(self.all_prefs))
-            else:
-                self.all_prefs[libraryid] = dict(self._get_copylist_prefs(self.all_prefs[self.lastlibid]))
-                
-        self.lastlibid = libraryid
-            
-        return self.all_prefs[libraryid]
-
-    def _save_prefs(self,prefs):
-        libraryid = get_library_uuid(get_gui().current_db)
-        self.all_prefs[libraryid] = prefs
+        if self.current_prefs == None or self.libraryid != libraryid:
+            #print("self.current_prefs == None(%s) or self.libraryid != libraryid(%s)"%(self.current_prefs == None,self.libraryid != libraryid))
+            self.libraryid = libraryid
+            self.current_prefs = get_library_config()
+        return self.current_prefs
         
     def __getitem__(self,k):            
         prefs = self._get_prefs()
         if k not in prefs:
-            # pulls from all_prefs.defaults automatically if not set
-            # in all_prefs
-            return self.all_prefs[k]
+            # pulls from default_prefs.defaults automatically if not set
+            # in default_prefs
+            return self.default_prefs[k]
         return prefs[k]
 
     def __setitem__(self,k,v):
         prefs = self._get_prefs()
         prefs[k]=v
-        self._save_prefs(prefs)
+        # self._save_prefs(prefs)
 
-    # to be avoided--can cause unexpected results as possibly ancient
-    # all_pref settings may be pulled.
     def __delitem__(self,k):
         prefs = self._get_prefs()
-        del prefs[k]
-        self._save_prefs(prefs)
+        if k in prefs:
+            del prefs[k]
 
-prefs = PrefsFacade(all_prefs)
+    def save_to_db(self):
+        set_library_config(self._get_prefs())
+        
+
+prefs = PrefsFacade(default_prefs)
     
 class ConfigWidget(QWidget):
 
@@ -165,8 +171,16 @@ class ConfigWidget(QWidget):
         if 'Generate Cover' not in plugin_action.gui.iactions:
             self.generatecover_tab.setEnabled(False)
 
-        self.columns_tab = CustomColumnsTab(self, plugin_action)
-        tab_widget.addTab(self.columns_tab, 'Custom Columns')
+        self.countpages_tab = CountPagesTab(self, plugin_action)
+        tab_widget.addTab(self.countpages_tab, 'Count Pages')
+        if 'Count Pages' not in plugin_action.gui.iactions:
+            self.countpages_tab.setEnabled(False)
+
+        self.std_columns_tab = StandardColumnsTab(self, plugin_action)
+        tab_widget.addTab(self.std_columns_tab, 'Standard Columns')
+
+        self.cust_columns_tab = CustomColumnsTab(self, plugin_action)
+        tab_widget.addTab(self.cust_columns_tab, 'Custom Columns')
 
         self.other_tab = OtherTab(self, plugin_action)
         tab_widget.addTab(self.other_tab, 'Other')
@@ -218,20 +232,48 @@ class ConfigWidget(QWidget):
         prefs['gc_site_settings'] = gc_site_settings
         prefs['allow_gc_from_ini'] = self.generatecover_tab.allow_gc_from_ini.isChecked()
 
-        # Custom Columns tab
+        # Count Pages tab
+        countpagesstats = []
+        
+        if self.countpages_tab.pagecount.isChecked():
+            countpagesstats.append('PageCount')
+        if self.countpages_tab.wordcount.isChecked():
+            countpagesstats.append('WordCount')
+        if self.countpages_tab.fleschreading.isChecked():
+            countpagesstats.append('FleschReading')
+        if self.countpages_tab.fleschgrade.isChecked():
+            countpagesstats.append('FleschGrade')
+        if self.countpages_tab.gunningfog.isChecked():
+            countpagesstats.append('GunningFog')
+            
+        prefs['countpagesstats'] = countpagesstats
+        
+        # Standard Columns tab
+        colsnewonly = {}
+        for (col,checkbox) in self.std_columns_tab.stdcol_newonlycheck.iteritems():
+            colsnewonly[col] = checkbox.isChecked()
+        prefs['std_cols_newonly'] = colsnewonly
 
+        # Custom Columns tab
         # error column
-        prefs['errorcol'] = unicode(self.columns_tab.errorcol.itemData(self.columns_tab.errorcol.currentIndex()).toString())
+        prefs['errorcol'] = unicode(self.cust_columns_tab.errorcol.itemData(self.cust_columns_tab.errorcol.currentIndex()).toString())
 
         # cust cols
         colsmap = {}
-        for (col,combo) in self.columns_tab.custcol_dropdowns.iteritems():
+        for (col,combo) in self.cust_columns_tab.custcol_dropdowns.iteritems():
             val = unicode(combo.itemData(combo.currentIndex()).toString())
             if val != 'none':
                 colsmap[col] = val
                 #print("colsmap[%s]:%s"%(col,colsmap[col]))
         prefs['custom_cols'] = colsmap
 
+        colsnewonly = {}
+        for (col,checkbox) in self.cust_columns_tab.custcol_newonlycheck.iteritems():
+            colsnewonly[col] = checkbox.isChecked()
+        prefs['custom_cols_newonly'] = colsnewonly
+        
+        prefs.save_to_db()
+
     def edit_shortcuts(self):
         self.save_settings()
         # Force the menus to be rebuilt immediately, so we have all our actions registered
@@ -289,7 +331,7 @@ class BasicTab(QWidget):
         self.l.addLayout(horz)
 
         self.updatemeta = QCheckBox('Default Update Calibre &Metadata?',self)
-        self.updatemeta.setToolTip("On each download, FFDL offers an option to update Calibre's metadata (title, author, URL, tags, custom columns, etc) from the web site. <br />This sets whether that will default to on or off.")
+        self.updatemeta.setToolTip("On each download, FFDL offers an option to update Calibre's metadata (title, author, URL, tags, custom columns, etc) from the web site. <br />This sets whether that will default to on or off. <br />Columns set to 'New Only' in the column tabs will only be set for new books.")
         self.updatemeta.setChecked(prefs['updatemeta'])
         self.l.addWidget(self.updatemeta)
 
@@ -298,16 +340,25 @@ class BasicTab(QWidget):
         self.updateepubcover.setChecked(prefs['updateepubcover'])
         self.l.addWidget(self.updateepubcover)
 
+        self.l.addSpacing(10)        
+
+        self.deleteotherforms = QCheckBox('Delete other existing formats?',self)
+        self.deleteotherforms.setToolTip('Check this to automatically delete all other ebook formats when updating an existing book.\nHandy if you have both a Nook(epub) and Kindle(mobi), for example.')
+        self.deleteotherforms.setChecked(prefs['deleteotherforms'])
+        self.l.addWidget(self.deleteotherforms)
+        
         self.updatecover = QCheckBox('Update Calibre Cover when Updating Metadata?',self)
         self.updatecover.setToolTip("Update calibre book cover image from EPUB when metadata is updated.  (EPUB only.)\nDoesn't go looking for new images on 'Update Calibre Metadata Only'.")
         self.updatecover.setChecked(prefs['updatecover'])
         self.l.addWidget(self.updatecover)
 
         self.keeptags = QCheckBox('Keep Existing Tags when Updating Metadata?',self)
-        self.keeptags.setToolTip('Existing tags will be kept and any new tags added.\nCompleted and In-Progress tags will be still be updated, if known.\nLast Updated tags will be updated if lastupdate in include_subject_tags.')
+        self.keeptags.setToolTip("Existing tags will be kept and any new tags added.\nCompleted and In-Progress tags will be still be updated, if known.\nLast Updated tags will be updated if lastupdate in include_subject_tags.\n(If Tags is set to 'New Only' in the Standard Columns tab, this has no effect.)")
         self.keeptags.setChecked(prefs['keeptags'])
         self.l.addWidget(self.keeptags)
 
+        self.l.addSpacing(10)        
+
         self.urlsfromclip = QCheckBox('Take URLs from Clipboard?',self)
         self.urlsfromclip.setToolTip('Prefill URLs from valid URLs in Clipboard when Adding New.')
         self.urlsfromclip.setChecked(prefs['urlsfromclip'])
@@ -319,16 +370,13 @@ class BasicTab(QWidget):
         self.updatedefault.setChecked(prefs['updatedefault'])
         self.l.addWidget(self.updatedefault)
 
-        self.deleteotherforms = QCheckBox('Delete other existing formats?',self)
-        self.deleteotherforms.setToolTip('Check this to automatically delete all other ebook formats when updating an existing book.\nHandy if you have both a Nook(epub) and Kindle(mobi), for example.')
-        self.deleteotherforms.setChecked(prefs['deleteotherforms'])
-        self.l.addWidget(self.deleteotherforms)
-        
         self.adddialogstaysontop = QCheckBox("Keep 'Add New from URL(s)' dialog on top?",self)
         self.adddialogstaysontop.setToolTip("Instructs the OS and Window Manager to keep the 'Add New from URL(s)'\ndialog on top of all other windows.  Useful for dragging URLs onto it.")
         self.adddialogstaysontop.setChecked(prefs['adddialogstaysontop'])
         self.l.addWidget(self.adddialogstaysontop)
 
+        self.l.addSpacing(10)        
+
         # this is a cheat to make it easier for users to realize there's a new include_images features.
         self.includeimages = QCheckBox("Include images in EPUBs?",self)
         self.includeimages.setToolTip("Download and include images in EPUB stories.  This is equivalent to adding:\n\n[epub]\ninclude_images:true\nkeep_summary_html:true\nmake_firstimage_cover:true\n\n ...to the top of personal.ini.  Your settings in personal.ini will override this.")
@@ -552,13 +600,61 @@ class GenerateCoverTab(QWidget):
         self.gcnewonly.setChecked(prefs['gcnewonly'])
         self.l.addWidget(self.gcnewonly)
 
-        self.allow_gc_from_ini = QCheckBox('Allow generate_cover_settings from personal.ini to override.',self)
+        self.allow_gc_from_ini = QCheckBox('Allow generate_cover_settings from personal.ini to override',self)
         self.allow_gc_from_ini.setToolTip("The personal.ini parameter generate_cover_settings allows you to choose a GC setting based on metadata rather than site, but it's much more complex.<br \>generate_cover_settings is ignored when this is off.")
         self.allow_gc_from_ini.setChecked(prefs['allow_gc_from_ini'])
         self.l.addWidget(self.allow_gc_from_ini)
             
         self.l.insertStretch(-1)
         
+class CountPagesTab(QWidget):
+
+    def __init__(self, parent_dialog, plugin_action):
+        self.parent_dialog = parent_dialog
+        self.plugin_action = plugin_action
+        QWidget.__init__(self)
+        
+        self.l = QVBoxLayout()
+        self.setLayout(self.l)
+
+        label = QLabel('These settings provide integration with the Count Pages Plugin.  Count Pages can automatically update custom columns with page, word and reading level statistics.  You have to create and configure the columns in Count Pages first.')
+        label.setWordWrap(True)
+        self.l.addWidget(label)
+        self.l.addSpacing(5)
+
+        label = QLabel('If any of the settings below are checked, when stories are added or updated, the Count Pages Plugin will be called to update the checked statistics.')
+        label.setWordWrap(True)
+        self.l.addWidget(label)
+        self.l.addSpacing(5)
+        
+        # 'PageCount', 'WordCount', 'FleschReading', 'FleschGrade', 'GunningFog'
+        self.pagecount = QCheckBox('Page Count',self)
+        self.pagecount.setToolTip('Which column and algorithm to use are configured in Count Pages.')
+        self.pagecount.setChecked('PageCount' in prefs['countpagesstats'])
+        self.l.addWidget(self.pagecount)
+            
+        self.wordcount = QCheckBox('Word Count',self)
+        self.wordcount.setToolTip('Which column and algorithm to use are configured in Count Words.\nWill overwrite word count from FFDL metadata if set to update the same custom column.')
+        self.wordcount.setChecked('WordCount' in prefs['countpagesstats'])
+        self.l.addWidget(self.wordcount)
+
+        self.fleschreading = QCheckBox('Flesch Reading Ease',self)
+        self.fleschreading.setToolTip('Which column and algorithm to use are configured in Count Pages.')
+        self.fleschreading.setChecked('FleschReading' in prefs['countpagesstats'])
+        self.l.addWidget(self.fleschreading)
+        
+        self.fleschgrade = QCheckBox('Flesch-Kincaid Grade Level',self)
+        self.fleschgrade.setToolTip('Which column and algorithm to use are configured in Count Pages.')
+        self.fleschgrade.setChecked('FleschGrade' in prefs['countpagesstats'])
+        self.l.addWidget(self.fleschgrade)
+        
+        self.gunningfog = QCheckBox('Gunning Fog Index',self)
+        self.gunningfog.setToolTip('Which column and algorithm to use are configured in Count Pages.')
+        self.gunningfog.setChecked('GunningFog' in prefs['countpagesstats'])
+        self.l.addWidget(self.gunningfog)
+        
+        self.l.insertStretch(-1)
+        
 class OtherTab(QWidget):
 
     def __init__(self, parent_dialog, plugin_action):
@@ -586,6 +682,12 @@ class OtherTab(QWidget):
         reset_confirmation_button.clicked.connect(self.reset_dialogs)
         self.l.addWidget(reset_confirmation_button)
         
+        view_prefs_button = QPushButton('&View library preferences...', self)
+        view_prefs_button.setToolTip(_(
+                    'View data stored in the library database for this plugin'))
+        view_prefs_button.clicked.connect(self.view_prefs)
+        self.l.addWidget(view_prefs_button)
+        
         self.l.insertStretch(-1)
         
     def reset_dialogs(self):
@@ -597,6 +699,10 @@ class OtherTab(QWidget):
                     _('Confirmation dialogs have all been reset'),
                     show=True,
                     show_copy_button=False)
+        
+    def view_prefs(self):
+        d = PrefsViewerDialog(self.plugin_action.gui, PREFS_NAMESPACE)
+        d.exec_()
 
 permitted_values = {
     'int' : ['numWords','numChapters'],
@@ -609,6 +715,7 @@ permitted_values = {
                      'language',
                      'series',
                      'characters',
+                     'ships',
                      'status',
                      'datePublished',
                      'dateUpdated',
@@ -645,6 +752,7 @@ titleLabels = {
     'status-I':'Status:In-Progress',
     'series':'Series',
     'characters':'Characters',
+    'ships':'Relationships',
     'datePublished':'Published',
     'dateUpdated':'Updated',
     'dateCreated':'Packaged',
@@ -685,7 +793,17 @@ class CustomColumnsTab(QWidget):
         self.l.addSpacing(5)
         
         self.custcol_dropdowns = {}
+        self.custcol_newonlycheck = {}
 
+        scrollable = QScrollArea()
+        scrollcontent = QWidget()
+        scrollable.setWidget(scrollcontent)
+        scrollable.setWidgetResizable(True)
+        self.l.addWidget(scrollable)
+
+        self.sl = QVBoxLayout()
+        scrollcontent.setLayout(self.sl)
+        
         for key, column in custom_columns.iteritems():
 
             if column['datatype'] in permitted_values:
@@ -693,8 +811,8 @@ class CustomColumnsTab(QWidget):
                 # for (k,v) in column.iteritems():
                 #     print("column['%s'] => %s"%(k,v))
                 horz = QHBoxLayout()
-                label = QLabel('%s(%s)'%(column['name'],key))
-                label.setToolTip("Update this %s column with..."%column['datatype'])
+                label = QLabel(column['name'])
+                label.setToolTip("Update this %s column(%s) with..."%(key,column['datatype']))
                 horz.addWidget(label)
                 dropdown = QComboBox(self)
                 dropdown.addItem('',QVariant('none'))
@@ -707,11 +825,18 @@ class CustomColumnsTab(QWidget):
                     dropdown.setToolTip("Metadata values valid for this type of column.\nValues that aren't valid for this enumeration column will be ignored.")
                 else:
                     dropdown.setToolTip("Metadata values valid for this type of column.")
-
                 horz.addWidget(dropdown)
-                self.l.addLayout(horz)
+
+                newonlycheck = QCheckBox("New Only",self)
+                newonlycheck.setToolTip("Write to %s(%s) only for new\nbooks, not updates to existing books."%(column['name'],key))
+                self.custcol_newonlycheck[key] = newonlycheck
+                if key in prefs['custom_cols_newonly']:
+                    newonlycheck.setChecked(prefs['custom_cols_newonly'][key])
+                horz.addWidget(newonlycheck)
+                
+                self.sl.addLayout(horz)
         
-        self.l.insertStretch(-1)
+        self.sl.insertStretch(-1)
 
         self.l.addSpacing(5)
         label = QLabel("Special column:")
@@ -733,6 +858,53 @@ class CustomColumnsTab(QWidget):
         self.errorcol.setCurrentIndex(self.errorcol.findData(QVariant(prefs['errorcol'])))
         horz.addWidget(self.errorcol)
         self.l.addLayout(horz)
-        
-        
+
         #print("prefs['custom_cols'] %s"%prefs['custom_cols'])
+
+
+class StandardColumnsTab(QWidget):
+
+    def __init__(self, parent_dialog, plugin_action):
+        self.parent_dialog = parent_dialog
+        self.plugin_action = plugin_action
+        QWidget.__init__(self)
+
+        columns=OrderedDict()
+        
+        columns["title"]="Title"
+        columns["authors"]="Author(s)"
+        columns["publisher"]="Publisher"
+        columns["tags"]="Tags"
+        columns["languages"]="Languages"
+        columns["pubdate"]="Published Date"
+        columns["timestamp"]="Date"
+        columns["comments"]="Comments"
+        columns["series"]="Series"
+        columns["identifiers"]="Ids(url id only)"
+
+        self.l = QVBoxLayout()
+        self.setLayout(self.l)
+
+        label = QLabel("The standard calibre metadata columns are listed below.  You may choose whether FFDL will fill each column automatically on updates or only for new books.")
+        label.setWordWrap(True)
+        self.l.addWidget(label)
+        self.l.addSpacing(5)
+        
+        self.stdcol_newonlycheck = {}
+
+        for key, column in columns.iteritems():
+            horz = QHBoxLayout()
+            label = QLabel(column)
+            #label.setToolTip("Update this %s column(%s) with..."%(key,column['datatype']))
+            horz.addWidget(label)
+
+            newonlycheck = QCheckBox("New Only",self)
+            newonlycheck.setToolTip("Write to %s only for new\nbooks, not updates to existing books."%column)
+            self.stdcol_newonlycheck[key] = newonlycheck
+            if key in prefs['std_cols_newonly']:
+                newonlycheck.setChecked(prefs['std_cols_newonly'][key])
+            horz.addWidget(newonlycheck)
+            
+            self.l.addLayout(horz)
+        
+        self.l.insertStretch(-1)

calibre-plugin/dialogs.py

@@ -106,26 +106,21 @@ class AddNewDialog(SizePersistedDialog):
 
         horz = QHBoxLayout()
         label = QLabel('If Story Already Exists?')
-        label.setToolTip("What to do if there's already an existing story with the same title and author.")
         horz.addWidget(label)
         self.collision = QComboBox(self)
+        self.collision.setToolTip("What to do if there's already an existing story with the same URL or title and author.")
         # add collision options
         self.set_collisions()
         i = self.collision.findText(prefs['collision'])
         if i > -1:
             self.collision.setCurrentIndex(i)
-        # self.collision.setToolTip(OVERWRITE+' will replace the existing story.\n'+
-        #                           UPDATE+' will download new chapters only and add to existing EPUB.\n'+
-        #                           ADDNEW+' will create a new story with the same title and author.\n'+
-        #                           SKIP+' will not download existing stories.\n'+
-        #                           CALIBREONLY+' will not download stories, but will update Calibre metadata.')
         label.setBuddy(self.collision)
         horz.addWidget(self.collision)
         self.l.addLayout(horz)
 
         horz = QHBoxLayout()
         self.updatemeta = QCheckBox('Update Calibre &Metadata?',self)
-        self.updatemeta.setToolTip('Update metadata for story in Calibre from web site?')
+        self.updatemeta.setToolTip("Update metadata for existing stories in Calibre from web site?\n(Columns set to 'New Only' in the column tabs will only be set for new books.)")
         self.updatemeta.setChecked(prefs['updatemeta'])
         horz.addWidget(self.updatemeta)
 
@@ -177,15 +172,17 @@ class FakeLineEdit():
     def text(self):
         pass
     
-class CollectURLDialog(QDialog):
+class CollectURLDialog(SizePersistedDialog):
     '''
     Collect single url for get urls.
     '''
-    def __init__(self, gui, title):
-        QDialog.__init__(self, gui)
+    def __init__(self, gui, title, url_text): 
+        SizePersistedDialog.__init__(self, gui, 'FanFictionDownLoader plugin:get story urls')
         self.gui = gui
         self.status=False
 
+        self.setMinimumWidth(300)
+        
         self.l = QGridLayout()
         self.setLayout(self.l)
 
@@ -194,6 +191,7 @@ class CollectURLDialog(QDialog):
         
         self.l.addWidget(QLabel("URL:"),1,0)
         self.url = QLineEdit(self)
+        self.url.setText(url_text)
         self.l.addWidget(self.url,1,1)
    
         self.ok_button = QPushButton('OK', self)
@@ -204,15 +202,16 @@ class CollectURLDialog(QDialog):
         self.cancel_button.clicked.connect(self.cancel)
         self.l.addWidget(self.cancel_button,2,1)
 
-        self.resize(self.sizeHint())
+        # restore saved size.
+        self.resize_dialog()
 
     def ok(self):
         self.status=True
-        self.hide()
+        self.accept()
 
     def cancel(self):
         self.status=False
-        self.hide()
+        self.reject()
 
 class UserPassDialog(QDialog):
     '''
@@ -371,7 +370,7 @@ class AuthorTableWidgetItem(ReadOnlyTableWidgetItem):
 
     #Qt uses a simple < check for sorting items, override this to use the sortKey
     def __lt__(self, other):
-        return self.sort_key < other.sort_key
+        return self.sort_key.lower() < other.sort_key.lower()
 
 class UpdateExistingDialog(SizePersistedDialog):
     def __init__(self, gui, header, prefs, icon, books,
@@ -431,9 +430,9 @@ class UpdateExistingDialog(SizePersistedDialog):
         options_layout.addWidget(self.fileform)
         
         label = QLabel('Update Mode:')
-        label.setToolTip("What sort of update to perform.  May set default from plugin configuration.")
         options_layout.addWidget(label)
         self.collision = QComboBox(self)
+        self.collision.setToolTip("What sort of update to perform.  May set default from plugin configuration.")
         # add collision options
         self.set_collisions()
         i = self.collision.findText(prefs['collision'])
@@ -444,7 +443,7 @@ class UpdateExistingDialog(SizePersistedDialog):
         options_layout.addWidget(self.collision)
 
         self.updatemeta = QCheckBox('Update Calibre &Metadata?',self)
-        self.updatemeta.setToolTip('Update metadata for story in Calibre from web site?  May set default from plugin configuration.')
+        self.updatemeta.setToolTip("Update metadata for existing stories in Calibre from web site?\n(Columns set to 'New Only' in the column tabs will only be set for new books.)")
         self.updatemeta.setChecked(prefs['updatemeta'])
         options_layout.addWidget(self.updatemeta)
                 
@@ -623,7 +622,7 @@ class StoryListTableWidget(QTableWidget):
         title_cell.setData(Qt.UserRole, QVariant(row))
         self.setItem(row, 1, title_cell)
         
-        self.setItem(row, 2, AuthorTableWidgetItem(book['author'], book['author_sort']))
+        self.setItem(row, 2, AuthorTableWidgetItem(", ".join(book['author']), ", ".join(book['author_sort'])))
         
         url_cell = ReadOnlyTableWidgetItem(book['url'])
         #url_cell.setData(Qt.UserRole, QVariant(book['url']))

calibre-plugin/ffdl_plugin.py

@@ -7,7 +7,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2012, Jim Miller'
 __docformat__ = 'restructuredtext en'
 
-import time, os, copy, threading, re
+import time, os, copy, threading, re, platform
 from ConfigParser import SafeConfigParser
 from StringIO import StringIO
 from functools import partial
@@ -19,6 +19,7 @@ from PyQt4.Qt import (QApplication, QMenu, QToolButton)
 from PyQt4.Qt import QPixmap, Qt
 from PyQt4.QtCore import QBuffer
 
+from calibre.constants import numeric_version as calibre_version
 
 from calibre.ptempfile import PersistentTemporaryFile, PersistentTemporaryDirectory, remove_dir
 from calibre.ebooks.metadata import MetaInformation, authors_to_string
@@ -27,6 +28,7 @@ from calibre.gui2 import error_dialog, warning_dialog, question_dialog, info_dia
 from calibre.gui2.dialogs.message_box import ViewLog
 from calibre.gui2.dialogs.confirm_delete import confirm
 from calibre.utils.date import local_tz
+from calibre.library.comments import sanitize_comments_html
 
 # The class that all interface action plugins must inherit from
 from calibre.gui2.actions import InterfaceAction
@@ -35,7 +37,7 @@ from calibre_plugins.fanfictiondownloader_plugin.common_utils import (set_plugin
                                          create_menu_action_unique, get_library_uuid)
 
 from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapters, writers, exceptions
-from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.htmlcleanup import stripHTML
+#from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.htmlcleanup import stripHTML
 from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.epubutils import get_dcsource, get_dcsource_chaptercount, get_story_url_from_html
 from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.geturls import get_urls_from_page
 
@@ -115,9 +117,8 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         # items to prevent GC removing it.
         self.menu_actions = []
         self.qaction.setMenu(self.menu)
-        self.menu.aboutToShow.connect(self.about_to_show_menu)
-
         self.menus_lock = threading.RLock()
+        self.menu.aboutToShow.connect(self.about_to_show_menu)
 
     def initialization_complete(self):
         # otherwise configured hot keys won't work until the menu's
@@ -133,10 +134,6 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         
     def rebuild_menus(self):
         with self.menus_lock:
-            # Show the config dialog
-            # The config dialog can also be shown from within
-            # Preferences->Plugins, which is why the do_user_config
-            # method is defined on the base plugin class
             do_user_config = self.interface_action_base_plugin.do_user_config
             self.menu.clear()
             self.actions_unique_map = {}
@@ -147,8 +144,6 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                                                        triggered=self.add_dialog )
 
             self.update_action = self.create_menu_item_ex(self.menu, '&Update Existing FanFiction Book(s)', image='plusplus.png',
-                                                          unique_name='Update Existing FanFiction Book(s)',
-                                                          shortcut_name='Update Existing FanFiction Book(s)',
                                                           triggered=self.update_existing)
 
             if 'Reading List' in self.gui.iactions and (prefs['addtolists'] or prefs['addtoreadlists']) :
@@ -168,48 +163,34 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
 
                 if addmenutxt:
                     self.add_send_action = self.create_menu_item_ex(self.menu, addmenutxt, image='plusplus.png',
-                                                                    unique_name=addmenutxt,
-                                                                    shortcut_name=addmenutxt,
                                                                     triggered=partial(self.update_lists,add=True))
 
                 if rmmenutxt:
                     self.add_remove_action = self.create_menu_item_ex(self.menu, rmmenutxt, image='minusminus.png',
-                                                                      unique_name=rmmenutxt,
-                                                                      shortcut_name=rmmenutxt,
                                                                       triggered=partial(self.update_lists,add=False))
                 
-                # try:
-                #     self.add_send_action.setEnabled( len(self.gui.library_view.get_selected_ids()) > 0 )
-                # except:
-                #     pass
-                # try:
-                #     self.add_remove_action.setEnabled( len(self.gui.library_view.get_selected_ids()) > 0 )
-                # except:
-                #     pass
-
             self.menu.addSeparator()
             self.get_list_action = self.create_menu_item_ex(self.menu, 'Get URLs from Selected Books', image='bookmarks.png',
-                                                            unique_name='Get URLs from Selected Books',
-                                                            shortcut_name='Get URLs from Selected Books',
                                                             triggered=self.get_list_urls)
 
-            self.get_list_action = self.create_menu_item_ex(self.menu, 'Get Story URLs from Web Page', image='view.png',
-                                                            unique_name='Get Story URLs from Web Page',
-                                                            shortcut_name='Get Story URLs from Web Page',
-                                                            triggered=self.get_urls_from_page)
+            self.get_list_url_action = self.create_menu_item_ex(self.menu, 'Get Story URLs from Web Page', image='view.png',
+                                                                triggered=self.get_urls_from_page)
 
-            self.menu.addSeparator()
-            self.config_action = create_menu_action_unique(self, self.menu, '&Configure Plugin', shortcut=False,
-                                                           image= 'config.png',
-                                                           unique_name='Configure FanFictionDownLoader',
-                                                           shortcut_name='Configure FanFictionDownLoader',
-                                                           triggered=partial(do_user_config,parent=self.gui))
+            # print("platform.system():%s"%platform.system())
+            # print("platform.mac_ver()[0]:%s"%platform.mac_ver()[0])
+            if not platform.mac_ver()[0]: # Some macs crash on these menu items for unknown reasons.
+                self.menu.addSeparator()
+                self.config_action = self.create_menu_item_ex(self.menu, '&Configure Plugin',
+                                                              image= 'config.png',
+                                                              unique_name='Configure FanFictionDownLoader',
+                                                              shortcut_name='Configure FanFictionDownLoader',
+                                                              triggered=partial(do_user_config,parent=self.gui))
             
-            self.config_action = create_menu_action_unique(self, self.menu, '&About Plugin', shortcut=False,
-                                                           image= 'images/icon.png',
-                                                           unique_name='About FanFictionDownLoader',
-                                                           shortcut_name='About FanFictionDownLoader',
-                                                           triggered=self.about)
+                self.about_action = self.create_menu_item_ex(self.menu, 'About Plugin',
+                                                             image= 'images/icon.png',
+                                                             unique_name='About FanFictionDownLoader', 
+                                                             shortcut_name='About FanFictionDownLoader', 
+                                                             triggered=self.about)
             
             # Before we finalize, make sure we delete any actions for menus that are no longer displayed
             for menu_id, unique_name in self.old_actions_unique_map.iteritems():
@@ -235,10 +216,12 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
     def create_menu_item_ex(self, parent_menu, menu_text, image=None, tooltip=None,
                            shortcut=None, triggered=None, is_checked=None, shortcut_name=None,
                            unique_name=None):
+        #print("create_menu_item_ex before %s"%menu_text)
         ac = create_menu_action_unique(self, parent_menu, menu_text, image, tooltip,
                                        shortcut, triggered, is_checked, shortcut_name, unique_name)
         self.actions_unique_map[ac.calibre_shortcut_unique_name] = ac.calibre_shortcut_unique_name
         self.menu_actions.append(ac)
+        #print("create_menu_item_ex after %s"%menu_text)
         return ac
 
     def plugin_button(self):
@@ -253,13 +236,23 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
             self._update_reading_lists(self.gui.library_view.get_selected_ids(),add)
 
     def get_urls_from_page(self):
-        d = CollectURLDialog(self.gui,"Get Story URLs from Web Page")
+
+        if prefs['urlsfromclip']:
+            try:
+                urltxt = self.get_urls_clip(storyurls=False)[0]
+            except:
+                urltxt = ""
+        
+        d = CollectURLDialog(self.gui,"Get Story URLs from Web Page",urltxt)
         d.exec_()
         if not d.status:
             return
-        print("URL:%s"%d.url.text())
+        print("get_urls_from_page URL:%s"%d.url.text())
 
-        url_list = get_urls_from_page("%s"%d.url.text())
+        ffdlconfig = SafeConfigParser()
+        ffdlconfig.readfp(StringIO(get_resources("plugin-defaults.ini")))
+        ffdlconfig.readfp(StringIO(prefs['personal.ini']))
+        url_list = get_urls_from_page("%s"%d.url.text(),ffdlconfig)
 
         if url_list:
             d = ViewLog(_("List of URLs"),"\n".join(url_list),parent=self.gui)
@@ -376,13 +369,14 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
             options['version'] = self.version
             print(self.version)
             self.start_downloads( options, update_books )
-        
-    def get_urls_clip(self):
+            
+    def get_urls_clip(self,storyurls=True):
         url_list = []
         if prefs['urlsfromclip']:
             for url in unicode(QApplication.instance().clipboard().text()).split():
-                if( self._is_good_downloader_url(url) ):
+                if not storyurls or self._is_good_downloader_url(url):
                     url_list.append(url)
+                    
         return url_list
 
     def apply_settings(self):
@@ -439,13 +433,6 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         print("url:%s"%url)
         skip_date_update = False
         
-        ## was self.ffdlconfig, but we need to be able to change it
-        ## when doing epub update.
-        ffdlconfig = SafeConfigParser()
-        ffdlconfig.readfp(StringIO(get_resources("plugin-defaults.ini")))
-        ffdlconfig.readfp(StringIO(prefs['personal.ini']))
-        adapter = adapters.getAdapter(ffdlconfig,url,fileform)
-
         options['personal.ini'] = prefs['personal.ini']
         if prefs['includeimages']:
             # this is a cheat to make it easier for users.
@@ -455,6 +442,13 @@ keep_summary_html:true
 make_firstimage_cover:true
 ''' + options['personal.ini']
 
+        ## was self.ffdlconfig, but we need to be able to change it
+        ## when doing epub update.
+        ffdlconfig = SafeConfigParser()
+        ffdlconfig.readfp(StringIO(get_resources("plugin-defaults.ini")))
+        ffdlconfig.readfp(StringIO(options['personal.ini']))
+        adapter = adapters.getAdapter(ffdlconfig,url,fileform)
+
         ## three tries, that's enough if both user/pass & is_adult needed,
         ## or a couple tries of one or the other
         for x in range(0,2):
@@ -480,11 +474,11 @@ make_firstimage_cover:true
 
         book['all_metadata'] = story.getAllMetadata(removeallentities=True)
         book['title'] = story.getMetadata("title", removeallentities=True)
-        book['author_sort'] = book['author'] = story.getMetadata("author", removeallentities=True)
+        book['author_sort'] = book['author'] = story.getList("author", removeallentities=True)
         book['publisher'] = story.getMetadata("site")
-        book['tags'] = writer.getTags() # getTags could be moved up into adapter now.  Adapter didn't used to know the fileform
-        book['comments'] = stripHTML(story.getMetadata("description")) #, removeallentities=True) comments handles entities better.
-        book['series'] = story.getMetadata("series")
+        book['tags'] = writer.getTags(removeallentities=True) # getTags could be moved up into adapter now.  Adapter didn't used to know the fileform
+        book['comments'] = sanitize_comments_html(story.getMetadata("description"))        
+        book['series'] = story.getMetadata("series", removeallentities=True)
         
         # adapter.opener is the element with a threadlock.  But del
         # adapter.opener doesn't work--subproc fails when it tries
@@ -496,6 +490,7 @@ make_firstimage_cover:true
         book['password'] = adapter.password
 
         book['icon'] = 'plus.png'
+        book['status'] = 'Add'
         if story.getMetadataRaw('datePublished'):
             # should only happen when an adapter is broken, but better to
             # fail gracefully.
@@ -504,6 +499,7 @@ make_firstimage_cover:true
         
         if collision in (CALIBREONLY):
             book['icon'] = 'metadata.png'
+            book['status'] = 'Meta'
         
         # Dialogs should prevent this case now.
         if collision in (UPDATE,UPDATEALWAYS) and fileform != 'epub':
@@ -520,36 +516,46 @@ make_firstimage_cover:true
         # only care about collisions when not ADDNEW
         elif collision != ADDNEW:
             # 'new' book from URL.  collision handling applies.
-            print("from URL")
+            print("from URL(%s)"%url)
 
-            # find dups
-            mi = MetaInformation(story.getMetadata("title", removeallentities=True),
-                                 [story.getMetadata("author", removeallentities=True)]) # author is a list.
-            identicalbooks = db.find_identical_books(mi)
-            ## removed for being overkill.
-            # for ib in identicalbooks:
-            #     # only *really* identical if URL matches, too.
-            #     # XXX make an option?
-            #     if self._get_story_url(db,ib) == url:
-            #         identicalbooks.append(ib)
-            #print("identicalbooks:%s"%identicalbooks)
+            # try to find by identifier url first.
+            searchstr = 'identifiers:"=url:%s"'%url.replace(":","|")
+            identicalbooks = db.search_getting_ids(searchstr, None)
+            if len(identicalbooks) < 1:
+                # find dups
+                authlist = story.getList("author", removeallentities=True)
+                if len(authlist) > 100 and calibre_version < (0, 8, 61):
+                    ## should be fixed from 0.8.61 on.  In the
+                    ## meantime, if it matches the title *and* first
+                    ## 100 authors, I'm prepared to assume it's a
+                    ## match.
+                    print("reduce author list to 100 only when calibre < 0.8.61")
+                    authlist = authlist[:100]
+                mi = MetaInformation(story.getMetadata("title", removeallentities=True),
+                                     authlist)
+                identicalbooks = db.find_identical_books(mi)
+                if len(identicalbooks) > 0:
+                    print("existing found by title/author(s)")
 
+            else:
+                print("existing found by identifier URL")
+                
             if collision == SKIP and identicalbooks:
                 raise NotGoingToDownload("Skipping duplicate story.","list_remove.png")
 
             if len(identicalbooks) > 1:
-                raise NotGoingToDownload("More than one identical book--can't tell which to update/overwrite.","minusminus.png")
+                raise NotGoingToDownload("More than one identical book by Identifer URL or title/author(s)--can't tell which book to update/overwrite.","minusminus.png")
 
             ## changed: add new book when CALIBREONLY if none found.
             if collision == CALIBREONLY and not identicalbooks:
                 collision = ADDNEW
                 options['collision'] = ADDNEW
-            #     raise NotGoingToDownload("Not updating Calibre Metadata, no existing book to update.","search_delete_saved.png")
 
             if len(identicalbooks)>0:
                 book_id = identicalbooks.pop()
                 book['calibre_id'] = book_id
                 book['icon'] = 'edit-redo.png'
+                book['status'] = 'Update'
 
         if book_id != None and collision != ADDNEW:
             if collision in (CALIBREONLY):
@@ -658,7 +664,6 @@ make_firstimage_cover:true
             custom_columns = self.gui.library_view.model().custom_columns
             if prefs['errorcol'] != '' and prefs['errorcol'] in custom_columns:
                 label = custom_columns[prefs['errorcol']]['label']
-                print("errorcol label:%s"%label)
                 ## if error column and all bad.
                 self.previous = self.gui.library_view.currentIndex()
                 LoopProgressDialog(self.gui,
@@ -696,7 +701,7 @@ make_firstimage_cover:true
             self._add_or_update_book(book,options,prefs,mi)
 
         if options['collision'] == CALIBREONLY or \
-                (options['updatemeta'] and book['good']):
+                ( (options['updatemeta'] or book['added']) and book['good'] ):
             self._update_metadata(db, book['calibre_id'], book, mi, options)
 
     def _update_bad_book(self,book,db=None,label='errorcol',
@@ -704,9 +709,9 @@ make_firstimage_cover:true
                                   'collision':ADDNEW,
                                   'updatemeta':True,
                                   'updateepubcover':True},):
-        
-        print("add/update bad %s %s %s"%(book['title'],book['url'],book['comment']))
-        db.set_custom(book['calibre_id'], book['comment'], label=label, commit=True)
+        if book['calibre_id']:
+            print("add/update bad %s %s %s"%(book['title'],book['url'],book['comment']))
+            db.set_custom(book['calibre_id'], book['comment'], label=label, commit=True)
 
     def _update_books_completed(self, book_list, options={}, showlist=True):
         
@@ -720,7 +725,7 @@ make_firstimage_cover:true
             self.gui.library_view.model().books_added(len(add_list))
             self.gui.library_view.model().refresh_ids(add_ids)
 
-        if update_ids:
+        if len(update_list):
             self.gui.library_view.model().refresh_ids(update_ids)
 
         current = self.gui.library_view.currentIndex()
@@ -744,7 +749,13 @@ make_firstimage_cover:true
     
         print("all done, remove temp dir.")
         remove_dir(options['tdir'])
-        
+
+        all_ids = add_ids
+        all_ids.extend(update_ids)
+        if 'Count Pages' in self.gui.iactions and len(prefs['countpagesstats']) and len(all_ids):
+            cp_plugin = self.gui.iactions['Count Pages']
+            cp_plugin.count_statistics(all_ids,prefs['countpagesstats'])
+
     def download_list_completed(self, job, options={}):
         if job.failed:
             self.gui.job_exception(job, dialog_title='Failed to Download Stories')
@@ -753,47 +764,72 @@ make_firstimage_cover:true
         self.previous = self.gui.library_view.currentIndex()
         db = self.gui.current_db
 
-        if display_story_list(self.gui,
-                              'Downloads finished, confirm to update Calibre',
-                              prefs,
-                              self.qaction.icon(),
-                              job.result,
-                              label_text='Stories will not be added or updated in Calibre without confirmation.',
-                              offer_skip=True):
-            
-            book_list = job.result
-            good_list = filter(lambda x : x['good'], book_list)
-            total_good = len(good_list)
+        book_list = job.result
+        good_list = filter(lambda x : x['good'], book_list)
+        bad_list = filter(lambda x : x['calibre_id'] and not x['good'], book_list)
+        payload = (good_list, bad_list, options)
+        
+        msg = '''
+<p>FFDL found <b>%s</b> good and <b>%s</b> bad updates.</p>
+<p>See log for details.</p>
+<p>Proceed with updating your library?</p>
+'''%(len(good_list),len(bad_list))
 
-            self.gui.status_bar.show_message(_('Adding/Updating %s books.'%total_good))
+        htmllog='<html><body><table border="1"><tr><th>Status</th><th>Title</th><th>Author</th><th>Comment</th><th>URL</th></tr>'
+        for book in good_list:
+            if 'status' in book:
+                status = book['status']
+            else:
+                status = 'Good'
+            htmllog = htmllog + '<tr><td>' + '</td><td>'.join([status,book['title'],", ".join(book['author']),book['comment'],book['url']]) + '</td></tr>'
+        
+        for book in bad_list:
+            if 'status' in book:
+                status = book['status']
+            else:
+                status = 'Bad'
+            htmllog = htmllog + '<tr><td>' + '</td><td>'.join([status,book['title'],", ".join(book['author']),book['comment'],book['url']]) + '</td></tr>'
+        
+        htmllog = htmllog + '</table></body></html>'
+        
+        self.gui.proceed_question(self._do_download_list_update,
+                payload, htmllog,
+                'FFDL log', 'FFDL download complete', msg,
+                show_copy_button=False)
+        
+        
+    def _do_download_list_update(self, payload):
+        
+        (good_list,bad_list,options) = payload
+        total_good = len(good_list)
 
-            if total_good > 0:
+        self.gui.status_bar.show_message(_('Adding/Updating %s books.'%total_good))
+
+        if total_good > 0:
+            LoopProgressDialog(self.gui,
+                               good_list,
+                               partial(self._update_book, options=options, db=self.gui.current_db),
+                               partial(self._update_books_completed, options=options),
+                               init_label="Updating calibre for FanFiction stories...",
+                               win_title="Update calibre for FanFiction stories",
+                               status_prefix="Updated")
+
+        total_bad = len(bad_list)
+
+        if total_bad > 0:
+            custom_columns = self.gui.library_view.model().custom_columns
+            if prefs['errorcol'] != '' and prefs['errorcol'] in custom_columns:
+                self.gui.status_bar.show_message(_('Adding/Updating %s BAD books.'%total_bad))
+                label = custom_columns[prefs['errorcol']]['label']
+                ## if error column and all bad.
                 LoopProgressDialog(self.gui,
-                                   good_list,
-                                   partial(self._update_book, options=options, db=self.gui.current_db),
-                                   partial(self._update_books_completed, options=options),
-                                   init_label="Updating calibre for FanFiction stories...",
-                                   win_title="Update calibre for FanFiction stories",
+                                   bad_list,
+                                   partial(self._update_bad_book, label=label, options=options, db=self.gui.current_db),
+                                   partial(self._update_books_completed, options=options, showlist=False),
+                                   init_label="Updating calibre for BAD FanFiction stories...",
+                                   win_title="Update calibre for BAD FanFiction stories",
                                    status_prefix="Updated")
 
-            bad_list = filter(lambda x : x['calibre_id'] and not x['good'], book_list)
-            total_bad = len(bad_list)
-
-            if total_bad > 0:
-                custom_columns = self.gui.library_view.model().custom_columns
-                if prefs['errorcol'] != '' and prefs['errorcol'] in custom_columns:
-                    self.gui.status_bar.show_message(_('Adding/Updating %s BAD books.'%total_bad))
-                    label = custom_columns[prefs['errorcol']]['label']
-                    print("errorcol label:%s"%label)
-                    ## if error column and all bad.
-                    LoopProgressDialog(self.gui,
-                                       bad_list,
-                                       partial(self._update_bad_book, label=label, options=options, db=self.gui.current_db),
-                                       partial(self._update_books_completed, options=options, showlist=False),
-                                       init_label="Updating calibre for BAD FanFiction stories...",
-                                       win_title="Update calibre for BAD FanFiction stories",
-                                       status_prefix="Updated")
-
                 
     def _add_or_update_book(self,book,options,prefs,mi=None):
         db = self.gui.current_db
@@ -816,6 +852,7 @@ make_firstimage_cover:true
             book['comment'] = "Adding format to book failed for some reason..."
             book['good']=False
             book['icon']='dialog_error.png'
+            book['status'] = 'Error'
 
         if prefs['deleteotherforms']:
             fmts = db.formats(book['calibre_id'], index_is_id=True).split(',')
@@ -830,6 +867,7 @@ make_firstimage_cover:true
         return book_id
 
     def _update_metadata(self, db, book_id, book, mi, options):
+        oldmi = db.get_metadata(book_id,index_is_id=True)
         if prefs['keeptags']:
             old_tags = db.get_tags(book_id)
             # remove old Completed/In-Progress only if there's a new one.
@@ -845,7 +883,6 @@ make_firstimage_cover:true
             mi.languages=[book['all_metadata']['langcode']]
         else:
             # Set language english, but only if not already set.
-            oldmi = db.get_metadata(book_id,index_is_id=True)
             if not oldmi.languages:
                 mi.languages=['eng']
 
@@ -857,9 +894,25 @@ make_firstimage_cover:true
 
         # set author link if found.  All current adapters have authorUrl, except anonymous on AO3.
         if 'authorUrl' in book['all_metadata']:
-            autid=db.get_author_id(book['author'])
-            db.set_link_field_for_author(autid, unicode(book['all_metadata']['authorUrl']),
-                                         commit=False, notify=False)
+            authurls = book['all_metadata']['authorUrl'].split(", ")
+            for i, auth in enumerate(book['author']):
+                autid=db.get_author_id(auth)
+                db.set_link_field_for_author(autid, unicode(authurls[i]),
+                                             commit=False, notify=False)
+
+        # implement 'newonly' flags here by setting to the current
+        # value again.
+        if not book['added']:
+            for (col,newonly) in prefs['std_cols_newonly'].iteritems():
+                if newonly:
+                    if col == "identifiers":
+                        mi.set_identifiers(oldmi.get_identifiers())
+                    else:
+                        try:
+                            mi.__setattr__(col,oldmi.__getattribute__(col))
+                        except AttributeError:
+                            print("AttributeError? %s"%col)
+                            pass
             
         db.set_metadata(book_id,mi)
 
@@ -874,6 +927,9 @@ make_firstimage_cover:true
                 print("%s not an existing column, skipping."%col)
                 continue
             coldef = custom_columns[col]
+            if col in prefs['custom_cols_newonly'] and prefs['custom_cols_newonly'][col] and not book['added']:
+                print("Skipping custom column(%s) update, set to New Books Only"%coldef['name'])
+                continue
             if not meta.startswith('status-') and meta not in book['all_metadata'] or \
                     meta.startswith('status-') and 'status' not in book['all_metadata']:
                 print("No value for %s, skipping custom column(%s) update."%(meta,coldef['name']))
@@ -895,8 +951,13 @@ make_firstimage_cover:true
                 db.set_custom(book_id, val, label=label, commit=False)
 
         db.commit()
-        
+
         if 'Generate Cover' in self.gui.iactions and (book['added'] or not prefs['gcnewonly']):
+            
+            # force a refresh if generating cover so complex composite
+            # custom columns are current and correct
+            db.refresh_ids([book_id])
+                
             gc_plugin = self.gui.iactions['Generate Cover']
             setting_name = None
             if prefs['allow_gc_from_ini']:
@@ -911,7 +972,7 @@ make_firstimage_cover:true
                 for line in adapter.getConfig('generate_cover_settings').splitlines():
                     if "=>" in line:
                         (template,regexp,setting) = map( lambda x: x.strip(), line.split("=>") )
-                        value = Template(template).substitute(book['all_metadata']).encode('utf8')
+                        value = Template(template).safe_substitute(book['all_metadata']).encode('utf8')
                         print("%s(%s) => %s => %s"%(template,value,regexp,setting))
                         if re.search(regexp,value):
                             setting_name = setting
@@ -937,7 +998,6 @@ make_firstimage_cover:true
         ## if error column set.
         if prefs['errorcol'] != '' and prefs['errorcol'] in custom_columns:
             label = custom_columns[prefs['errorcol']]['label']
-            print("_update_book label:%s"%label)
             db.set_custom(book['calibre_id'], '', label=label, commit=True) # book['comment']
 
     def _get_clean_reading_lists(self,lists):
@@ -1001,19 +1061,19 @@ make_firstimage_cover:true
                         message="<p>You configured FanFictionDownLoader to automatically update Reading List '%s', but you don't have a list of that name?</p>"%l
                         confirm(message,'fanfictiondownloader_no_reading_list_%s'%l, self.gui)
 
-    def _find_existing_book_id(self,db,book,matchurl=True):
-        mi = MetaInformation(book["title"],[book["author"]]) # author is a list.
-        identicalbooks = db.find_identical_books(mi)
-        if matchurl: # only *really* identical if URL matches, too.
-            for ib in identicalbooks:
-                if self._get_story_url(db,ib) == book['url']:
-                    return ib
-        if identicalbooks:
-            return identicalbooks.pop()
-        return None
+    # def _find_existing_book_id(self,db,book,matchurl=True):
+    #     mi = MetaInformation(book["title"],book["author"]) # author is a list.
+    #     identicalbooks = db.find_identical_books(mi)
+    #     if matchurl: # only *really* identical if URL matches, too.
+    #         for ib in identicalbooks:
+    #             if self._get_story_url(db,ib) == book['url']:
+    #                 return ib
+    #     if identicalbooks:
+    #         return identicalbooks.pop()
+    #     return None
     
     def _make_mi_from_book(self,book):
-        mi = MetaInformation(book['title'],[book['author']]) # author is a list.
+        mi = MetaInformation(book['title'],book['author']) # author is a list.
         mi.set_identifiers({'url':book['url']})
         mi.publisher = book['publisher']
         mi.tags = book['tags']
@@ -1050,8 +1110,7 @@ make_firstimage_cover:true
         book['good'] = True
         book['calibre_id'] = None
         book['title'] = 'Unknown'
-        book['author'] = 'Unknown'
-        book['author_sort'] = 'Unknown'
+        book['author_sort'] = book['author'] = ['Unknown'] # list
         book['begin'] = None
         book['end'] = None
             
@@ -1067,8 +1126,7 @@ make_firstimage_cover:true
         book['good'] = good
         book['calibre_id'] = idval
         book['title'] = 'Unknown'
-        book['author'] = 'Unknown'
-        book['author_sort'] = 'Unknown'
+        book['author_sort'] = book['author'] = ['Unknown'] # list
         book['begin'] = None
         book['end'] = None
             
@@ -1084,7 +1142,7 @@ make_firstimage_cover:true
         book['good'] = True
         book['calibre_id'] = mi.id
         book['title'] = mi.title
-        book['author'] = authors_to_string(mi.authors)
+        book['author'] = mi.authors
         book['author_sort'] = mi.author_sort
         book['comment'] = ''
         book['url'] = ""
@@ -1099,6 +1157,7 @@ make_firstimage_cover:true
             book['comment'] = "No story URL found."
             book['good'] = False
             book['icon'] = 'search_delete_saved.png'
+            book['status'] = 'Not Found'
         else:
             # get normalized url or None.
             book['url'] = self._is_good_downloader_url(url)
@@ -1107,6 +1166,7 @@ make_firstimage_cover:true
                 book['comment'] = "URL is not a valid story URL."
                 book['good'] = False
                 book['icon']='dialog_error.png'
+                book['status'] = 'Bad URL'
     
     def _get_story_url(self, db, book_id):
         identifiers = db.get_identifiers(book_id,index_is_id=True)

calibre-plugin/jobs.py

@@ -109,6 +109,9 @@ def do_download_for_worker(book,options):
     when run as a worker job
     '''
     try:
+        # import logging
+        # logging.basicConfig(level=logging.DEBUG,format="%(levelname)s:%(filename)s(%(lineno)d):%(message)s")
+        
         book['comment'] = 'Download started...'
 
         ffdlconfig = SafeConfigParser()
@@ -117,6 +120,11 @@ def do_download_for_worker(book,options):
         
         if not options['updateepubcover'] and 'epub_for_update' in book and options['collision'] in (UPDATE, UPDATEALWAYS):
             ffdlconfig.set("overrides","never_make_cover","true")
+
+        # images only for epub, even if the user mistakenly turned it
+        # on else where.
+        if options['fileform'] != "epub":
+            ffdlconfig.set("overrides","include_images","false")
         
         adapter = adapters.getAdapter(ffdlconfig,book['url'],options['fileform'])
         adapter.is_adult = book['is_adult'] 
@@ -154,11 +162,13 @@ def do_download_for_worker(book,options):
             # update now handled by pre-populating the old images and
             # chapters in the adapter rather than merging epubs.
             urlchaptercount = int(story.getMetadata('numChapters'))
-            (url,chaptercount,
+            (url,
+             chaptercount,
              adapter.oldchapters,
              adapter.oldimgs,
              adapter.oldcover,
-             adapter.calibrebookmark) = get_update_data(book['epub_for_update'])
+             adapter.calibrebookmark,
+             adapter.logfile) = get_update_data(book['epub_for_update'])
 
             print("Do update - epub(%d) vs url(%d)" % (chaptercount, urlchaptercount))
             print("write to %s"%outfile)
@@ -177,6 +187,7 @@ def do_download_for_worker(book,options):
         book['good']=False
         book['comment']=unicode(e)
         book['icon']='dialog_error.png'
+        book['status'] = 'Error'
         print("Exception: %s:%s"%(book,unicode(e)))
         traceback.print_exc()
 

defaults.ini

@@ -38,6 +38,7 @@ category_label:Category
 genre_label:Genre
 language_label:Language
 characters_label:Characters
+ships_label:Relationships
 series_label:Series
 ## Completed/In-Progress
 status_label:Status
@@ -70,7 +71,7 @@ version_label:FFDL Version
 ## You can include extra text or HTML that will be included as-is in
 ## the title page. Eg: titlepage_entries: ...,<br />,summary,<br />,...
 ## All current formats already include title and author.
-titlepage_entries: series,category,genre,language,characters,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,description
+titlepage_entries: series,category,genre,language,characters,ships,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,description
 
 ## Try to collect series name and number of this story in series.
 ## Some sites (ab)use 'series' for reading lists and personal
@@ -121,11 +122,20 @@ allow_unsafe_filename: false
 
 ## entries to make epub subjects and calibre tags
 ## lastupdate creates two tags: "Last Update Year/Month: %Y/%m" and "Last Update: %Y/%m/%d"
-include_subject_tags: extratags, genre, category, characters, lastupdate, status
+include_subject_tags: extratags, genre, category, characters, ships, lastupdate, status
 
 ## extra tags (comma separated) to include, primarily for epub.
 extratags: FanFiction
 
+## extra categories, genres, characters, ships and warnings can be
+## configured.  Used primarily for sites that are dedicated to a genre
+## or 'ship and so don't included it for every story.
+#extracategories:
+#extragenres:
+#extracharacters:
+#extraships:
+#extrawarnings:
+
 ## number of seconds to sleep between calls to the story site.  May by
 ## useful if pulling large numbers of stories or if the site is slow.
 #slow_down_sleep_time:0.5
@@ -154,6 +164,45 @@ extratags: FanFiction
 ## doesn't work on some devices either.)
 #replace_hr: false
 
+## If set false, the summary will have all html stripped.
+## Both this and include_images must be true to get images in the
+## summary.
+keep_summary_html:true
+
+## Don't like the numbers at the start of chapter titles on some
+## sites?  You can use strip_chapter_numbers to strip them off.  Just
+## want to make them all look the same?  Strip them off, then add them
+## back on with add_chapter_numbers.  Don't like the way it strips
+## numbers or adds them back?  See chapter_title_strip_pattern and
+## chapter_title_add_pattern.
+strip_chapter_numbers:false
+add_chapter_numbers:false
+
+## (Two versions of chapter_title_strip_pattern are shown below.  You
+## should only have one uncommented.)
+## This version will remove the leading number from:
+## "1." => ""
+## "1. The Beginning" => "The Beginning"
+## "1: Start" => "Start"
+## "2, Chapter the second" => "Chapter the second"
+## etc
+chapter_title_strip_pattern:^[0-9]+[\.: -]+
+
+## This version will strip all of the above *plus* remove 'Chapter 1':
+## "Chapter 1" => ""
+## "1. Chapter 1" => ""
+## "1. Chapter 1, Bob's First Clue" => "Bob's First Clue"
+## "Chapter 2 - Pirates Place" => "Pirates Place"
+## etc
+#chapter_title_strip_pattern:^([0-9]+[\.: -]+)?(Chapter *[0-9]+[\.:, -]*)?
+
+## Uses a python template substitution.  The ${index} is the 'chapter'
+## number and ${title} is the chapter title, after applying
+## chapter_title_strip_pattern.  Those are the only variables available.
+## "The Beginning" => "1. The Beginning" 
+chapter_title_add_pattern:${index}. ${title}
+
+
 ## Each output format has a section that overrides [defaults]
 [html]
 
@@ -197,9 +246,24 @@ windows_eol: true
 zip_output: false
 
 ## epub carries the TOC in metadata.
-## mobi generated from epub will have a TOC at the end.
+## mobi generated from epub by calibre will have a TOC at the end.
 include_tocpage: false
 
+## include a Update Log page before the story text.  If included, the
+## log will be updated each time the epub is an all the metadata
+## fields that have changed since the last update (typically
+## dateUpdated,numChapters,numWords at a minimum) will be shown.
+## Great for tracking when chapters came out and when the description,
+## etc changed.
+include_logpage: false
+
+## items to include in the log page Empty metadata entries, or those
+## that haven't changed since the last update, will *not* appear, even
+## if in the list.  You can include extra text or HTML that will be
+## included as-is in each log entry. Eg: logpage_entries: ...,<br />,
+## summary,<br />,...
+logpage_entries: dateCreated,datePublished,dateUpdated,numChapters,numWords,status,title,author,description,category,genre,rating,warnings
+
 ## epub->mobi conversions typically don't like tables.
 titlepage_use_table: false
 
@@ -214,10 +278,12 @@ background_color: ffffff
 ## Allow customization of CSS.  Make sure to keep at least one space
 ## at the start of each line and to escape % to %%.  Also need
 ## background_color to be in the same section, if included in CSS.
+## 'adobe-text-layout: optimizeSpeed;' prevents hyphenation on newer Nooks
 output_css: 
  body { background-color: #%(background_color)s;
         text-align: justify;
-        margin: 2%%; }
+        margin: 2%%;
+	adobe-text-layout: optimizeSpeed; }
  pre { font-size: x-small; }
  sml { font-size: small; }
  h1 { text-align: center; }
@@ -244,11 +310,6 @@ output_css:
 ## stories.  Images will be converted to jpg for size if possible.
 #include_images:false
 
-## If not set, the summary will have all html stripped for safety.
-## Both this and include_images must be true to get images in the
-## summary.
-#keep_summary_html:false
-
 ## If set, the first image found will be made the cover image.  If
 ## keep_summary_html is true, any images in summary will be before any
 ## in chapters.
@@ -301,6 +362,11 @@ nook_img_fix:true
 ## URLs like: http://test1.com?sid=12345
 [test1.com]
 extratags: FanFiction,Testing
+extracategories:Fafner
+extragenres:Romance,Fluff
+extracharacters:Reginald Smythe-Smythe,Mokona
+extraships:Smythe-Smythe/Mokona
+extrawarnings:Extreme Bogosity
 
 ## If necessary, you can define [<site>:<format>] sections to
 ## customize the formats differently for the same site.  Overrides
@@ -327,6 +393,11 @@ extratags: FanFiction,Testing,HTML
 #is_adult:true
 
 [ashwinder.sycophanthex.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+extracharacters:Severus Snape,Hermione Granger
+extraships:Severus Snape/Hermione Granger
+
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -335,6 +406,9 @@ extratags: FanFiction,Testing,HTML
 #password:yourpassword
 
 [castlefans.org]
+## Site dedicated to these categories/characters/ships
+extracategories:Castle
+
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -357,7 +431,25 @@ extratags: FanFiction,Testing,HTML
 ## cover image.  This lets you exclude them.
 cover_exclusion_regexp:/images/.*?ribbon.gif
 
+[dark-solace.org]
+## Site dedicated to these categories/characters/ships
+extracategories:Buffy: The Vampire Slayer
+extracharacters:Buffy, Spike
+extraships:Spike/Buffy
+
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
 [dramione.org]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+extracharacters:Draco Malfoy,Hermione Granger
+extraships:Draco Malfoy/Hermione Granger
+
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -371,6 +463,9 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 #is_adult:true
 
 [erosnsappho.sycophanthex.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
@@ -381,6 +476,9 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 cover_exclusion_regexp:/images/.*?ribbon.gif
 
 [fanfiction.mugglenet.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -394,6 +492,10 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 #is_adult:true
 
 [fanfiction.portkey.org]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+extraships:Harry Potter/Hermione Granger
+
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
@@ -407,23 +509,93 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 #username:YourName
 #password:yourpassword
 
+[finestories.com]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+[grangerenchanted.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+extracharacters:Hermione Granger
+
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
+[hlfiction.net]
+## Site dedicated to these categories/characters/ships
+extracategories:Highlander
+
+[ksarchive.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Star Trek
+extracharacters:Kirk,Spock
+extraships:Kirk/Spock
+
 [lumos.sycophanthex.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+[merlinfic.dtwins.co.uk]
+## Site dedicated to these categories/characters/ships
+extracategories:Merlin
+
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
 [national-library.net]
+## Site dedicated to these categories/characters/ships
+extracategories:West Wing
 
 [ncisfic.com]
+## Site dedicated to these categories/characters/ships
+extracategories:NCIS
 
 [nfacommunity.com]
+## Site dedicated to these categories/characters/ships
+extracategories:NCIS
+
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+[nha.magical-worlds.us]
+## Site dedicated to these categories/characters/ships
+extracategories:Buffy: The Vampire Slayer
+extracharacters:Willow
+
 [occlumency.sycophanthex.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+extracharacters:Severus Snape
+
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -432,6 +604,9 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 #password:yourpassword
 
 [onedirectionfanfiction.com]
+## Site dedicated to these categories/characters/ships
+extracategories:One Direction
+
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -445,12 +620,28 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 #is_adult:true
 
 [ponyfictionarchive.net]
+## Site dedicated to these categories/characters/ships
+extracategories:My Little Pony: Friendship is Magic
+
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+[pretendercentre.com]
+## Site dedicated to these categories/characters/ships
+extracategories:The Pretender
+
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
 [samdean.archive.nu]
+## Site dedicated to these categories/characters/ships
+extracategories:Supernatural
+extracharacters:Sam,Dean
+extraships:Sam/Dean
 
 [sg1-heliopolis.com]
 ## Some sites do not require a login, but do require the user to
@@ -459,10 +650,45 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 #is_adult:true
 
 [stargate-atlantis.org]
+## Site dedicated to these categories/characters/ships
+extracategories:Stargate: Atlantis
+
+[svufiction.com]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
 
 [thehexfiles.net]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+extracharacters:Draco Malfoy,Harry Potter
+extraships:Harry Potter/Draco Malfoy
+
+[themasque.net]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
 
 [thequidditchpitch.org]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -476,14 +702,40 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 #is_adult:true
 
 [www.adastrafanfic.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Star Trek
+
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+[www.dracoandginny.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+extracharacters:Draco Malfoy,Ginny Weasley
+extraships:Draco Malfoy/Ginny Weasley
+
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
 [www.thealphagate.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Stargate: SG-1
 
 [www.checkmated.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -498,6 +750,10 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 #is_adult:true
 
 [www.dokuga.com]
+## Site dedicated to these categories/characters/ships
+extracategories:InuYasha
+extracharacters:Sesshoumaru,Kagome
+extraships:Sesshoumaru/Kagome
 
 [www.fanfiction.net]
 ## fanfiction.net's 'cover' images are really just tiny thumbnails.
@@ -515,6 +771,9 @@ never_make_cover: true
 [www.ficbook.net]
 
 [www.fictionalley.org]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
@@ -536,6 +795,9 @@ extratags:
 #password:yourpassword
 
 [www.fimfiction.net]
+## Site dedicated to these categories/characters/ships
+extracategories:My Little Pony: Friendship is Magic
+
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
@@ -547,18 +809,33 @@ extratags:
 #fail_on_password: false
 
 [www.harrypotterfanfiction.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
 [www.hpfandom.net]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+[www.hpfanficarchive.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
 [www.ik-eternal.net]
+## Site dedicated to these categories/characters/ships
+extracategories:InuYasha
+extracharacters:InuYasha,Kagome
+extraships:InuYasha/Kagome
+
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -572,6 +849,9 @@ extratags:
 #is_adult:true
 
 [www.libraryofmoria.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Lord of the Rings
+
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
@@ -580,6 +860,9 @@ extratags:
 [www.mediaminer.org]
 
 [www.midnightwhispers.ca]
+## Site dedicated to these categories/characters/ships
+extracategories:Queer as Folk
+
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
@@ -590,8 +873,14 @@ extratags:
 cover_exclusion_regexp:/stories/999/images/.*?_trophy.png
 
 [www.ncisfiction.com]
+## Site dedicated to these categories/characters/ships
+extracategories:NCIS
 
 [www.phoenixsong.net]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+extraships:Harry Potter/Ginny Weasley
+
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -600,16 +889,43 @@ cover_exclusion_regexp:/stories/999/images/.*?_trophy.png
 #password:yourpassword
 
 [www.potionsandsnitches.net]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
 
 [www.prisonbreakfic.net]
+## Site dedicated to these categories/characters/ships
+extracategories:Prison Break
+
+[www.scarvesandcoffee.net]
+## Site dedicated to these categories/characters/ships
+extracategories:Glee
+extracharacters:Kurt Hummel,Blaine Anderson
+
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+[www.sinful-desire.org]
+## Site dedicated to these categories/characters/ships
+extracategories:Supernatural
+
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
 
 [www.siye.co.uk]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+extracharacters:Harry Potter,Ginny Weasley
+extraships:Harry Potter/Ginny Weasley
 
 [www.squidge.org/peja]
 # www.squidge.org/peja calls it Fandom <shrug>
 category_label:Fandom
 # Remove numWords -- www.squidge.org/peja word counts are inaccurate
-titlepage_entries: series,category,genre,language,characters,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,site,description
+titlepage_entries: series,category,genre,language,characters,ships,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,site,description
 
 [www.squidge.org/peja:txt]
 ## Add URLs since there aren't links.
@@ -617,6 +933,16 @@ titlepage_entries: series,category,genre,language,characters,status,datePublishe
 titlepage_entries: series,category,genre,language,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,site,storyUrl, authorUrl, description
 
 [www.storiesofarda.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Lord of the Rings
+
+[www.thepetulantpoetess.com]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
 
 [www.thewriterscoffeeshop.com]
 ## Some sites require login (or login for some rated stories) The
@@ -647,7 +973,14 @@ collect_series: false
 #username:YourName
 #password:yourpassword
 
+[www.twilightarchives.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Twilight
+
 [www.twilighted.net]
+## Site dedicated to these categories/characters/ships
+extracategories:Twilight
+
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -659,6 +992,9 @@ collect_series: false
 collect_series: false
 
 [www.twiwrite.net]
+## Site dedicated to these categories/characters/ships
+extracategories:Twilight
+
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -670,6 +1006,11 @@ collect_series: false
 collect_series: false
 
 [www.walkingtheplank.org]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+extracharacters:Severus Snape,Harry Potter
+extraships:Severus Snape/Harry Potter
+
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
@@ -677,7 +1018,28 @@ collect_series: false
 
 [www.whofic.com]
 
+[www.wizardtales.net]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
+[www.wolverineandrogue.com]
+## Site dedicated to these categories/characters/ships
+extracategories:X-Men Movie
+extracharacters:Wolverine,Rogue
+
 [www.wraithbait.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Stargate: Atlantis
+
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.

downloader.py

@@ -16,8 +16,6 @@
 #
 
 import logging
-## XXX cli option for logging level.
-logging.basicConfig(level=logging.INFO,format="%(levelname)s:%(filename)s(%(lineno)d):%(message)s")
 
 import sys, os
 from os.path import normpath, expanduser, isfile, join
@@ -74,9 +72,18 @@ def main():
    parser.add_option("-l", "--list",
                      action="store_true", dest="list",
                      help="Get list of valid story URLs from page given.",)
+   parser.add_option("-d", "--debug",
+                     action="store_true", dest="debug",
+                     help="Show debug output while downloading.",)
    
    (options, args) = parser.parse_args()
 
+   if options.debug:
+       logging.basicConfig(level=logging.DEBUG,format="%(levelname)s:%(filename)s(%(lineno)d):%(message)s")
+   else:
+       logging.basicConfig(level=logging.INFO,format="%(levelname)s:%(filename)s(%(lineno)d):%(message)s")
+
+   
    if len(args) != 1:
        parser.error("incorrect number of arguments")
 
@@ -114,6 +121,11 @@ def main():
 
    if options.update and not options.updatecover:
        config.set("overrides","never_make_cover","true")
+
+   # images only for epub, even if the user mistakenly turned it
+   # on else where.
+   if options.format != "epub":
+       config.set("overrides","include_images","false")
        
    if options.options:
        for opt in options.options:
@@ -121,7 +133,7 @@ def main():
            config.set("overrides",var,val)
 
    if options.list:
-       retlist = get_urls_from_page(args[0])
+       retlist = get_urls_from_page(args[0], config)
        print "\n".join(retlist)
                
        return
@@ -141,11 +153,16 @@ def main():
        ## Check for include_images and absence of PIL, give warning.
        if adapter.getConfig('include_images'):
            try:
-               import Image
+               from calibre.utils.magick import Image
+               logging.debug("Using calibre.utils.magick")
            except:
-               print "You have include_images enabled, but Python Image Library(PIL) isn't found.\nImages will be included full size in original format.\nContinue? (y/n)?"
-               if not sys.stdin.readline().strip().lower().startswith('y'):
-                   return
+               try:
+                   import Image
+                   logging.debug("Using PIL")
+               except:
+                   print "You have include_images enabled, but Python Image Library(PIL) isn't found.\nImages will be included full size in original format.\nContinue? (y/n)?"
+                   if not sys.stdin.readline().strip().lower().startswith('y'):
+                       return
                
 
        ## three tries, that's enough if both user/pass & is_adult needed,
@@ -181,11 +198,13 @@ def main():
                    # update now handled by pre-populating the old
                    # images and chapters in the adapter rather than
                    # merging epubs.
-                   (url,chaptercount,
+                   (url,
+                    chaptercount,
                     adapter.oldchapters,
                     adapter.oldimgs,
                     adapter.oldcover,
-                    adapter.calibrebookmark) = get_update_data(args[0])
+                    adapter.calibrebookmark,
+                    adapter.logfile) = get_update_data(args[0])
 
                    writeStory(config,adapter,"epub")
                    

example.ini

@@ -1,5 +1,5 @@
 ## This is an example of what your personal configuration might look
-## like.
+## like.  Uncomment options by removing the '#' in front of them.
 
 [defaults]
 ## Some sites also require the user to confirm they are adult for
@@ -7,34 +7,86 @@
 ## personal.ini, not defaults.ini.
 #is_adult:true
 
+[epub]
+## include images from img tags in the body and summary of stories.
+## Images will be converted to jpg for size if possible.  Images work
+## in epub format only.  To get mobi or other format with images,
+## download as epub and use Calibre to convert.
+#include_images:true
+
+## If not set, the summary will have all html stripped for safety.
+## Both this and include_images must be true to get images in the
+## summary.
+#keep_summary_html:true
+
+## If set, the first image found will be made the cover image.  If
+## keep_summary_html is true, any images in summary will be before any
+## in chapters.
+#make_firstimage_cover:true
+
+## Resize images down to width, height, preserving aspect ratio.
+## Nook size, with margin.
+#image_max_size: 580, 725
+
+## Change image to grayscale, if graphics library allows, to save
+## space.
+#grayscale_images: false
+
+
 ## Most common, I expect will be using this to save username/passwords
-## for different sites.
+## for different sites.  Here are a few examples.
+
 [www.twilighted.net]
 #username:YourPenname
 #password:YourPassword
+## default is false
+#collect_series: true
 
 [www.ficwad.com]
 #username:YourUsername
 #password:YourPassword
 
+[www.twiwrite.net]
+#username:YourName
+#password:yourpassword
+## default is false
+#collect_series: true
+
 [www.adastrafanfic.com]
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.
 #is_adult:true
 
-## The [defaults] section here will override the system [defaults],
-## but not format, site for site:format sections.
-[defaults]
-## Directories only useful in commandline or zip files.
-#output_filename: books/${title}-${siteabbrev}_${storyId}${formatext}
-#output_filename: books/${site}/${authorId}/${title}-${storyId}${formatext}
+[www.thewriterscoffeeshop.com]
+#username:YourName
+#password:yourpassword
+#is_adult:true
+## default is false
+#collect_series: true
+
+[www.fictionalley.org]
+#is_adult:true
+
+[www.harrypotterfanfiction.com]
+#is_adult:true
+
+[www.fimfiction.net]
+#is_adult:true
+#fail_on_password: false
+
+[www.tthfanfic.org]
+#is_adult:true
+## tth is a little unusual--it doesn't require user/pass, but the site
+## keeps track of which chapters you've read and won't send another
+## update until it thinks you're up to date.  This way, on download,
+## it thinks you're up to date.
+#username:YourName
+#password:yourpassword
 
-## For example, zip_output here will turn on zip for html and txt, but
-## not epub because the system [epub] section explicitly says
-## zip_output: false (epubs *are* specially formated zip files.)
-#zip_output: true
-#zip_filename: ${title}-${siteabbrev}_${storyId}${formatext}.zip
 
 ## This section will override anything in the system defaults or other
 ## sections here.
 [overrides]
+## default varies by site.  Set true here to force all sites to
+## collect series.
+#collect_series: true

fanficdownloader/adapters/__init__.py

@@ -81,6 +81,23 @@ import adapter_ponyfictionarchivenet
 import adapter_sg1heliopoliscom
 import adapter_ncisficcom
 import adapter_nationallibrarynet
+import adapter_themasquenet
+import adapter_pretendercentrecom
+import adapter_darksolaceorg
+import adapter_finestoriescom
+import adapter_hpfanficarchivecom
+import adapter_svufictioncom
+import adapter_twilightarchivescom
+import adapter_wizardtalesnet
+import adapter_nhamagicalworldsus
+import adapter_hlfictionnet
+import adapter_grangerenchantedcom
+import adapter_dracoandginnycom
+import adapter_scarvesandcoffeenet
+import adapter_thepetulantpoetesscom
+import adapter_wolverineandroguecom
+import adapter_sinfuldesireorg
+import adapter_merlinficdtwinscouk
 
 ## This bit of complexity allows adapters to be added by just adding
 ## importing.  It eliminates the long if/else clauses we used to need
@@ -101,7 +118,7 @@ for x in imports():
             
 def getAdapter(config,url,fileform=None):
     ## fix up leading protocol.
-    fixedurl = re.sub(r"(?i)^[htp]+[:/]+","http://",url.strip())
+    fixedurl = re.sub(r"(?i)^[htps]+[:/]+","http://",url.strip())
     if not fixedurl.startswith("http"):
         fixedurl = "http://%s"%url
     ## remove any trailing '#' locations.
@@ -116,8 +133,13 @@ def getAdapter(config,url,fileform=None):
     if( domain != parsedUrl.netloc ):
         fixedurl = fixedurl.replace(parsedUrl.netloc,domain)
 
-    logging.debug("site:"+domain)
+    logging.debug("trying url:"+url)
     cls = getClassFor(domain)
+    if not cls and domain.startswith("www."):
+        domain = domain.replace("www.","")
+        logging.debug("trying site:without www: "+domain)
+        cls = getClassFor(domain)
+        fixedurl = fixedurl.replace("http://www.","http://")
     if not cls:
         logging.debug("trying site:www."+domain)
         cls = getClassFor("www."+domain)

fanficdownloader/adapters/adapter_adastrafanficcom.py

@@ -37,7 +37,6 @@ class AdAstraFanficComSiteAdapter(BaseSiteAdapter):
                                # Most sites that claim to be
                                # iso-8859-1 (and some that claim to be
                                # utf8) are really windows-1252.
-        self.story.addToList("category","Star Trek")
         self.is_adult=False
         
         # get storyId from url--url validation guarantees query is only sid=1234

fanficdownloader/adapters/adapter_archiveofourownorg.py

@@ -50,8 +50,17 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
         self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
         logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/works/'+self.story.getMetadata('storyId'))
+        # get storyId from url--url validation guarantees query correct
+        m = re.match(self.getSiteURLPattern(),url)
+        if m:
+            self.story.setMetadata('storyId',m.group('id'))
+            logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+            # normalized story URL.
+            self._setURL('http://' + self.getSiteDomain() + '/works/'+self.story.getMetadata('storyId'))
+        else:
+            raise exceptions.InvalidStoryURL(url,
+                                             self.getSiteDomain(),
+                                             self.getSiteExampleURLs())
         
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','ao3')
@@ -70,10 +79,11 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
         return ['www.archiveofourown.org','archiveofourown.org']
 
     def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/works/123456"
+        return "http://"+self.getSiteDomain()+"/works/123456 http://"+self.getSiteDomain()+"/collections/Some_Archive/works/123456"
 
     def getSiteURLPattern(self):
-        return re.escape("http://")+"(www\.)?"+re.escape(self.getSiteDomain()+"/works/")+r"\d+(/chapters/\d+)?/?$"
+        # http://archiveofourown.org/collections/Smallville_Slash_Archive/works/159770
+        return re.escape("http://")+"(www.)?"+re.escape(self.getSiteDomain())+r"(/collections/[^/]+)?/works/(?P<id>\d+)(/chapters/\d+)?/?$"
         
     ## Login
     def needToLoginCheck(self, data):
@@ -155,14 +165,15 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
         self.story.setMetadata('title',a.string)
 		
         # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"^/users/\w+/pseuds/\w+"))
-        if a == None: # ao3 allows for author 'Anonymous' with no author link.
+        alist = soup.findAll('a', href=re.compile(r"^/users/\w+/pseuds/\w+"))
+        if len(alist) < 1: # ao3 allows for author 'Anonymous' with no author link.
             self.story.setMetadata('author','Anonymous')
             self.story.setMetadata('authorUrl',self.url)
         else:
-            self.story.setMetadata('authorId',a['href'].split('/')[2])
-            self.story.setMetadata('authorUrl','http://'+self.host+a['href'])
-            self.story.setMetadata('author',a.text)
+            for a in alist:
+                self.story.addToList('authorId',a['href'].split('/')[2])
+                self.story.addToList('authorUrl','http://'+self.host+a['href'])
+                self.story.addToList('author',a.text)
 
         # Find the chapters:
         chapters=soup.findAll('a', href=re.compile(r'/works/'+self.story.getMetadata('storyId')+"/chapters/\d+$"))
@@ -180,7 +191,7 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
 
         a = metasoup.find('blockquote',{'class':'userstuff'})
         if a != None:
-            self.setDescription(url,a.text)
+            self.setDescription(url,a)
             #self.story.setMetadata('description',a.text)
 		
         a = metasoup.find('dd',{'class':"rating tags"})
@@ -220,9 +231,9 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
                 self.story.addToList('characters',char.string)
         a = metasoup.find('dd',{'class':"relationship tags"})
         if a != None:
-            chars = a.findAll('a',{'class':"tag"})
-            for char in chars:
-                self.story.addToList('characters',char.string)
+            ships = a.findAll('a',{'class':"tag"})
+            for ship in ships:
+                self.story.addToList('ships',ship.string)
 		
 
         stats = metasoup.find('dl',{'class':'stats'})

fanficdownloader/adapters/adapter_ashwindersycophanthexcom.py

@@ -56,11 +56,6 @@ class AshwinderSycophantHexComAdapter(BaseSiteAdapter):
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','asph')
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Harry Potter")
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%m/%d/%Y"

fanficdownloader/adapters/adapter_castlefansorg.py

@@ -80,11 +80,6 @@ class CastleFansOrgAdapter(BaseSiteAdapter): # XXX
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','cslf') # XXX
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Castle") # XXX
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%b %d, %Y" # XXX

fanficdownloader/adapters/adapter_chaossycophanthexcom.py

@@ -56,11 +56,6 @@ class ChaosSycophantHexComAdapter(BaseSiteAdapter):
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','csph')
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        #self.story.addToList("category","Harry Potter")
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%m/%d/%Y"

fanficdownloader/adapters/adapter_checkmatedcom.py

@@ -44,25 +44,20 @@ class CheckmatedComAdapter(BaseSiteAdapter):
         self.username = "" # if left empty, site doesn't return any message at all.
         self.password = ""
         self.is_adult=False
-        
+
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
         logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-        
+
         self._setURL('http://' + self.getSiteDomain() + '/story.php?story='+self.story.getMetadata('storyId'))
-        
+
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','chm')
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Harry Potter")
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%b %d, %Y"
-            
+
     @staticmethod # must be @staticmethod, don't remove it.
     def getSiteDomain():
         # The site domain.  Does have www here, if it uses it.
@@ -82,7 +77,7 @@ class CheckmatedComAdapter(BaseSiteAdapter):
             return True
         else:
             return False
-        
+
     def performLogin(self, url):
         params = {}
 
@@ -93,8 +88,8 @@ class CheckmatedComAdapter(BaseSiteAdapter):
             params['name'] = self.getConfig("username")
             params['pass'] = self.getConfig("password")
         params['login'] = 'yes'
-        params['submit'] = 'login'	
-	
+        params['submit'] = 'login'
+
         loginUrl = 'http://' + self.getSiteDomain()+'/login.php'
         d = self._fetchUrl(loginUrl,params)
         e = self._fetchUrl(url)
@@ -135,33 +130,33 @@ class CheckmatedComAdapter(BaseSiteAdapter):
         # The actual text that is used to announce you need to be an
         # adult varies from site to site.  Again, print data before
         # the title search to troubleshoot.
-            
+
         if "Access denied. This story has not been validated by the adminstrators of this site." in data:
             raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
+
         # use BeautifulSoup HTML parser to make everything easier to find.
         soup = bs.BeautifulSoup(data)
         # print data
 
         # Now go hunting for all the meta data and the chapter list.
-        
+
         ## Title
         a = soup.findAll('span', {'class' : 'storytitle'})
         self.story.setMetadata('title',a[0].string)
-        
+
         # Find authorid and URL from... author url.
         a = a[1].find('a', href=re.compile(r"authors.php\?name\=\w+"))
         self.story.setMetadata('authorId',a['href'].split('=')[1])
         self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
         self.story.setMetadata('author',a.string)
-			
+
         a = soup.find('select', {'name' : 'chapter'})
         if a == None:
             self.chapterUrls.append((self.story.getMetadata('title'),url))
         else:
             for chapter in a.findAll('option'):
                 self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/story.php?story='+self.story.getMetadata('storyId')+'&chapter='+chapter['value']))
-			
+
         self.story.setMetadata('numChapters',len(self.chapterUrls))
 
         # eFiction sites don't help us out a lot with their meta data
@@ -174,7 +169,7 @@ class CheckmatedComAdapter(BaseSiteAdapter):
             except:
                 return ""
 
-				
+
         # website does not keep track of word count, and there is no convenient way to calculate it
 
         summary = soup.find('fieldset')
@@ -182,21 +177,23 @@ class CheckmatedComAdapter(BaseSiteAdapter):
         summary.name='div'
         self.setDescription(url,summary)
 
-		
+
         # <span class="label">Rated:</span> NC-17<br /> etc
         table = soup.findAll('div', {'class' : 'text'})[1]
         for labels in table.findAll('tr'):
             value = labels.findAll('td')[1]
             label = labels.findAll('td')[0]
 
-			
+
             if 'Rating' in stripHTML(label):
                 self.story.setMetadata('rating', stripHTML(value))
 
             if 'Ship' in stripHTML(label):
+                if value.string != "none/none":
+                    self.story.addToList('ships',value.string)
                 for char in value.string.split('/'):
                     if char != 'none':
-                        self.story.addToList('characters',char)		
+                        self.story.addToList('characters',char)
 
             if 'Status' in stripHTML(label):
                 if value.find('img', {'src' : 'img/incomplete.gif'}) == None:
@@ -206,11 +203,11 @@ class CheckmatedComAdapter(BaseSiteAdapter):
 
             if 'Published' in stripHTML(label):
                 self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-            
+
             if 'Updated' in stripHTML(label):
                 self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-			
-        a = self._fetchUrl(self.story.getMetadata('authorUrl')+'&cat=stories')	
+
+        a = self._fetchUrl(self.story.getMetadata('authorUrl')+'&cat=stories')
         for story in bs.BeautifulSoup(a).findAll('table', {'class' : 'storyinfo'}):
             a = story.find('a', href=re.compile(r"review.php\?s\="+self.story.getMetadata('storyId')+'&act=view'))
             if a != None:
@@ -220,8 +217,8 @@ class CheckmatedComAdapter(BaseSiteAdapter):
                     if 'genre' in stripHTML(label):
                         for genre in value.findAll('img'):
                             self.story.addToList('genre',genre['title'])
-        
-            
+
+
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
@@ -229,11 +226,11 @@ class CheckmatedComAdapter(BaseSiteAdapter):
 
         soup = bs.BeautifulSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
+
         div = soup.find('div', {'id' : 'resizeableText'})
         div.find('div', {'class' : 'storyTools'}).extract()
 
         if None == div:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
+
         return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_darksolaceorg.py

@@ -0,0 +1,274 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return DarkSolaceOrgAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class DarkSolaceOrgAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+		
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/elysian/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','dksl')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%d %B %Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'dark-solace.org'
+        
+    @classmethod
+    def getAcceptDomains(cls):
+        return ['www.dark-solace.org','dark-solace.org']
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/elysian/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://")+"(www\.)?"+re.escape(self.getSiteDomain()+"/elysian/viewstory.php?sid=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'This story contains adult content not suitable for children' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['rememberme'] = '1'
+        params['sid'] = ''
+        params['intent'] = ''
+        params['submit'] = 'Submit'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/elysian/user.php'
+        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "User Account Page" not in d : #Member Account
+            logging.info("Failed to login to URL %s as %s, or have no authorization to access the story" % (loginUrl, params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            addurl="&ageconsent=ok"
+            self.performLogin(url)
+            data = self._fetchUrl(url+addurl)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title and author
+        a = soup.find('div', {'id' : 'pagetitle'})
+        
+        aut = a.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',aut['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/elysian/'+aut['href'])
+        self.story.setMetadata('author',aut.string)
+        aut.extract()
+        
+        self.story.setMetadata('title',a.string[:(len(a.string)-3)])
+
+        # Find the chapters:
+        chapters=soup.find('select', {'name' : 'chapter'})
+        if chapters != None:
+            for chapter in chapters.findAll('option'):
+                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/elysian/viewstory.php?sid='+self.story.getMetadata('storyId')+'&chapter='+chapter['value']))
+        else:
+            self.chapterUrls.append((self.story.getMetadata('title'),url))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+        
+        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
+        
+        for list in asoup.findAll('div', {'class' : re.compile('listbox\s+')}):
+            a = list.find('a', href=re.compile(r'viewstory.php\?sid='))
+            if a != None:
+                if 'viewstory.php?sid='+self.story.getMetadata('storyId') in a['href']:
+                    break
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+                
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = list.findAll('span', {'class' : 'classification'})
+        for labelspan in labels:
+            label = labelspan.text
+            value = labelspan.nextSibling
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not (defaultGetattr(value,'class') == 'classification' or "Chapters: " in stripHTML(value)):
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value[:len(value)-2])
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'categories.php\?catid=\d+'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                for char in value.string.split(', '):
+                    if not 'None' in char:
+                        self.story.addToList('characters',char)
+
+            if 'Genre' in label:
+                for genre in value.string.split(', '):
+                    if not 'None' in genre:
+                        self.story.addToList('genre',genre)
+
+            if 'Warnings' in label:
+                for warning in value.string.split(', '):
+                    if not 'None' in warning:
+                        self.story.addToList('warnings',warning)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = list.find('a', href=re.compile(r"series.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/elysian/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            i=1
+            for a in storyas:
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if ('viewstory.php?sid='+self.story.getMetadata('storyId')) in a['href']:
+                        self.setSeries(series_name, i)
+                        break
+                    i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_dokugacom.py

@@ -56,11 +56,6 @@ class DokugaComAdapter(BaseSiteAdapter):
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','dkg')
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","InuYasha")
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         if 'fanfiction' in self.story.getMetadata('section'):

fanficdownloader/adapters/adapter_dracoandginnycom.py

@@ -0,0 +1,298 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return DracoAndGinnyComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class DracoAndGinnyComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','dcagn')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%b %d, %Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.dracoandginny.com'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Registered Users Only' in data \
+                or 'There is no such account on our website' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['cookiecheck'] = '1'
+        params['submit'] = 'Submit'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
+        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "Member Account" not in d : #Member Account
+            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=2"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid & error in url.
+                addurl = addurl.replace("&","&")
+                url = self.url+'&index=1'+addurl
+                logging.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        content=soup.find('div',{'class' : 'listbox'})
+        
+        self.setDescription(url,content.find('blockquote'))
+        
+        for genre in content.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')):
+            self.story.addToList('genre',genre.string)
+
+        for warning in content.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')):
+            self.story.addToList('warnings',warning.string)
+        
+        labels = content.findAll('b')
+        
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value.split(' |')[0])
+                
+            if 'Rating' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value.nextSibling).split(' |')[0], self.dateformat))
+            
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            i=1
+            for a in storyas:
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        break
+                    i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'class' : 'listbox'})
+        
+        if None == div:
+            div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_dramioneorg.py

@@ -55,14 +55,9 @@ class DramioneOrgAdapter(BaseSiteAdapter):
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','drmn')
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Harry Potter")
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%dth %B %Y"
+        self.dateformat = "%d %B %Y"
             
     @staticmethod # must be @staticmethod, don't remove it.
     def getSiteDomain():
@@ -228,11 +223,11 @@ class DramioneOrgAdapter(BaseSiteAdapter):
                     self.story.setMetadata('status', 'In-Progress')
 
             if 'Published' in label:
-                value=value.replace('st','th').replace('nd','th').replace('rd','th')
+                value=re.sub(r"(\d+)(st|nd|rd|th)",r"\1",value)
                 self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
             
             if 'Updated' in label:
-                value=value.replace('st','th').replace('nd','th').replace('rd','th')
+                value=re.sub(r"(\d+)(st|nd|rd|th)",r"\1",value)
                 self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
 
         try:

fanficdownloader/adapters/adapter_erosnsapphosycophanthexcom.py

@@ -56,11 +56,6 @@ class ErosnSapphoSycophantHexComAdapter(BaseSiteAdapter):
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','essph')
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Harry Potter")
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%d/%m/%y"

fanficdownloader/adapters/adapter_fanfictionnet.py

@@ -88,7 +88,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
             else:
                 raise e
             
-        if "Unable to locate story with id of " in data:
+        if "Unable to locate story" in data:
             raise exceptions.StoryDoesNotExist(url)
 
         # some times "Chapter not found...", sometimes "Chapter text not found..."
@@ -176,7 +176,9 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         genrelist = metalist[0].split('/') # Hurt/Comfort already changed above.
         goodgenres=True
         for g in genrelist:
-            if g not in ffnetgenres:
+            print("g:(%s)"%g)
+            if g.strip() not in ffnetgenres:
+                print("g not in ffnetgenres")
                 goodgenres=False
         if goodgenres:
             self.story.extendList('genre',genrelist)
@@ -184,7 +186,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
 
         donechars = False
         while len(metalist) > 0:
-            if  metalist[0].startswith('Reviews') or metalist[0].startswith('Chapters') or metalist[0].startswith('Status') or metalist[0].startswith('id:'):
+            if  metalist[0].startswith('Reviews') or metalist[0].startswith('Chapters') or metalist[0].startswith('Status') or metalist[0].startswith('id:') or metalist[0].startswith('Favs:') or metalist[0].startswith('Follows:'):
                 pass
             elif  metalist[0].startswith('Updated'):
                 self.story.setMetadata('dateUpdated',makeDate(metalist[0].split(':')[1].strip(), '%m-%d-%y'))
@@ -239,6 +241,15 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
                         ## additional to what ever the
                         ## slow_down_sleep_time setting is.
         data = self._fetchUrl(url)
+
+        # some ancient stories have body tags inside them that cause
+        # soup parsing to discard the content.  For story text we
+        # don't care about anything before "<div class='storytextp"
+        # (there's a space after storytextp, so no close quote(')) and
+        # this kills any body tags.
+        data = data[data.index("<div class='storytextp"):]
+        data.replace("<body","<notbody").replace("<BODY","<NOTBODY")
+        
         soup = bs.BeautifulSoup(data)
 
         ## Remove the 'share' button.

fanficdownloader/adapters/adapter_fanfiktionde.py

@@ -161,9 +161,9 @@ class FanFiktionDeAdapter(BaseSiteAdapter):
             self.story.addToList('genre',genre)
             
         if 'fertiggestellt' in head.text:
-            self.story.setMetadata('status', 'Fertiggestellt')
+            self.story.setMetadata('status', 'Completed')
         else:
-            self.story.setMetadata('status', 'In Arbeit')
+            self.story.setMetadata('status', 'In Progress')
 
         #find metadata on the author's page
         asoup = bs.BeautifulSoup(self._fetchUrl("http://"+self.getSiteDomain()+"?a=q&a1=v&t=nickdetailsstories&lbi=stories&ar=0&nick="+self.story.getMetadata('authorId')))

fanficdownloader/adapters/adapter_ficbooknet.py

@@ -201,7 +201,7 @@ class FicBookNetAdapter(BaseSiteAdapter):
                 break
 				
         summary=soup.find('span', {'class' : 'urlize'})
-        self.setDescription(url,summary.text)
+        self.setDescription(url,summary)
         #self.story.setMetadata('description', summary.text)
             
     # grab the text for an individual chapter.

fanficdownloader/adapters/adapter_fictionalleyorg.py

@@ -37,7 +37,6 @@ class FictionAlleyOrgSiteAdapter(BaseSiteAdapter):
                                # Most sites that claim to be
                                # iso-8859-1 (and some that claim to be
                                # utf8) are really windows-1252.
-        self.story.addToList("category","Harry Potter")
         self.is_adult=False
         
         # get storyId from url--url validation guarantees query correct

fanficdownloader/adapters/adapter_ficwadcom.py

@@ -124,7 +124,7 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
 
         # description
         storydiv = soup.find("div",{"id":"story"})
-        self.setDescription(url,storydiv.find("blockquote",{'class':'summary'}).p.string)
+        self.setDescription(url,storydiv.find("blockquote",{'class':'summary'}).p)
         #self.story.setMetadata('description', storydiv.find("blockquote",{'class':'summary'}).p.string)
 
         # most of the meta data is here:

fanficdownloader/adapters/adapter_fimfictionnet.py

@@ -27,6 +27,8 @@ from .. import BeautifulSoup as bs
 from ..htmlcleanup import stripHTML
 from .. import exceptions as exceptions
 
+from ..bbcodeutils.bbcodeparser import bbcodeparser
+
 from base_adapter import BaseSiteAdapter,  makeDate
 
 def getClass():
@@ -133,19 +135,23 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
             if storyMetadata["categories"][category]:
                 self.story.addToList("genre", category) 
 
-        self.story.addToList("category", "My Little Pony")
-        
         self.story.setMetadata("numWords", str(storyMetadata["words"]))
         
-        
         # fimfic is the first site with an explicit cover image.
         if self.getConfig('include_images') and "image" in storyMetadata.keys():
-            coverurl = storyMetadata["image"]
+            if "full_image" in storyMetadata:
+                coverurl = storyMetadata["full_image"]
+            else:
+                coverurl = storyMetadata["image"]
             if coverurl.startswith('//static.fimfiction.net'): # fix for img urls missing 'http:'
                 coverurl = "http:"+coverurl
             self.story.addImgUrl(self,self.url,coverurl,self._fetchUrlRaw,cover=True)
-            
-        self.setDescription(self.url, storyMetadata["description"])
+
+
+        # the fimfic API gives bbcode for desc, not html.
+        # btw, bbcode honors newlines, html doesn't.  change newlines to br tags.
+        self.setDescription(self.url,
+                            bbcodeparser().parse(storyMetadata["description"]).html(doDeepCopy=False).replace('\r','').replace('\n','<br />'))
         
         # Dates are in Unix time
         # Take the publish date from the first chapter posted
@@ -155,8 +161,18 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
         self.story.setMetadata("dateUpdated", datetime.fromtimestamp(rawDateUpdated))
         
         soup = bs.BeautifulSoup(data).find("div", {"class":"story"})
-        for character in [character_icon["title"] for character_icon in soup.findAll("a", {"class":"character_icon"})]:
-            self.story.addToList("characters", character)
+        # fimfic stopped putting the char name on or around the char
+        # icon now for some reason.  Pull it from the image name with
+        # some heuristics.
+        for character in [character_icon["src"] for character_icon in soup.findAll("img", {"class":"character_icon"})]:
+            # //static.fimfiction.net/images/characters/twilight_sparkle.png
+            # 5th split /, remove last four, replace _, capitolize every word(title())
+            char = character.split('/')[5][:-4].replace('_',' ').title()
+            if char == 'Oc':
+                char = "OC"
+            if char == 'Cmc':
+                char = "Cutie Mark Crusaders"
+            self.story.addToList("characters", char)
             
             
     def getChapterText(self, url):

fanficdownloader/adapters/adapter_finestoriescom.py

@@ -0,0 +1,293 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return FineStoriesComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class FineStoriesComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url
+        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2].split(':')[0])
+        if 'storyInfo' in self.story.getMetadata('storyId'):
+            self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/s/storyInfo.php?id='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','fnst')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%Y-%m-%d"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'finestories.com'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/s/10537 http://"+self.getSiteDomain()+"/s/10537:4010 http://"+self.getSiteDomain()+"/library/storyInfo.php?id=10537"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain())+r"/(s|library)?/(storyInfo.php\?id=)?\d+(:\d+)?(;\d+)?$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Free Registration' in data \
+                or "Invalid Password!" in data \
+                or "Invalid User Name!" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['theusername'] = self.username
+            params['thepassword'] = self.password
+        else:
+            params['theusername'] = self.getConfig("username")
+            params['thepassword'] = self.getConfig("password")
+        params['rememberMe'] = '1'
+        params['page'] = 'http://finestories.com/'
+        params['submit'] = 'Login'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/login.php'
+        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['theusername']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "My Account" not in d : #Member Account
+            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['theusername']))
+            raise exceptions.FailedToLogin(url,params['theusername'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'/s/'+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.text)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"/a/\w+"))
+        self.story.setMetadata('authorId',a['href'].split('/')[2])
+        self.story.setMetadata('authorUrl','http://'+self.host+a['href'])
+        self.story.setMetadata('author',a.text)
+
+        # Find the chapters:
+        chapters = soup.findAll('a', href=re.compile(r'/s/'+self.story.getMetadata('storyId')+":\d+$"))
+        if len(chapters) != 0:
+            for chapter in chapters:
+                # just in case there's tags, like <i> in chapter titles.
+                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+chapter['href']))
+        else:
+            self.chapterUrls.append((self.story.getMetadata('title'),'http://'+self.host+'/s/'+self.story.getMetadata('storyId')))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # surprisingly, the detailed page does not give enough details, so go to author's page
+        
+        skip=0
+        i=0
+        while i == 0:
+            asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')+"&skip="+str(skip)))
+        
+            a = asoup.findAll('td', {'class' : 'lc2'})
+            for lc2 in a:
+                if lc2.find('a')['href'] == '/s/'+self.story.getMetadata('storyId'):
+                    i=1
+                    break
+                if a[len(a)-1] == lc2:
+                    skip=skip+10
+        
+        for cat in lc2.findAll('div', {'class' : 'typediv'}):
+            self.story.addToList('category',cat.text)
+        
+        self.story.setMetadata('numWords', lc2.findNext('td', {'class' : 'num'}).text)
+        
+        lc4 = lc2.findNext('td', {'class' : 'lc4'})
+        
+        
+        try:
+            a = lc4.find('a', href=re.compile(r"/library/show_series.php\?id=\d+"))
+            i = a.parent.text.split('(')[1].split(')')[0]
+            self.setSeries(a.text, i)
+        except:
+            pass
+        try:
+            a = lc4.find('a', href=re.compile(r"/library/universe.php\?id=\d+"))
+            self.story.addToList("category",a.text)
+        except:
+            pass
+            
+        for a in lc4.findAll('span', {'class' : 'help'}):
+            a.extract()
+            
+        self.setDescription('http://'+self.host+'/s/'+self.story.getMetadata('storyId'),lc4.text.split('[More Info')[0])
+            
+        for b in lc4.findAll('b'):
+            label = b.text
+            value = b.nextSibling
+            
+            if 'For Age' in label:
+                self.story.setMetadata('rating', value)
+                
+            if 'Tags' in label:
+                for genre in value.split(', '):
+                    self.story.addToList('genre',genre)
+                    
+            if 'Posted' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value.split('/ (')[0]), self.dateformat))
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value.split('/ (')[0]), self.dateformat))
+                
+            if 'Concluded' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value.split('/ (')[0]), self.dateformat))
+                
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value.split('/ (')[0]), self.dateformat))
+                
+        status = lc4.find('span', {'class' : 'ab'})
+        if  status != None:
+            self.story.setMetadata('status', 'In-Progress')
+            if "Last Activity" in status.text:
+                self.story.setMetadata('dateUpdated', makeDate(status.text.split('Activity: ')[1].split(')')[0], self.dateformat))
+        else:
+            self.story.setMetadata('status', 'Completed')
+
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+        
+        # some big chapters are split over several pages
+        pager = div.find('span', {'class' : 'pager'})
+        if pager != None:
+            urls=pager.findAll('a')
+            urls=urls[:len(urls)-1]
+            
+            
+            for ur in urls:
+                soup = bs.BeautifulSoup(self._fetchUrl("http://"+self.getSiteDomain()+ur['href']),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+                div1 = soup.find('div', {'id' : 'story'})
+                
+                # appending next section
+                last=div.findAll('p')
+                next=div1.find('span', {'class' : 'conTag'}).nextSibling
+            
+                last[len(last)-1]=last[len(last)-1].append(next)
+                div.append(div1)
+            
+            
+            
+        # removing all the left-over stuff    
+        for a in div.findAll('span'):
+            a.extract()
+            
+        for a in div.findAll('h1'):
+            a.extract()
+        for a in div.findAll('h2'):
+            a.extract()
+        for a in div.findAll('h3'):
+            a.extract()
+        for a in div.findAll('h4'):
+            a.extract()
+        for a in div.findAll('br'):
+            a.extract()
+        for a in div.findAll('div', {'class' : 'date'}):
+            a.extract()
+            
+        a = div.find('form')
+        if a != None:
+            b = a.nextSibling
+            while b != None:
+                a.extract()
+                a=b
+                b=b.nextSibling
+        
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_grangerenchantedcom.py

@@ -0,0 +1,297 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return GrangerEnchantedCom
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class GrangerEnchantedCom(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        self.story.setMetadata('section',self.parsedUrl.path.split('/',)[1])
+             
+        # normalized story URL.
+        if "malfoymanor" in self.parsedUrl.netloc:
+            self._setURL('http://malfoymanor.' + self.getSiteDomain() + '/themanor/viewstory.php?sid='+self.story.getMetadata('storyId'))
+            self.story.addToList("category","The Manor")
+        else:
+            self._setURL('http://' + self.getSiteDomain() + '/enchant/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','gech')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%d/%b/%Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'grangerenchanted.com'
+        
+    @classmethod
+    def getAcceptDomains(cls):
+        return ['grangerenchanted.com','malfoymanor.grangerenchanted.com']
+
+    def getSiteExampleURLs(self):
+        return "http://grangerenchanted.com/enchant/viewstory.php?sid=1234 http://malfoymanor.grangerenchanted.com/themanor/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return r"http://(malfoymanor.)?grangerenchanted.com/(enchant|themanor)?/viewstory.php\?sid=\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Registered Users Only' in data \
+                or 'There is no such account on our website' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['cookiecheck'] = '1'
+        params['submit'] = 'Submit'
+    
+        if "enchant" in self.story.getMetadata('section'):
+            loginUrl = 'http://grangerenchanted.com/enchant/user.php?action=login'
+        else:
+            loginUrl = 'http://malfoymanor.grangerenchanted.com/themanor/user.php?action=login'
+        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "Member Account" not in d : #Member Account
+            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=1"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid & error in url.
+                addurl = addurl.replace("&","&")
+                url = self.url+'&index=1'+addurl
+                logging.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+self.story.getMetadata('section')+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=4'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+self.story.getMetadata('section')+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            i=1
+            for a in storyas:
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        break
+                    i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story1'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_harrypotterfanfictioncom.py

@@ -37,7 +37,6 @@ class HarryPotterFanFictionComSiteAdapter(BaseSiteAdapter):
                                # Most sites that claim to be
                                # iso-8859-1 (and some that claim to be
                                # utf8) are really windows-1252.
-        self.story.addToList("category","Harry Potter")
         self.is_adult=False
         
         # get storyId from url--url validation guarantees query is only psid=1234

fanficdownloader/adapters/adapter_hlfictionnet.py

@@ -0,0 +1,228 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return HLFictionNetAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class HLFictionNetAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','hlf')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%m/%d/%y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'hlfiction.net'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title and author
+        a = soup.find('div', {'id' : 'pagetitle'})
+        
+        aut = a.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',aut['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+aut['href'])
+        self.story.setMetadata('author',aut.string)
+        aut.extract()
+        
+        self.story.setMetadata('title',a.string[:(len(a.string)-3)])
+
+        # Find the chapters:
+        chapters=soup.find('select')
+        if chapters != None:
+            for chapter in chapters.findAll('option'):
+                # just in case there's tags, like <i> in chapter titles. 
+                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/viewstory.php?sid='+self.story.getMetadata('storyId')+'&chapter='+chapter['value']))
+        else:
+            self.chapterUrls.append((self.story.getMetadata('title'),url))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+        
+        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
+        
+        for list in asoup.findAll('div', {'class' : re.compile('listbox\s+')}):
+            a = list.find('a')
+            if ('viewstory.php?sid='+self.story.getMetadata('storyId')) in a['href']:
+                break
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = list.findAll('span', {'class' : 'classification'})
+        for labelspan in labels:
+            label = labelspan.string
+            value = labelspan.nextSibling
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'classification':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value[:len(value)-2])
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'categories.php\?catid=\d+'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                for char in value.string.split(', '):
+                    if not 'None' in char:
+                        self.story.addToList('characters',char)
+
+            if 'Genre' in label:
+                for genre in value.string.split(', '):
+                    if not 'None' in genre:
+                        self.story.addToList('genre',genre)
+
+            if 'Warnings' in label:
+                for warning in value.string.split(', '):
+                    if not 'None' in warning:
+                        self.story.addToList('warnings',warning)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = list.find('a', href=re.compile(r"series.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            i=1
+            for a in storyas:
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if ('viewstory.php?sid='+self.story.getMetadata('storyId')) in a['href']:
+                        self.setSeries(series_name, i)
+                        break
+                    i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_hpfandomnet.py

@@ -60,11 +60,6 @@ class HPFandomNetAdapterAdapter(BaseSiteAdapter): # XXX
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','hpfdm') # XXX
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Harry Potter") # XXX
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%Y.%m.%d" # XXX

fanficdownloader/adapters/adapter_hpfanficarchivecom.py

@@ -0,0 +1,216 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return HPFanficArchiveComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class HPFanficArchiveComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/stories/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','hpffa')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%B %d, %Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.hpfanficarchive.com'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/stories/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/stories/viewstory.php?sid=")+r"\d+$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/stories/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/stories/'+chapter['href']))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) # XXX
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/stories/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            i=1
+            for a in storyas:
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        break
+                    i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+        
+
+        soup = bs.BeautifulSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_iketernalnet.py

@@ -55,11 +55,6 @@ class IkEternalNetAdapter(BaseSiteAdapter):
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','ike')
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","InuYasha")
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%B %d, %Y"

fanficdownloader/adapters/adapter_ksarchivecom.py

@@ -60,11 +60,6 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','ksa') # XXX
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Star Trek") # XXX
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%b/%d/%Y" # XXX
@@ -167,7 +162,7 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
         self.story.setMetadata('authorId',a['href'].split('=')[1])
         self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
+        self.story.setMetadata('author',stripHTML(a))
 
         # Find the chapters:
         for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
@@ -190,14 +185,20 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
         labels = soup.findAll('span',{'class':'label'})
         for labelspan in labels:
             value = labelspan.nextSibling
-            label = labelspan.string
+            label = stripHTML(labelspan)
 
             if 'Summary' in label:
                 ## Everything until the next span class='label'
                 svalue = ""
                 while not defaultGetattr(value,'class') == 'label':
                     svalue += str(value)
-                    value = value.nextSibling
+                    # poor HTML(unclosed <p> for one) can cause run on
+                    # over the next label.
+                    if '<span class="label">' in svalue:
+                        svalue = svalue[0:svalue.find('<span class="label">')]
+                        break
+                    else:
+                        value = value.nextSibling
                 self.setDescription(url,svalue)
                 #self.story.setMetadata('description',stripHTML(svalue))
 
@@ -209,48 +210,50 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
 
             if 'Categories' in label:
                 cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-                catstext = [cat.string for cat in cats]
+                catstext = [stripHTML(cat) for cat in cats]
                 for cat in catstext:
-                    if cat.string.strip() in ('Poetry','Essays'):
-                        self.story.addToList('category',cat.string)
+                    # ran across one story with an empty <a href="browse.php?type=categories&amp;catid=1"></a>
+                    # tag in the desc once.
+                    if cat and cat.strip() in ('Poetry','Essays'):
+                        self.story.addToList('category',stripHTML(cat))
 
             if 'Characters' in label:
                 self.story.addToList('characters','Kirk')
                 self.story.addToList('characters','Spock')
                 chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-                charstext = [char.string for char in chars]
+                charstext = [stripHTML(char) for char in chars]
                 for char in charstext:
-                    self.story.addToList('characters',char.string)
+                    self.story.addToList('characters',stripHTML(char))
 
             ## Not all sites use Genre, but there's no harm to
             ## leaving it in.  Check to make sure the type_id number
             ## is correct, though--it's site specific.
             if 'Genre' in label:
                 genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) # XXX
-                genrestext = [genre.string for genre in genres]
+                genrestext = [stripHTML(genre) for genre in genres]
                 self.genre = ', '.join(genrestext)
                 for genre in genrestext:
-                    self.story.addToList('genre',genre.string)
+                    self.story.addToList('genre',stripHTML(genre))
 
             ## In addition to Genre (which is very site specific) KSA
             ## has 'Story Type', which is much more what most sites
             ## call genre.
             if 'Story Type' in label:
                 genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=5')) # XXX
-                genrestext = [genre.string for genre in genres]
+                genrestext = [stripHTML(genre) for genre in genres]
                 self.genre = ', '.join(genrestext)
                 for genre in genrestext:
-                    self.story.addToList('genre',genre.string)
+                    self.story.addToList('genre',stripHTML(genre))
 
             ## Not all sites use Warnings, but there's no harm to
             ## leaving it in.  Check to make sure the type_id number
             ## is correct, though--it's site specific.
             if 'Warnings' in label:
                 warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
-                warningstext = [warning.string for warning in warnings]
+                warningstext = [stripHTML(warning) for warning in warnings]
                 self.warning = ', '.join(warningstext)
                 for warning in warningstext:
-                    self.story.addToList('warnings',warning.string)
+                    self.story.addToList('warnings',stripHTML(warning))
 
             if 'Completed' in label:
                 if 'Yes' in value:
@@ -269,7 +272,7 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
         try:
             # Find Series name from series URL.
             a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
-            series_name = a.string
+            series_name = stripHTML(a)
             series_url = 'http://'+self.host+'/'+a['href']
 
             # use BeautifulSoup HTML parser to make everything easier to find.

fanficdownloader/adapters/adapter_libraryofmoriacom.py

@@ -56,11 +56,6 @@ class LibraryOfMoriaComAdapter(BaseSiteAdapter):
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','lom')
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Lord of the Rings")
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%B %d, %Y"

fanficdownloader/adapters/adapter_lumossycophanthexcom.py

@@ -56,11 +56,6 @@ class LumosSycophantHexComAdapter(BaseSiteAdapter):
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','lsph')
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Harry Potter")
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%m/%d/%Y"

fanficdownloader/adapters/adapter_merlinficdtwinscouk.py

@@ -0,0 +1,290 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return MerlinFicDtwinsCoUk
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class MerlinFicDtwinsCoUk(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','mrfd')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%b %d, %Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'merlinfic.dtwins.co.uk'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Registered Users Only' in data \
+                or 'There is no such account on our website' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['cookiecheck'] = '1'
+        params['submit'] = 'Submit'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
+        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "Member Account" not in d : #Member Account
+            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=4"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid & error in url.
+                addurl = addurl.replace("&","&")
+                url = self.url+'&index=1'+addurl
+                logging.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+                    
+            if 'Pairing' in label:
+                ships = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+                for ship in ships:
+                    self.story.addToList('ships',ship.string)
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=3'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            i=1
+            for a in storyas:
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        break
+                    i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_midnightwhispersca.py

@@ -60,11 +60,6 @@ class MidnightwhispersCaAdapter(BaseSiteAdapter): # XXX
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','mw') # XXX
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Queer as Folk") # XXX
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%B %d, %Y" # XXX

fanficdownloader/adapters/adapter_mugglenetcom.py

@@ -59,11 +59,6 @@ class MuggleNetComAdapter(BaseSiteAdapter): # XXX
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','mgln') # XXX
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Harry Potter") # XXX
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%m/%d/%y" # XXX

fanficdownloader/adapters/adapter_nationallibrarynet.py

@@ -55,11 +55,6 @@ class NationalLibraryNetAdapter(BaseSiteAdapter):
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','ntlb')
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","West Wing")
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%m-%d-%y"
@@ -154,6 +149,10 @@ class NationalLibraryNetAdapter(BaseSiteAdapter):
                 for char in value.string.split(', '):
                     self.story.addToList('characters',char)
                     
+            if 'Pairing' in label:
+                for char in value.string.split(', '):
+                    self.story.addToList('ships',char)
+                    
             if 'Warnings' in label:
                 for warning in value.string.split(', '):
                     self.story.addToList('warnings',warning)

fanficdownloader/adapters/adapter_ncisficcom.py

@@ -55,11 +55,6 @@ class NCISFicComAdapter(BaseSiteAdapter):
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','ncisf')
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","NCIS")
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%m-%d-%y"
@@ -154,6 +149,10 @@ class NCISFicComAdapter(BaseSiteAdapter):
                 for char in value.string.split(', '):
                     self.story.addToList('characters',char)
                     
+            if 'Pairing' in label:
+                for char in value.string.split(', '):
+                    self.story.addToList('ships',char)
+                    
             if 'Warnings' in label:
                 for warning in value.string.split(', '):
                     self.story.addToList('warnings',warning)

fanficdownloader/adapters/adapter_ncisfictioncom.py

@@ -56,11 +56,6 @@ class NCISFictionComAdapter(BaseSiteAdapter):
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','ncisfn')
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","NCIS")
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%d/%m/%Y"
@@ -157,6 +152,11 @@ class NCISFictionComAdapter(BaseSiteAdapter):
                 for char in chars:
                     self.story.addToList('characters',char.string)
 
+            if 'Pairing' in label:
+                ships = value.findAll('a')
+                for ship in ships:
+                    self.story.addToList('ships',ship.string)
+
             if 'Genre' in label:
                 genres = value.findAll('a')
                 for genre in genres:

fanficdownloader/adapters/adapter_nfacommunitycom.py

@@ -60,11 +60,6 @@ class NfaCommunityComAdapter(BaseSiteAdapter): # XXX
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','nfa') # XXX
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","NCIS") # XXX
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%m/%d/%Y" # XXX

fanficdownloader/adapters/adapter_nhamagicalworldsus.py

@@ -0,0 +1,211 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return NHAMagicalWorldsUsAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class NHAMagicalWorldsUsAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+		
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','nha')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = " %m/%d/%y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'nha.magical-worlds.us'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+        url = self.url
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+            
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
+		
+        try:
+            # in case link points somewhere other than the first chapter
+            a = soup.findAll('option')[1]['value']
+            self.story.setMetadata('storyId',a.split('=',)[1])
+            url = 'http://'+self.host+'/'+a
+            soup = bs.BeautifulSoup(self._fetchUrl(url))
+        except:
+            pass
+		
+        for info in asoup.findAll('table', {'width' : '100%', 'bordercolor' : re.compile(r'#')}):
+            a = info.find('a')
+            if ('viewstory.php?sid='+self.story.getMetadata('storyId')) in a['href']:
+                self.story.setMetadata('title',a.string)
+                break
+		
+
+        # Find the chapters:
+        chapters=soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+'&chapter=\d+$'))
+        if len(chapters) == 0:
+            self.chapterUrls.append((self.story.getMetadata('title'),url))
+        else:
+            for chapter in chapters:
+                # just in case there's tags, like <i> in chapter titles.
+                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d):
+            try:
+                return d.name
+            except:
+                return ""
+				
+        cats = info.findAll('a',href=re.compile('categories.php'))
+        for cat in cats:
+            self.story.addToList('category',cat.string)
+					
+        a = info.find('a', href=re.compile(r'reviews.php\?sid='+self.story.getMetadata('storyId')))
+        val = a.nextSibling
+        svalue = ""
+        while not defaultGetattr(val) == 'br':
+            val = val.nextSibling
+        val = val.nextSibling
+        while not defaultGetattr(val) == 'br':
+            svalue += str(val)
+            val = val.nextSibling
+        self.setDescription(url,svalue)
+        
+        #does not provide convenient way to get word count
+        labels = info.findAll('i')
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = stripHTML(labelspan)
+
+            if 'Rating' in label:
+                self.story.setMetadata('rating', value.split(' -')[0])
+
+            if 'Genres' in label:
+                genres = value.string.split(', ')
+                for genre in genres:
+                    if 'None' not in genre:
+                        self.story.addToList('genre',genre.split(' -')[0])
+            
+            if 'Characters' in label:
+                chars = value.string.split(', ')
+                for char in chars:
+                    if 'None' not in char:
+                        self.story.addToList('characters',char.split(' -')[0])
+
+            if 'Warnings' in label:
+                warnings = value.string.split(', ')
+                for warning in warnings:
+                    if 'None' not in warning:
+                        self.story.addToList('warnings',warning.split(' -')[0])
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(value.split(' -')[0], self.dateformat))
+            
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(value.split(' -')[0], self.dateformat))
+
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+		
+        data = self._fetchUrl(url)
+
+        soup = bs.BeautifulSoup(data, selfClosingTags=('br','hr','span','center')) # some chapters seem to be hanging up on those tags, so it is safer to close them
+        
+        story = soup.find('div', {"id" : "story"})
+
+        if None == story:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,story)

fanficdownloader/adapters/adapter_occlumencysycophanthexcom.py

@@ -56,11 +56,6 @@ class OcclumencySycophantHexComAdapter(BaseSiteAdapter):
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','osph')
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Harry Potter")
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%m/%d/%Y"

fanficdownloader/adapters/adapter_onedirectionfanfictioncom.py

@@ -55,11 +55,6 @@ class OneDirectionFanfictionComAdapter(BaseSiteAdapter):
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','odf')
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","One Direction")
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%m/%d/%y"

fanficdownloader/adapters/adapter_phoenixsongnet.py

@@ -55,11 +55,6 @@ class PhoenixSongNetAdapter(BaseSiteAdapter):
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','phs')
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Harry Potter")
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%B %d %Y"

fanficdownloader/adapters/adapter_ponyfictionarchivenet.py

@@ -60,11 +60,6 @@ class PonyFictionArchiveNetAdapter(BaseSiteAdapter):
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','pffa')
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","My Little Pony")
-            
     @staticmethod # must be @staticmethod, don't remove it.
     def getSiteDomain():
         # The site domain.  Does have www here, if it uses it.

fanficdownloader/adapters/adapter_portkeyorg.py

@@ -62,11 +62,6 @@ class PortkeyOrgAdapter(BaseSiteAdapter): # XXX
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','prtky') # XXX
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Harry Potter") # XXX
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%d/%m/%y" # XXX

fanficdownloader/adapters/adapter_potionsandsnitchesnet.py

@@ -37,7 +37,6 @@ class PotionsAndSnitchesNetSiteAdapter(BaseSiteAdapter):
                                # Most sites that claim to be
                                # iso-8859-1 (and some that claim to be
                                # utf8) are really windows-1252.
-        self.story.addToList("category","Harry Potter")
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
@@ -142,7 +141,7 @@ class PotionsAndSnitchesNetSiteAdapter(BaseSiteAdapter):
                 chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
                 charstext = [char.string for char in chars]
                 for char in charstext:
-                    if char == "!Snape and Harry (required)":
+                    if "Snape and Harry (required)" in char:
                         self.story.addToList('characters',"Snape")
                         self.story.addToList('characters',"Harry")
                     else:

fanficdownloader/adapters/adapter_pretendercentrecom.py

@@ -0,0 +1,250 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return PretenderCenterComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class PretenderCenterComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/missingpieces/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','ptdc')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%d/%m/%y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'pretendercentre.com'
+        
+    @classmethod
+    def getAcceptDomains(cls):
+        return ['www.pretendercentre.com','pretendercentre.com']
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/missingpieces/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://")+"(www\.)?"+re.escape(self.getSiteDomain()+"/missingpieces/viewstory.php?sid=")+r"\d+$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=4"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid & error in url.
+                addurl = addurl.replace("&","&")
+                url = self.url+'&index=1'+addurl
+                logging.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.text)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/missingpieces/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/missingpieces/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+                
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) # XXX
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/missingpieces/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            i=1
+            for a in storyas:
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        break
+                    i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.findAll('div', {'id' : 'story'})[1]
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_prisonbreakficnet.py

@@ -55,11 +55,6 @@ class PrisonBreakFicNetAdapter(BaseSiteAdapter):
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','pbf')
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Prison Break")
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%B %d, %Y"
@@ -157,6 +152,11 @@ class PrisonBreakFicNetAdapter(BaseSiteAdapter):
                 for char in chars:
                     self.story.addToList('characters',char.string)
 
+            if 'Pairing' in label:
+                ships = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=3')) # XXX
+                for ship in ships:
+                    self.story.addToList('ships',ship.string.replace(" and ","/"))
+
             if 'Genre' in label:
                 genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) # XXX
                 for genre in genres:

fanficdownloader/adapters/adapter_samdeanarchivenu.py

@@ -55,11 +55,6 @@ class SamDeanArchiveNuAdapter(BaseSiteAdapter):
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','sda')
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Supernatural")
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%m/%d/%y"

fanficdownloader/adapters/adapter_scarvesandcoffeenet.py

@@ -0,0 +1,244 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return ScarvesAndCoffeeNetAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class ScarvesAndCoffeeNetAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','scacf')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%m/%d/%Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.scarvesandcoffee.net'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Registered Users Only' in data \
+                or 'There is no such account on our website' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=20"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid & error in url.
+                addurl = addurl.replace("&","&")
+                url = self.url+'&index=1'+addurl
+                logging.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('genre',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            i=1
+            for a in storyas:
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        break
+                    i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_sg1heliopoliscom.py

@@ -58,7 +58,7 @@ class SG1HeliopolisComAdapter(BaseSiteAdapter):
 
         # If all stories from the site fall into the same category,
         # the site itself isn't likely to label them as such, so we
-        # do.
+        # do.  Can't use extracategories, could be Atlantis or SG-1
         if 'atlantis' in self.story.getMetadata('section'):
             self.story.addToList("category","Stargate: Atlantis")
         else:

fanficdownloader/adapters/adapter_sinfuldesireorg.py

@@ -0,0 +1,248 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return SinfulDesireOrgAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class SinfulDesireOrgAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/archive/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','snds')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%m/%d/%Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.sinful-desire.org'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/archive/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/archive/viewstory.php?sid=")+r"\d+$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&warning=5"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid & error in url.
+                addurl = addurl.replace("&","&")
+                url = self.url+'&index=1'+addurl
+                logging.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/archive/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/archive/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=3'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/archive/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            i=1
+            for a in storyas:
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        break
+                    i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_siyecouk.py

@@ -56,11 +56,6 @@ class SiyeCoUkAdapter(BaseSiteAdapter): # XXX
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','siye') # XXX
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Harry Potter") # XXX
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%Y.%m.%d" # XXX

fanficdownloader/adapters/adapter_stargateatlantisorg.py

@@ -55,11 +55,6 @@ class StargateAtlantisOrgAdapter(BaseSiteAdapter):
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','stat')
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Stargate: Atlantis")
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%B %d %Y"

fanficdownloader/adapters/adapter_storiesofardacom.py

@@ -55,11 +55,6 @@ class StoriesOfArdaComAdapter(BaseSiteAdapter):
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','soa')
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Lord of the Rings")
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%m/%d/%Y"

fanficdownloader/adapters/adapter_svufictioncom.py

@@ -0,0 +1,271 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return SVUFictionComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class SVUFictionComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','svuf')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%b %d, %Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'svufiction.com'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Registered Users Only' in data \
+                or 'There is no such account on our website' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['cookiecheck'] = '1'
+        params['submit'] = 'Submit'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
+        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "Member Account" not in d : #Member Account
+            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+    
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=6"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+                
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+                
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid & error in url.
+                addurl = addurl.replace("&","&")
+                url = self.url+'&index=1'+addurl
+                logging.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        pt = soup.find('div', {'class' : 'title'})
+        a = pt.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        if a.text != "":
+            self.story.setMetadata('title',a.text)
+        else:
+            self.story.setMetadata('title','Banner')
+        
+        # Find authorid and URL from... author url.
+        a = pt.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        rating=pt.text.split('[')[1].split(']')[0]
+        self.story.setMetadata('rating', rating)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        bottom=soup.find('div', {'class' : 'bottom'}).text
+        self.story.setMetadata('numWords', bottom.split('Word count: ')[1].split(' Read:')[0])
+        self.story.setMetadata('datePublished', makeDate(bottom.split('Published: ')[1].split(' Updated:')[0], self.dateformat))
+        self.story.setMetadata('dateUpdated', makeDate(bottom.split('Updated: ')[1], self.dateformat))
+
+        
+        content=soup.find('div', {'class' : 'content'})
+        value=content.find('h1').nextSibling
+        svalue = ""
+        while not defaultGetattr(value,'class') == 'smaller':
+            svalue += str(value)
+            value = value.nextSibling
+        self.setDescription(url,svalue)
+        
+        status = content.text.split('Completed: ')[1]
+        if 'Yes' in status:
+            self.story.setMetadata('status', 'Completed')
+        else:
+            self.story.setMetadata('status', 'In-Progress')
+        
+        cats = content.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+        for cat in cats:
+            self.story.addToList('category',cat.string)
+            
+        chars = content.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+        for char in chars:
+            self.story.addToList('characters',char.string)
+            
+        genres = content.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+        for genre in genres:
+            self.story.addToList('genre',genre.string)
+            
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            i=1
+            for a in storyas:
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        break
+                    i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_test1.py

@@ -73,8 +73,8 @@ class TestSiteAdapter(BaseSiteAdapter):
             self.story.setMetadata(u'title',"Test Story Title "+self.story.getMetadata('storyId'))
             self.story.setMetadata('author','Test Author aa')
         self.story.setMetadata('storyUrl',self.url)
-        self.story.setMetadata('description',u'Description '+self.crazystring+u''' Done
-
+        self.setDescription(self.url,u'Description '+self.crazystring+u''' Done
+<p>
 Some more longer description.  "I suck at summaries!"  "Better than it sounds!"  "My first fic"
 ''')
         self.story.setMetadata('datePublished',makeDate("1975-03-15","%Y-%m-%d"))
@@ -103,8 +103,19 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
         
         self.story.setMetadata('rating','Tweenie')
         
-        self.story.setMetadata('authorId','98765')
-        self.story.setMetadata('authorUrl','http://author/url')
+
+        if self.story.getMetadata('storyId') == '673':
+            self.story.addToList('author','Author From List')
+            self.story.addToList('author','Author From List 2')
+        
+            self.story.addToList('authorId','98765')
+            self.story.addToList('authorId','98765-2')
+        
+            self.story.addToList('authorUrl','http://author/url')
+            self.story.addToList('authorUrl','http://author/url-2')
+        else:
+            self.story.setMetadata('authorId','98765')
+            self.story.setMetadata('authorUrl','http://author/url')
 
         self.story.addToList('warnings','Swearing')
         self.story.addToList('warnings','Violence')
@@ -114,35 +125,40 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
         self.story.addToList('category','Crossover')
         self.story.addToList('category',u'Puella Magi Madoka Magica/魔法少女まどか★マギカ')
         self.story.addToList('category',u'Magical Girl Lyrical Nanoha')
+
         self.story.addToList('genre','Fantasy')
         self.story.addToList('genre','SF')
         self.story.addToList('genre','Noir')
         
+        self.story.addToList('characters','Bob Smith')
+        self.story.addToList('characters','George Johnson')
+        self.story.addToList('characters','Fred Smythe')
+        
         self.chapterUrls = [(u'Prologue '+self.crazystring,self.url+"&chapter=1"),
                             ('Chapter 1, Xenos on Cinnabar',self.url+"&chapter=2"),
                             ('Chapter 2, Sinmay on Kintikin',self.url+"&chapter=3"),
                             ('Chapter 3, Over Cinnabar',self.url+"&chapter=4"),
                             ('Chapter 4',self.url+"&chapter=5"),
                             ('Chapter 5',self.url+"&chapter=6"),
-                            ('Chapter 6',self.url+"&chapter=6"),
-                            # ('Chapter 7',self.url+"&chapter=6"),
-                            # ('Chapter 8',self.url+"&chapter=6"),
-                            # ('Chapter 9',self.url+"&chapter=6"),
-                            # ('Chapter 0',self.url+"&chapter=6"),
-                            # ('Chapter a',self.url+"&chapter=6"),
-                            # ('Chapter b',self.url+"&chapter=6"),
-                            # ('Chapter c',self.url+"&chapter=6"),
-                            # ('Chapter d',self.url+"&chapter=6"),
-                            # ('Chapter e',self.url+"&chapter=6"),
-                            # ('Chapter f',self.url+"&chapter=6"),
-                            # ('Chapter g',self.url+"&chapter=6"),
-                            # ('Chapter h',self.url+"&chapter=6"),
-                            # ('Chapter i',self.url+"&chapter=6"),
-                            # ('Chapter j',self.url+"&chapter=6"),
-                            # ('Chapter k',self.url+"&chapter=6"),
-                            # ('Chapter l',self.url+"&chapter=6"),
-                            # ('Chapter m',self.url+"&chapter=6"),
-                            # ('Chapter n',self.url+"&chapter=6"),
+                            #('Chapter 6',self.url+"&chapter=7"),
+                            #('Chapter 7',self.url+"&chapter=8"),
+                            #('Chapter 8',self.url+"&chapter=9"),
+                            #('Chapter 9',self.url+"&chapter=0"),
+                            #('Chapter 0',self.url+"&chapter=a"),
+                            #('Chapter a',self.url+"&chapter=b"),
+                            #('Chapter b',self.url+"&chapter=c"),
+                            #('Chapter c',self.url+"&chapter=d"),
+                            #('Chapter d',self.url+"&chapter=e"),
+                            #('Chapter e',self.url+"&chapter=f"),
+                            #('Chapter f',self.url+"&chapter=g"),
+                            #('Chapter g',self.url+"&chapter=h"),
+                            #('Chapter h',self.url+"&chapter=i"),
+                            #('Chapter i',self.url+"&chapter=j"),
+                            #('Chapter j',self.url+"&chapter=k"),
+                            #('Chapter k',self.url+"&chapter=l"),
+                            #('Chapter l',self.url+"&chapter=m"),
+                            #('Chapter m',self.url+"&chapter=n"),
+                            #('Chapter n',self.url+"&chapter=o"),
                             ]
         self.story.setMetadata('numChapters',len(self.chapterUrls))
                             
@@ -170,6 +186,7 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
 <p>http://test1.com?sid=670 - Succeeds, but sleeps 2sec on each chapter</p>
 <p>http://test1.com?sid=671 - Succeeds, but sleeps 2sec metadata only</p>
 <p>http://test1.com?sid=672 - Succeeds, quick meta, sleeps 2sec chapters only</p>
+<p>http://test1.com?sid=673 - Succeeds, multiple authors</p>
 <p>Odd sid's will be In-Progress, evens complete.  sid&lt;10 will be assigned one of four languages and included in a series.</p>
 </div>
 '''

fanficdownloader/adapters/adapter_thealphagatecom.py

@@ -55,11 +55,6 @@ class TheAlphaGateComAdapter(BaseSiteAdapter):
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','tag')
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Stargate: SG-1")
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%d %b %Y"

fanficdownloader/adapters/adapter_thehexfilesnet.py

@@ -55,11 +55,6 @@ class TheHexFilesNetAdapter(BaseSiteAdapter):
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','thf')
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Harry Potter")
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%Y.%m.%d"

fanficdownloader/adapters/adapter_themasquenet.py

@@ -0,0 +1,271 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return TheMasqueNetAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class TheMasqueNetAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        if self.parsedUrl.path.split('/',)[1] == 'wiktt':
+            self.story.addToList("category","Harry Potter")
+            self.story.setMetadata('section','/wiktt/efiction/')
+            self.dateformat = "%m/%d/%Y"
+        else:
+            self.story.addToList("category","Originals")
+            self.story.setMetadata('section','/efiction/')
+            self.dateformat = "%b %d, %Y"
+            
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + self.story.getMetadata('section') + 'viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','msq')
+        
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'themasque.net'
+        
+    def getSiteExampleURLs(self):
+        return "http://themasque.net/wiktt/efiction/viewstory.php?sid=1234 http://themasque.net/efiction/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain())+"(/wiktt)?/efiction"+re.escape("/viewstory.php?sid=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Registered Users Only' in data \
+                or 'There is no such account on our website' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['cookiecheck'] = '1'
+        params['submit'] = 'Submit'
+    
+        loginUrl = 'http://' + self.getSiteDomain()  + self.story.getMetadata('section') + 'user.php?action=login'
+        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "Member Account" not in d : #Member Account
+            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=4"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid & error in url.
+                addurl = addurl.replace("&","&")
+                url = self.url+'&index=1'+addurl
+                logging.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host + self.story.getMetadata('section') + chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+                
+# summary, rated, word count, categories, characters, genre, warnings, completed, published, updated, seires
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.text
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_thepetulantpoetesscom.py

@@ -0,0 +1,239 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return ThePetulantPoetessComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class ThePetulantPoetessComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId') +'&i=1')
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','tpp')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%Y/%m/%d"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.thepetulantpoetess.com'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'You must be a member to read this story.' in data \
+                or "The Ministry of Magic does not have a record of that password. " in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['cookiecheck'] = '1'
+        params['submit'] = 'Submit'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
+        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "My Account Page" not in d : #Member Account
+            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        chapters=soup.find('select', {'name' : 'sid'})
+        if chapters == None:
+            self.chapterUrls.append((self.story.getMetadata('title'),url))
+        else:
+            for chapter in chapters.findAll('option', value=re.compile(r"viewstory.php\?sid=\d+&i=1")):
+                # just in case there's tags, like <i> in chapter titles.
+                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['value']))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+        
+        # make sure that the story id is from the first chapter
+        self.story.setMetadata('storyId',self.chapterUrls[0][1].split('=')[1].split('&')[0])
+        
+        #locate the story on author's page
+        index = 1
+        found = 0
+        while found == 0:
+            asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')+"&page="+str(index)))
+        
+            for info in asoup.findAll('td', {'class' : 'highlightcolor1'}):
+                a = info.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+                if a != None:
+                    self.story.setMetadata('title',a.text)                
+                    found = 1
+                    break
+            index=index+1
+            
+        # extract metadata
+        b=info.find('b')
+        b.find('a').extract()
+        self.story.setMetadata('rating', b.text.split('[')[1].split(']')[0])
+        
+        info = info.findNext('td', {'colspan' : '2'}) 
+        for label in info.findAll('b'):
+            value = label.nextSibling
+            
+            if "Category" in label.text:
+                for cat in info.findAll('a'):
+                    self.story.addToList('category',cat.string)
+            
+            if "Characters" in label.text:
+                for char in value.split(', '):
+                    self.story.addToList('characters',char)
+                    
+            if "Genres" in label.text:
+                for genre in value.split(', '):
+                    if "General" not in genre:
+                        self.story.addToList('genre',genre)
+                    
+            if "Warnings" in label.text:
+                for warning in value.split(', '):
+                    if "none" not in warning.lower():
+                        self.story.addToList('warnings',warning)
+                        
+                        
+        info = info.findNext('td', {'class' : 'tblborder'}) 
+        info.find('b').extract()
+        self.setDescription(url,info)
+        
+        info = info.findNext('td', {'class' : 'highlightcolor2'}) 
+        for label in info.findAll('b'):
+            value = label.nextSibling
+            
+            if "Completed" in label.text:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+                    
+            if 'Published' in label.text:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label.text:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+                
+            if 'Word Count' in label.text:
+                self.story.setMetadata('numWords', value)
+            
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.findAll('table')[2].findAll('td')[1]
+        for a in div.findAll('div'):
+            a.extract()
+        div.name='div'
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_thequidditchpitchorg.py

@@ -60,11 +60,6 @@ class TheQuidditchPitchOrgAdapter(BaseSiteAdapter): # XXX
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','tqdpch') # XXX
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Harry Potter") # XXX
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%m/%d/%Y" # XXX

fanficdownloader/adapters/adapter_thewriterscoffeeshopcom.py

@@ -133,7 +133,7 @@ class TheWritersCoffeeShopComSiteAdapter(BaseSiteAdapter):
 
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))

fanficdownloader/adapters/adapter_tthfanficorg.py

@@ -151,12 +151,33 @@ class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter):
         self.story.setMetadata('authorId',a['href'].split('/')[1].split('-')[1])
         self.story.setMetadata('authorUrl','http://'+self.host+a['href'])
         self.story.setMetadata('author',stripHTML(a))
+        authorurl = 'http://'+self.host+a['href']
 
+        ainfo = soup.find('a', href='/StoryInfo-%s-1'%self.story.getMetadata('storyId'))
+        if ainfo != None: # indicates multiple authors/contributors.
+            try:
+                # going to pull part of the meta data from author list page.
+                infourl = 'http://'+self.host+ainfo['href']
+                logging.debug("**StoryInfo** URL: "+infourl)
+                infodata = self._fetchUrl(infourl)
+                infosoup = bs.BeautifulSoup(infodata)
+
+                for a in infosoup.findAll('a',href=re.compile(r"^/Author-\d+")):
+                    self.story.addToList('authorId',a['href'].split('/')[1].split('-')[1])
+                    self.story.addToList('authorUrl','http://'+self.host+a['href'].replace("/Author-","/AuthorStories-"))
+                    self.story.addToList('author',stripHTML(a))                
+                                            
+            except urllib2.HTTPError, e:
+                if e.code == 404:
+                    raise exceptions.StoryDoesNotExist(url)
+                else:
+                    raise e
+    
         try:
-            # going to pull part of the meta data from author list page.
-            logging.debug("**AUTHOR** URL: "+self.story.getMetadata('authorUrl'))
-            authordata = self._fetchUrl(self.story.getMetadata('authorUrl'))
-            descurl=self.story.getMetadata('authorUrl')
+            # going to pull part of the meta data from *primary* author list page.
+            logging.debug("**AUTHOR** URL: "+authorurl)
+            authordata = self._fetchUrl(authorurl)
+            descurl=authorurl
             authorsoup = bs.BeautifulSoup(authordata)
             # author can have several pages, scan until we find it.
             while( not authorsoup.find('a', href=re.compile(r"^/Story-"+self.story.getMetadata('storyId'))) ):
@@ -209,7 +230,7 @@ class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter):
                 if not BtVSNonX:
                     BtVS = False # Don't add BtVS if Non-Crossover, unless it's a BtVS/AtS Non-Crossover
 
-        print("BtVS: %s BtVSNonX: %s"%(BtVS,BtVSNonX))
+        #print("BtVS: %s BtVSNonX: %s"%(BtVS,BtVSNonX))
         if BtVS:
             self.story.addToList('category','Buffy: The Vampire Slayer')
             

fanficdownloader/adapters/adapter_twilightarchivescom.py

@@ -0,0 +1,185 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return TwilightArchivesComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class TwilightArchivesComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL. http://www.twilightarchives.com/read/9353
+        self._setURL('http://' + self.getSiteDomain() + '/read/'+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','twa')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%d %b %y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.twilightarchives.com'
+       
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/read/1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://" + self.getSiteDomain()+"/read/")+r"\d+(/d+)?$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('h1')
+        self.story.setMetadata('title',a.string)
+        
+        # Find the chapters:
+        chapters=soup.find('ol', {'class' : 'chapters'})
+        if chapters != None:
+            for chapter in chapters.findAll('a', href=re.compile(r'/read/'+self.story.getMetadata('storyId')+"/\d+$")):
+                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+chapter['href']))
+        else:
+            self.chapterUrls.append((self.story.getMetadata('title'),url))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+                
+# rated, genre, warnings, seires
+
+        summary = soup.find('p', {'class' : 'images'})
+        self.setDescription(url,summary)
+
+        for c in soup.findAll('h2', {'class' : 'title'}):
+            div = c.nextSibling.nextSibling
+            
+            if 'Information' in c.text:
+                for dt in div.findAll('dt'):
+                    dd=dt.nextSibling.nextSibling
+                    
+                    if 'Author' in dt.text:
+                        a=dd.find('a')
+                        self.story.setMetadata('authorId',a['href'].split('/')[2])
+                        self.story.setMetadata('authorUrl','http://'+self.host+a['href'])
+                        self.story.setMetadata('author',a.text)
+                        
+                    if 'Words' in dt.text:
+                        self.story.setMetadata('numWords', dd.text)
+                        
+                    if 'Published' in dt.text:
+                        self.story.setMetadata('datePublished', makeDate(stripHTML(dd.text), self.dateformat))
+            
+                    if 'Updated' in dt.text:
+                        self.story.setMetadata('dateUpdated', makeDate(stripHTML(dd.text), self.dateformat))
+                    
+                    if 'Status' in dt.text:
+                        if 'Complete' in dd.text:
+                            self.story.setMetadata('status', 'Completed')
+                        else:
+                            self.story.setMetadata('status', 'In-Progress')
+
+            if 'Categories' in c.text:
+                for a in div.findAll('a'):
+                    self.story.addToList('category',a.text)
+                    
+            if 'Characters' in c.text:
+                for a in div.findAll('a'):
+                    self.story.addToList('category',a.text)
+
+            if 'Series' in c.text:
+                a=div.find('a')
+                series_name = a.text
+                series_url = 'http://'+self.host+a['href']
+                
+                seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+                storyas = seriessoup.find('tbody').findAll('a', href=re.compile(r'^/read/\d+$'))
+                i=1
+                for a in storyas:
+                    if a['href'] == ('/read/'+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        break
+                    i+=1
+                    
+        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
+        a=asoup.find('tbody').find('a', href=re.compile(r'^/read/'+self.story.getMetadata('storyId')))
+        self.story.setMetadata('rating',a.parent.nextSibling.nextSibling.text)
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'class' : 'size images medium'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_twilightednet.py

@@ -37,7 +37,6 @@ class TwilightedNetSiteAdapter(BaseSiteAdapter):
                                # Most sites that claim to be
                                # iso-8859-1 (and some that claim to be
                                # utf8) are really windows-1252.
-        self.story.addToList("category","Twilight")
         self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
         self.password = ""
         

fanficdownloader/adapters/adapter_twiwritenet.py

@@ -37,7 +37,6 @@ class TwiwriteNetSiteAdapter(BaseSiteAdapter):
                                # Most sites that claim to be
                                # iso-8859-1 (and some that claim to be
                                # utf8) are really windows-1252.
-        self.story.addToList("category","Twilight")
         self.is_adult = False
         self.username = "NoneGiven" # if left empty, twiwrite.net doesn't return any message at all.
         self.password = ""

fanficdownloader/adapters/adapter_walkingtheplankorg.py

@@ -53,11 +53,6 @@ class WalkingThePlankOrgAdapter(BaseSiteAdapter):
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','wtp')
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Harry Potter")
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%b %d, %Y"

fanficdownloader/adapters/adapter_wizardtalesnet.py

@@ -0,0 +1,299 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return WizardTalesNetAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class WizardTalesNetAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','wzt')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%m/%d/%Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.wizardtales.net'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Registered Users Only' in data \
+                or 'There is no such account on our website' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['cookiecheck'] = '1'
+        params['submit'] = 'Submit'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
+        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "Member Account" not in d : #Member Account
+            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=4"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid & error in url.
+                addurl = addurl.replace("&","&")
+                url = self.url+'&index=1'+addurl
+                logging.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        pt = soup.find('div', {'id' : 'pagetitle'})
+        a = pt.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.text)
+        
+        # Find authorid and URL from... author url.
+        a = pt.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        rating=pt.text.split('[')[1].split(']')[0]
+        self.story.setMetadata('rating', rating)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        content=soup.find('div',{'class' : 'content'})
+        
+        for genre in content.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')):
+            self.story.addToList('genre',genre.string)
+
+        for warning in content.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')):
+            self.story.addToList('warnings',warning.string)
+        
+        labels = content.findAll('b')
+        
+        value = labels[0].previousSibling
+        svalue = ""
+        while value != None:
+            val = value
+            value = value.previousSibling
+        while "Categories" not in val:
+            svalue += str(val)
+            val = val.nextSibling
+        self.setDescription(url,svalue)
+        
+        
+        
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', stripHTML(value).split(': ')[1].split(';')[0])
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value).split(': ')[1].split(';')[0], self.dateformat))
+            
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value).split(': ')[1], self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            i=1
+            for a in storyas:
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        break
+                    i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.findAll('div', {'id' : 'story'})[1]
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_wolverineandroguecom.py

@@ -0,0 +1,216 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return WolverineAndRogueComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class WolverineAndRogueComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/wrfa/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','wrfa')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%m/%d/%Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.wolverineandrogue.com'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/wrfa/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/wrfa/viewstory.php?sid=")+r"\d+$"
+
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        pt = soup.find('div', {'id' : 'pagetitle'})
+        a = pt.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.text)
+        
+        # Find authorid and URL from... author url.
+        a = pt.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/wrfa/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        rating=pt.text.split('(')[1].split(')')[0]
+        self.story.setMetadata('rating', rating)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/wrfa/'+chapter['href']))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        content=soup.find('div',{'class' : 'content'})
+        labels = soup.findAll('span',{'class':'label'})
+        
+        value = labels[0].previousSibling
+        svalue = ""
+        while value != None:
+            val = value
+            value = value.previousSibling
+        while "Categories" not in str(val):
+            svalue += str(val)
+            val = val.nextSibling
+        self.setDescription(url,svalue)
+        
+        
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value.split(' -')[0])
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Complete' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(value.split(' -')[0], self.dateformat))
+            
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/wrfa/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            i=1
+            for a in storyas:
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        break
+                    i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_wraithbaitcom.py

@@ -56,11 +56,6 @@ class WraithBaitComAdapter(BaseSiteAdapter):
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','wb')
 
-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Stargate")
-
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%d  %b %Y"
@@ -119,10 +114,11 @@ class WraithBaitComAdapter(BaseSiteAdapter):
         self.story.setMetadata('title',a.string)
         
         # Find authorid and URL from... author url.
-        a = pt.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
+        alist = pt.findAll('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        for a in alist:
+            self.story.addToList('authorId',a['href'].split('=')[1])
+            self.story.addToList('authorUrl','http://'+self.host+'/'+a['href'])
+            self.story.addToList('author',a.string)
 
         rating=pt.text.split('[')[1].split(']')[0]
         self.story.setMetadata('rating', rating)
@@ -215,8 +211,7 @@ class WraithBaitComAdapter(BaseSiteAdapter):
 
         logging.debug('Getting chapter text from: %s' % url)
 
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        soup = bs.BeautifulSoup(self._fetchUrl(url))
         
         div = soup.find('div', {'id' : 'story'})
 

fanficdownloader/adapters/adapter_yourfanfictioncom.py

@@ -36,6 +36,13 @@ class YourFanfictionComAdapter(BaseSiteAdapter):
     def __init__(self, config, url):
         BaseSiteAdapter.__init__(self, config, url)
 
+        # yourfanfiction.com blocks the default user-agent.  However,
+        # when asked, they said it was just general anti-spam, not
+        # targeted as us and offered to 'whitelist our IP'.  Clearly,
+        # that wouldn't work, but it does let me do this in good
+        # conscience:
+        self.opener.addheaders = [('User-agent', 'FFDL/1.6')]
+
         self.decode = ["Windows-1252",
                        "utf8"] # 1252 is a superset of iso-8859-1.
                                # Most sites that claim to be
@@ -108,7 +115,7 @@ class YourFanfictionComAdapter(BaseSiteAdapter):
         # viewstory.php?sid=1654&ageconsent=ok&warning=5
         #print data
         #m = re.search(r"'viewstory.php\?sid=1882(&warning=4)'",data)
-        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
+        m = re.search(r"'viewstory.php\?sid=\d+((&ageconsent=ok)?&warning=\d+)'",data)
         if m != None:
             if self.is_adult or self.getConfig("is_adult"):
                 # We tried the default and still got a warning, so
@@ -116,7 +123,8 @@ class YourFanfictionComAdapter(BaseSiteAdapter):
                 # link and reload data.
                 addurl = m.group(1)
                 # correct stupid & error in url.
-                addurl = addurl.replace("&","&")
+                # explicitly put ageconsent because google appengine regexp doesn't include it for some reason.
+                addurl = addurl.replace("&","&")+'&ageconsent=ok'
                 url = self.url+'&index=1'+addurl
                 logging.debug("URL 2nd try: "+url)
 
@@ -133,9 +141,17 @@ class YourFanfictionComAdapter(BaseSiteAdapter):
         if "Access denied. This story has not been validated by the adminstrators of this site." in data:
             raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
             
+        # because for some reason, this works while simple 'print data' errors on ascii conversion.
+        # loopdata = data
+        # chklen=5000
+        # while len(loopdata) > 0:
+        #     if len(loopdata) < 5000:
+        #         chklen = len(loopdata)
+        #     logging.info("loopdata: %s" % loopdata[:chklen])
+        #     loopdata = loopdata[chklen:]
+            
         # use BeautifulSoup HTML parser to make everything easier to find.
         soup = bs.BeautifulSoup(data)
-        # print data
 
         # Now go hunting for all the meta data and the chapter list.
         
@@ -175,9 +191,11 @@ class YourFanfictionComAdapter(BaseSiteAdapter):
             if 'Summary' in label:
                 ## Everything until the next span class='label'
                 svalue = ""
-                while not defaultGetattr(value,'class') == 'label':
+                while value and not defaultGetattr(value,'class') == 'label':
                     svalue += str(value)
                     value = value.nextSibling
+                # sometimes poorly formated desc (<p> w/o </p>) leads
+                # to all labels being included.
                 self.setDescription(url,svalue)
                 #self.story.setMetadata('description',stripHTML(svalue))
 
@@ -238,7 +256,6 @@ class YourFanfictionComAdapter(BaseSiteAdapter):
             storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
             i=1
             for a in storyas:
-                print("series a['href']:%s"%a['href'])
                 # skip 'report this' and 'TOC' links
                 if 'contact.php' not in a['href'] and 'index' not in a['href']:
                     if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):

fanficdownloader/adapters/base_adapter.py

@@ -89,6 +89,7 @@ class BaseSiteAdapter(Configurable):
         self.oldimgs = None
         self.oldcover = None # (data of existing cover html, data of existing cover image)
         self.calibrebookmark = None
+        self.logfile = None
         ## order of preference for decoding.
         self.decode = ["utf8",
                        "Windows-1252"] # 1252 is a superset of
@@ -198,7 +199,8 @@ class BaseSiteAdapter(Configurable):
                 if (self.chapterFirst!=None and index < self.chapterFirst) or \
                         (self.chapterLast!=None and index > self.chapterLast):
                     self.story.addChapter(removeEntities(title),
-                                          None)
+                                          None,
+                                          self)
                 else:
                     if self.oldchapters and index < len(self.oldchapters):
                         data = self.utf8FromSoup(None,
@@ -207,7 +209,8 @@ class BaseSiteAdapter(Configurable):
                     else:
                         data = self.getChapterText(url)
                     self.story.addChapter(removeEntities(title),
-                                          removeEntities(data))
+                                          removeEntities(data),
+                                          self)
             self.storyDone = True
             
             # include image, but no cover from story, add default_cover_image cover.
@@ -229,6 +232,8 @@ class BaseSiteAdapter(Configurable):
             # cheesy way to carry calibre bookmark file forward across update.
             if self.calibrebookmark:
                 self.story.calibrebookmark = self.calibrebookmark
+            if self.logfile:
+                self.story.logfile = self.logfile
                 
         return self.story
 
@@ -286,7 +291,7 @@ class BaseSiteAdapter(Configurable):
     def setDescription(self,url,svalue):
         #print("\n\nsvalue:\n%s\n"%svalue)
         if self.getConfig('keep_summary_html'):
-            if isinstance(svalue,str) or isinstance(svalue,unicode):
+            if isinstance(svalue,basestring):
                 svalue = bs.BeautifulSoup(svalue)
             self.story.setMetadata('description',self.utf8FromSoup(url,svalue))
         else:

fanficdownloader/bbcodeutils/bbcode2html.py

@@ -0,0 +1,325 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+#
+# Author:        Pau Sanchez (contact@pausanchez.com)
+# Version:       v1.0
+# Last Modified: 2010/09/15
+# 
+# For the latest version check out:
+#   http://www.codigomanso.com/en/projects
+# 
+# My blog:
+#   http://www.codigomanso.com/en/  - English Version
+#   http://www.codigomanso.com/es/  - Spanish Version
+#
+
+import sys
+import os
+import re
+import urllib
+
+class bbcode2html:
+  '''
+  This class gets a parsed BBCode and transforms it to valid HTML
+
+  Useful functions of this class:
+    html
+    convertToHTML
+
+  Example:
+    > parser = bbcodeparser ()
+    > parser.parse ('[b]bold[/b]')
+    > bbcode2html (parser).html()
+    <b>bold</b>
+
+    # This is faster for huge strings but changes the parser object internally
+    > bbcode2html (parser).html(doDeepCopy = False)
+    <b>bold</b>
+  '''
+  def __init__ (self, parser):
+    self._parser = parser
+    return
+
+  def html (self, allowClassAttr = False, doDeepCopy = True, parser = None):
+    '''
+    Convert current parsed code to HTML
+
+    Example:
+      code = bbcodeparser ('[b]bold[/b]')
+      code.html() -> '<b>bold</b>'
+    '''
+    if parser is None:
+      parser = self._parser
+
+    tokens = parser
+    if instanceof (parser, bbcodeparser):
+      tokens = parser.getTokens()
+
+    return bbcode2html.convertToHTML (tokens, allowClassAttr = allowClassAttr, doDeepCopy = doDeepCopy)
+
+  @staticmethod
+  def htmlString (string):
+    toReplace = {
+      u'<' : '&lt;',
+      u'>' : '&gt;',
+      u'"' : "&quot;",
+      u'&' : "&amp;"
+    }
+    for entity in toReplace:
+      string = string.replace(entity, toReplace[entity])
+    return string
+
+  @staticmethod
+  def getValidTags ():
+    simpleTags = ['b', 'u', 'i', 'sup', 'sub', 'ul', 'ol', 'li', 'table', 'tr', 'th', 'td', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6']
+    validTags = {
+      'p'         : { 'color' : 'color', 'size' : 'size', 'font' : 'font' },
+      'color'     : { 'color' : 'color' },
+      'size'      : { 'size' : 'size' },
+      'font'      : { 'font' : 'font' },
+      'img'       : { 'alt' : 'alt', 'title' : 'title', 'width' : 'width' , 'height' : 'height', 'img' : 'img'},
+      'url'       : { 'href' : 'href', 'url' : 'href', 'link'  : 'href', 'title' : 'title' },
+      's'         : { },
+      'code'      : { },
+      'quote'     : { },
+      'list'      : { 'list' : 'type' },
+      'email'     : { 'email': 'href'},
+      'google'    : { 'google':  'google'},
+      'wikipedia' : { 'wikipedia' : 'wikipedia', 'language' : 'language', 'lang' : 'lang'}
+    }
+
+    for tag in simpleTags:
+      validTags[tag] = { }
+    return validTags
+
+  @staticmethod
+  def convertToHTML (tokens, allowClassAttr = False, validTags = None, doDeepCopy = True):
+    '''
+    Convert internally parsed BBCode to XHTML
+
+    @doDeepCopy 
+      True:  it does a deep copy of tokens so this list will remain unchanged
+      False: tokens will be modified internally, but the output will be produced like 5x faster
+             it's a good idea to use False only when this is the last operation
+    '''
+    # do a deep copy
+    if doDeepCopy:
+      import copy
+      tokens = copy.deepcopy (tokens)
+
+    # filter invalid tags and attributes
+    if validTags is None:
+      validTags = bbcode2html.getValidTags()    
+
+    bbcode2html._filterInvalidTagsAndAttributes (tokens, validTags, allowClassAttr)
+    
+    # Start to convert
+    index       = 0
+    tokenLength = len (tokens)
+
+    # use a list for the output (an order of magnitude faster than using string concatenation)
+    htmlList = []
+    lastListOpener = []
+
+    while index < tokenLength:
+
+      if isinstance (tokens [index], basestring):
+        htmlList.append (bbcode2html.htmlString (tokens [index]))
+        index += 1
+        continue
+
+      token     = tokens[index] 
+      tag       = token['tag']  # opening or closing simple tag. e.g: 'b', '/b', '/u', ...
+      tagName   = (tag[1:] if tag[0] == '/' else tag)
+      tagOpener = (u'/' if tag[0] == '/' else u'')
+      tokenArgs = (token['args'] if 'args' in token else {})
+
+      # opening or closing simple tag COLOR / SIZE
+      if (tagName in ['p', 'color', 'size', 'font']): 
+        style  = ''
+        style += ((u' color: ' + tokenArgs['color'] + u';') if ('color' in tokenArgs) else '')
+        style += ((u' font-size: ' + tokenArgs['size'] + u'pt;') if ('size' in tokenArgs) else '')
+        style += ((u' font-family: ' + tokenArgs['font'] + u';') if ('font' in tokenArgs) else '')
+        style  = style.strip()
+
+        pArgs = {}
+        if style != '':
+          pArgs ['style'] = style
+
+        if 'class' in tokenArgs:
+          pArgs ['class'] = tokenArgs['class']
+
+        if ('args' not in token) and (tagName != 'p'):
+          if (tagOpener == '/'): # if closing tag, close it
+            htmlList.append (u'</span>')
+          index += 1
+          continue
+
+        if tagName != 'p':
+          tag = tagOpener + u'span'
+
+        htmlList.append (bbcode2html.xml (tag, pArgs))
+
+      # IMG tag
+      elif tag == 'img' and (index+2 < tokenLength):
+        if 'img' in tokenArgs:
+          # has the form of <width>x<height> ?
+          sizeMatch = re.match (u'^\s*(\d+)[xX](\d+)\s*$', tokenArgs['img'])
+          if sizeMatch is not None:
+            tokenArgs['width']  = sizeMatch.group(1)
+            tokenArgs['height'] = sizeMatch.group(2)
+          # then assume is the alternative text
+          else:
+            tokenArgs['alt'] = tokenArgs['img']
+          del tokenArgs['img']
+
+        # add the source of the image
+        tokenArgs ['src'] = tokens[index+1]
+
+        # [img]http://www.whatever.com/pic.jpg[/img]
+        htmlList.append (
+          bbcode2html.xml ('img', tokenArgs, soloTag=True)
+        )
+        index += 2 # skip next token and closing tag
+
+      # URL tag
+      elif tag == 'url':
+        if ('args' not in token) and (index+2 < tokenLength):
+          # [url]http://www.google.com[/url]
+          htmlList.append (bbcode2html.xml ('a', { 'href' : tokens[index+1] }))
+        else:
+          # [url=http://www.google.com]Google[/url]
+          # [url link=http://www.google.com title="This is Google"]Google[/url]
+          htmlList.append (bbcode2html.xml ('a', tokenArgs))
+  
+      # URL closing tag (sometimes needed)
+      elif (tag == '/url') or (tag == '/email'):
+        htmlList.append (u'</a>')
+
+      # Email tag
+      elif tag == 'email':
+        if ('args' not in token) and (index+2 < tokenLength):
+          # [email]asdf@asdf.com]
+          htmlList.append (bbcode2html.xml ('a', { 'href' : u'mailto:' + tokens[index+1].strip() }))
+        else:
+          # [email=asdf@asfd.com]john smith[/email]
+          if 'href' in tokenArgs:
+            tokenArgs['href'] = u'mailto:' + tokenArgs['href']
+          htmlList.append (bbcode2html.xml ('a', tokenArgs))
+  
+      elif tagName == 'list':
+        if tagOpener == '/':
+          htmlList.append (bbcode2html.xml (u'/' + lastListOpener.pop()))
+        else:
+          if ('type' not in tokenArgs):
+            htmlList.append (bbcode2html.xml (tagOpener + u'ul', tokenArgs))
+            lastListOpener.append ('ul')
+          else:
+            htmlList.append (bbcode2html.xml (tagOpener + u'ol', tokenArgs))
+            lastListOpener.append ('ol')
+
+      elif tagName == '*':
+        htmlList.append (bbcode2html.xml (tagOpener + u'li', tokenArgs))
+
+      elif (tagName == 's'):
+        tokenArgs['style'] =  'text-decoration: line-through;'
+        htmlList.append (bbcode2html.xml (tagOpener + u'span', tokenArgs))
+
+      elif (tagName == 'code'):
+        htmlList.append (bbcode2html.xml (tagOpener + u'pre', tokenArgs))
+
+      elif (tagName == 'quote'):
+        htmlList.append (bbcode2html.xml (tagOpener + u'blockquote', tokenArgs))
+
+      elif (tagName == 'google'):
+        htmlList.append (
+          bbcode2html.xml (
+            tagOpener + u'a', 
+            {'href' : 'http://www.google.com/search?q=' + urllib.quote_plus (tokens[index+1])},
+            tokens[index+1]
+          )
+        )
+        index += 2
+
+      elif (tagName == 'wikipedia'):
+        subdomain = 'www'
+        for arg  in ['lang', 'language', 'wikipedia']:
+          if arg in tokenArgs:
+            subdomain = tokenArgs[arg] 
+
+        htmlList.append (
+          bbcode2html.xml (
+            tagOpener + u'a', 
+            {'href' : 'http://' + subdomain + '.wikipedia.org/wiki/' + tokens[index+1].replace (' ', '_')},
+            tokens[index+1]
+          )
+        )
+        index += 2
+
+      elif (tagName in validTags):
+        htmlList.append (
+          bbcode2html.xml (tag, tokenArgs)
+        )
+
+      else:
+        # ignore this tag
+        pass 
+
+      index += 1
+
+    return ''.join (htmlList)
+
+  @staticmethod
+  def _filterInvalidTagsAndAttributes (tokens, validTags, allowClassAttr):
+    '''
+    Helper function to filter out invalid attributes from the tokens list
+    '''
+    # add 'class' attribute as valid (mapping 'class' itself)
+    if allowClassAttr:
+      for attr in validTags:
+        validTags[attr]['class'] = 'class'
+
+    # remove invalid attributes from tokens
+    for tindex in range(0, len(tokens)):
+      if isinstance (tokens[tindex], dict) and ('args' in tokens[tindex]) and (tokens[tindex]['tag'] in validTags):
+        validList = validTags[tokens[tindex]['tag']]
+
+        filteredArgs = {}
+        for arg in tokens[tindex]['args']:
+          if arg in validList:
+            # rename the argument
+            filteredArgs[validList[arg]] = tokens[tindex]['args'][arg]
+          else:
+            pass # do not include this arg in the filteredArgs
+
+        tokens[tindex]['args'] = filteredArgs
+
+    return
+
+  @staticmethod
+  def xml (tag, attrs = {}, text = None, soloTag = False):
+    '''
+    Helper function to produce valid XML output
+    '''
+    xml = u'<' + tag.lower()
+
+    # make sure we sort attributes alphabetically (for deterministic output)
+    # Faster but non-deterministic:
+    #  for (key, value) in attrs.iteritems():
+    #    xml += u' ' + key + u'="' + value + u'"'
+    for key in sorted (attrs.keys()):
+      xml += u' ' + key + u'="' + attrs[key] + u'"'
+      
+    # close tag
+    if text is None:
+      if soloTag:
+        xml += u' />'
+      else:
+        xml += u'>'
+    else:
+      xml += u'>' + text + u'</' + tag.lower() + '>'
+
+    return xml
+
+
+

fanficdownloader/bbcodeutils/bbcodebuilder.py

@@ -0,0 +1,77 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+#
+# Author:        Pau Sanchez (contact@pausanchez.com)
+# Version:       v1.0
+# Last Modified: 2010/09/15
+# 
+# For the latest version check out:
+#   http://www.codigomanso.com/en/projects
+# 
+# My blog:
+#   http://www.codigomanso.com/en/  - English Version
+#   http://www.codigomanso.com/es/  - Spanish Version
+#
+
+import sys
+import os
+import re
+import hashlib
+
+class bbcodebuilder:
+  '''
+  This class helps to build BBCode programmatically.
+
+  The function names are used as the tag name, then the first parameter
+  is the string that goes inside the tags and any extra parameter is 
+  appended as a parameter to the tag
+
+  Examples:
+    > bbcode = bbcodebuilder()  # create a instance!
+
+    > print bbcode.b ('bold')
+    [b]bold[/b]
+
+    > print bbcode.color ('this goes in red', 'red')
+    [color=red]this goes in red[/color]
+
+    > print bbcode.url ('Google', 'http://www.google.com')
+    [url=http://www.google.com]Google[/url]
+    
+    > print bbcode.alist('item 1', 'item 2')
+    [list=a]
+      [*]item 1
+      [*]item 2
+    [/list]
+
+
+  This solution is based on the recipe found on:
+    http://code.activestate.com/recipes/576831-simple-bbcode-support/
+  '''
+
+  def __getattr__(self, name):
+    '''
+    This is a generic getter that returns a function which gets the first parameter
+    as the string that goes between the tags, and extra parameters as tag parameters.
+
+    The name of the attribute is used as the tag name
+    '''
+    class bbcodebuilder_helper:
+      def __init__(self, name):
+        self._name = name
+
+      def __call__(self, string, *args):
+        return u'[{0}{1}]{2}[/{0}]'.format(self._name, (u'=' + u','.join(map(str, args))) if args else u'', string)
+    
+    return bbcodebuilder_helper (name)
+
+  def list(self, *items):
+    return u'[list]' + u''.join(map(lambda item: u"\n  [*]" + item, items)) + u"\n[/list]"
+
+  def nlist(self, *items):
+    return u'[list=1]' + u''.join(map(lambda item: u"\n  [*]" + item, items)) + u"\n[/list]"
+
+  def alist(self, *items):
+    return u'[list=a]' + u''.join(map(lambda item: u"\n  [*]" + item, items)) + u"\n[/list]"
+
+

fanficdownloader/bbcodeutils/bbcodeparser.py

@@ -0,0 +1,251 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+#
+# Author:        Pau Sanchez (contact@pausanchez.com)
+# Version:       v1.0
+# Last Modified: 2010/09/15
+# 
+# For the latest version check out:
+#   http://www.codigomanso.com/en/projects
+# 
+# My blog:
+#   http://www.codigomanso.com/en/  - English Version
+#   http://www.codigomanso.com/es/  - Spanish Version
+#
+
+import sys
+import os
+import re
+import hashlib
+
+class bbcodeparser:
+  '''
+  This class parses BBCode into a internal structure to allow later processing and
+  conversion to HTML.
+
+  The parser tries to fix invalid code (like unclosed tags)
+
+  Useful URLs:
+    http://en.wikipedia.org/wiki/BBCode
+    http://www.bbcode.org/reference.php
+
+  Example:
+    > bbcode = bbcodeparser ()
+    > bbcode.parse ('[b]text in bold[/b]').html()
+    <b>text in bold</b>
+
+    # dump HTML
+    > bbcode.parse ('[p][color=red]text in red').html()
+    <p><span style="color:red;">text in red</span></p>
+
+    # dump fixed BBCode
+    > bbcode.parse ('[p][color=red]text in red').bbcode()
+    [p][color=red]text in red[/color][/p]
+
+    > bbcode.parse ('This [b][i]code[/b] will be fixed[/invalid]').bbcode()
+    This [b][i]code[/i][/b] will be fixed
+
+    # dump fixed bbcode
+    > str (bbcodeparse ('This [b][i]code[/b] will be fixed[/invalid]'))
+    This [b][i]code[/i][/b] will be fixed
+  '''
+  _bbcode = ''
+  _tokens = []
+
+  def __init__ (self, bbcode = '', fixInvalidCode = True):
+    ''' Initialize and parse bbcode string (if any is given)
+    '''
+    self.parse (bbcode, fixInvalidCode)
+    return
+
+  def __str__ (self):
+    return self.bbcode()
+
+  def parse (self, bbcode = None, fixInvalidCode = True):
+    '''
+    It will parse and return the token list, trying to fix tags if
+    fixInvalidCode is True
+
+    It will return the current object to allow chaining
+
+    Example:
+      code = bbcode()
+      code.parse ('<b>bold</b>', True) -> 
+      code.parse ('<b>bold<i>italics</b>', True) -> internally will add the missing '</i>'
+    '''
+    if bbcode is not None:
+      self._bbcode = bbcode
+      self._tokens = self.tokenize (bbcode)
+      if fixInvalidCode:
+        self._tokens = self.fixWrongTags (self._tokens)
+
+    return self
+
+  # return ALL tokens
+  def getTokens (self):
+    return self._tokens
+
+  def bbcode (self):
+    '''
+    Dump BBCode again. This is useful for dumping valid BBCode
+    '''
+    bbcode = []
+    for token in self._tokens:
+      if token is None:
+        continue
+
+      if isinstance (token, basestring):
+        bbcode.append (token.replace (u'[', u'\[').replace (u']', u'\]'))
+        continue
+      
+      tag       = token['tag']  # opening or closing simple tag. e.g: 'b', '/b', '/u', ...
+      tagOpener = (u'/' if tag[0] == u'/' else u'')
+
+      if (tagOpener == '/') or ('args' not in token):
+        bbcode.append (u'[' + tag + u']')
+      else:
+        # process args
+        argstr = ''
+
+        # the arg with the same name as the tag repersents the '=whatever'
+        if tag in token['args']:
+          if re.match ('\s|"', token['args'][tag]) is None:
+            argstr = u'=' + token['args'][tag]
+          else:
+            argstr = u'="' + token['args'][tag].replace (u'"', u'\"') + u'"'
+
+        for (k,v) in token['args'].iteritems():
+          if k == tag: # already processed
+            continue
+          argstr += ' ' + k + u'="' + v.replace (u'"', u'\"') + u'"'
+          
+        bbcode.append (u'[' + tag + argstr + ']')
+
+    return u''.join (bbcode)
+
+  def html (self, allowClassAttr = False, doDeepCopy = True):
+    '''
+    Convert current parsed code to HTML
+
+    @allowClassAttr
+      Is something like [b class="asdf"] allowed?
+
+    @doDeepCopy 
+      True:  it does a deep copy of tokens so this list will remain unchanged
+      False: tokens will be modified internally, but the output will be produced like 5x faster
+             it's a good idea to use False when the string parsed is huge and this is the
+             last operation on the string
+
+    Example:
+      code = bbcode ('[b]bold[/b]')
+      code.html() -> '<b>bold</b>'
+    '''
+    from bbcode2html import bbcode2html
+    return bbcode2html.convertToHTML (self._tokens, allowClassAttr = allowClassAttr, doDeepCopy = doDeepCopy)
+
+
+  @staticmethod
+  def fixWrongTags (inTokenList):
+    ''' Add missing tokens that have not been closed properly and try to fix some scenarios
+    '''
+    opened       = []
+    outTokenList = []
+    for token in inTokenList:
+      # normal string... do nothing
+      if isinstance(token, basestring):
+        outTokenList.append (token)
+      else:
+        # if starts with '/' is closing a tag
+        if token['tag'][0] == '/':
+          while (len (opened) > 0) and (opened[-1] != token['tag'][1:]):
+            outTokenList.append ({'tag' : '/' + opened[-1] })
+            del opened[-1]
+
+          if len(opened):
+            del opened[-1]
+            outTokenList.append (token)
+
+        # opening tag
+        else:
+          # if I open the same tag I opened before, close it, and open it again
+          if (len(opened) > 0) and (token['tag'] == opened[-1]):
+            outTokenList.append ({'tag' : '/' + opened[-1] })
+          else:
+            opened.append (token['tag'])
+          outTokenList.append (token)
+
+    # close all elements that have not been closed
+    while len(opened):
+      outTokenList.append ({'tag' : '/' + opened[-1] })
+      del opened[-1]
+
+    return outTokenList
+
+  @staticmethod
+  def tokenize(code):
+    '''
+    Tokenize BBCode tags and parameters
+
+    Return the token list using a internal format. See the example:
+      [
+        { 'tag' : 'p', 'args' : { 'font' : 'arial' } },
+        'This is ',
+        { 'tag' : 'url', 'args' : {'url' : 'http://www.google.com'} },
+        'a link to google',
+        { 'tag' : '/url' },
+        { 'tag' : '/p' }
+      ]
+    '''
+    re_tags    = re.compile (r'(\[[^]]+\])', re.DOTALL | re.UNICODE)
+    re_tagName = re.compile (r'\[([^]=\s]+)([^]]*)\]',  re.DOTALL | re.UNICODE)
+    #re_tagArgs = re.compile (r'\s*([^=]*)=(("([^"]+)")|([^\s]+))',  re.DOTALL | re.UNICODE)
+    re_tagArgs = re.compile (r'\s*([\w]*)=(("([^"]+)")|([^\s]+))',  re.DOTALL | re.UNICODE)
+
+    # get a unique name and replace escaped braces 
+    unique = hashlib.md5(code).hexdigest()
+    code   = code.replace ('\[', unique+'_OPEN_BRACE')
+    code   = code.replace ('\]', unique+'_CLOSE_BRACE')
+
+    splitted = re_tags.split(code)
+
+    outTokenList = []
+    for token in splitted:
+      if len(token) == 0:
+        continue
+      
+      if token[0] == '[':
+        match   = re_tagName.match (token)
+        tagName = match.group(1)
+        tagArgs = match.group(2)
+        
+        tagToken = { 'tag' : tagName.lower() }
+
+        # parse arguments (if any)
+        if len(tagArgs) > 0:
+          allArgs = re_tagArgs.findall(tagArgs)
+
+          tagArgs = {}
+          for arg in allArgs:
+            # if the argument has no name, use the tagName itself
+            argName  = (arg[0] if arg[0] != '' else tagName)
+            argValue = (arg[3] if (arg[1][0] == '"') else arg[4])
+
+            tagArgs[argName.lower()] = argValue.replace ('\"', '"')
+
+          tagToken['args'] = tagArgs
+
+        outTokenList.append (tagToken)
+
+      # append the text as it is
+      else:
+        outTokenList.append (token)
+
+    # restore escaped braces back (once code is parsed)
+    restoredTokenList = []
+    for token in outTokenList:
+      if isinstance (token, basestring):
+        token = token.replace (unique+'_OPEN_BRACE', '[').replace (unique+'_CLOSE_BRACE', ']')
+      restoredTokenList.append (token)
+
+    return restoredTokenList
+

fanficdownloader/bbcodeutils/readme.txt

@@ -0,0 +1,81 @@
+AUTHOR
+  Pau Sanchez
+  http://www.codigomanso.com/
+
+VERSION:
+  bbcodeutils v1.0
+  
+LICENSE
+  This code is licensed under Creative Commons Attribution 3.0
+  http://creativecommons.org/licenses/by/3.0/
+
+  You can use this python module or any part of the code you want as long as you add
+  my name as a contributor to your project.
+  
+DESCRIPTION
+  This module can be used to produce HTML from BBCode, to generate BBCode or to fix invalid BBCode.
+
+  The classes are:
+    - bbcodeparser
+    - bbcodebuilder
+    - bbcode2html
+  
+  You can use bbcodeparser to parse BBCode and to produce output in any format you want.
+
+  Open the python file to find more information and examples of use of each class. It can
+  be a good idea to check the test.py for examples
+
+  To run the unit tests:
+    > python test.py
+  
+  To run the performance test:
+    > python test.py BBCodeTests.performanceTest  
+
+
+EXAMPLES OF BBCode:
+
+  [b] -> bold
+  [u] -> underline
+  [i] -> italic
+
+  [center] -> center the text inside
+  [color=XXX] -> change color of text
+  [size=XXX] -> change size of text
+
+  Lists:
+  [ul] -> unordered list
+  [ol] -> ordered list
+  [li] -> list item
+
+  [list] -> start unordered list
+  [*]    -> list item
+  [list=1] -> start a list of numbers
+  [list=a] -> start a list of alphabetic characters
+
+  Advanced:
+  [url] -> link to url
+  [url=http://link/url/]text[/url]  
+  [url link=http://link/url/ title="This is the title"]text[/url]  
+
+  [img]http://to/image[/img]
+  [img=230x330]http://to/image[/img]
+  [img="Alt text here"]http://to/image[/img]
+  [img="Alt text here" width=320 height=240]http://to/image[/img]
+
+  [email]asdf@asdf.com[/email]
+  [email=john@asdf.com]John Smith[/email]
+
+  [google]search this[/google]
+  [wikipedia]Tom Hanks[/wikipedia]
+  [wikipedia lang=es]Tom Hanks[/wikipedia]
+
+  Tables:
+  [table]
+  [tr]
+  [th]
+  [td]
+
+  Advanced:
+  [google]
+  [wikipedia]
+

fanficdownloader/bbcodeutils/test.py

@@ -0,0 +1,420 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+#
+# Author:        Pau Sanchez (contact@pausanchez.com)
+# Version:       v1.0
+# Last Modified: 2010/09/15
+# 
+# For the latest version check out:
+#   http://www.codigomanso.com/en/projects
+# 
+# My blog:
+#   http://www.codigomanso.com/en/  - English Version
+#   http://www.codigomanso.com/es/  - Spanish Version
+#
+
+from bbcodeparser import bbcodeparser
+from bbcodebuilder import bbcodebuilder
+
+import random
+import unittest
+
+class BBCodeTests(unittest.TestCase):
+  def setUp (self):
+    self.bbcode = bbcodeparser()
+    return
+
+  def testConstructor (self):
+    self.assertEqual (bbcodeparser ('whatever').html(), 'whatever')
+    self.assertEqual (bbcodeparser ('[b]bold[/b]').html(), '<b>bold</b>')
+    self.assertEqual (str (bbcodeparser ('[b]bold[/b]')), '[b]bold[/b]')
+    return
+
+  def testBold (self):
+    self.assertEqual (self.bbcode.parse ('whatever').html(), 'whatever')
+    self.assertEqual (self.bbcode.parse ('[b]bold[/b]').html(), '<b>bold</b>')
+    self.assertEqual (self.bbcode.parse ('[B]bold[/b]').html(), '<b>bold</b>')
+    self.assertEqual (self.bbcode.parse ('this is [B]bold[/B]').html(), 'this is <b>bold</b>')
+    return
+
+  def testItalic (self):
+    self.assertEqual (self.bbcode.parse ('[i]italic[/i]').html(), '<i>italic</i>')
+    return
+
+  def testUnderline (self):
+    self.assertEqual (self.bbcode.parse ('[u]italic[/u]').html(), '<u>italic</u>')
+    return
+
+  def testURLs (self):
+    self.assertEqual (
+      self.bbcode.parse ('[url]http://www.google.com[/url]').html(),
+      '<a href="http://www.google.com">http://www.google.com</a>'
+    )
+    self.assertEqual (
+      self.bbcode.parse ('[url=http://www.google.com]Google[/url]').html(),
+      '<a href="http://www.google.com">Google</a>'
+    )
+    self.assertEqual (
+      self.bbcode.parse ('[url="http://www.google.com"]Google[/url]').html(),
+      '<a href="http://www.google.com">Google</a>'
+    )
+    self.assertEqual (
+      self.bbcode.parse ('[url="http://www.google.com" title="Search Engine"]Google[/url]').html(),
+      '<a href="http://www.google.com" title="Search Engine">Google</a>'
+    )
+    self.assertEqual (
+      self.bbcode.parse ('[url link="http://www.google.com"]Google[/url]').html(),
+      '<a href="http://www.google.com">Google</a>'
+    )
+    return
+
+  def testPTag (self):
+    self.assertEqual (
+      self.bbcode.parse ('[p color=#0000ff]blue[/p]').html(),
+      u'<p style="color: #0000ff;">blue</p>'
+    )
+    self.assertEqual (
+      self.bbcode.parse ('[p size=12]12pt font[/p]').html(),
+      u'<p style="font-size: 12pt;">12pt font</p>'
+    )
+
+    self.assertEqual (
+      self.bbcode.parse ('[p font=arial]arial[/p]').html(),
+      u'<p style="font-family: arial;">arial</p>'
+    )
+
+    self.assertEqual (
+      self.bbcode.parse ('[p font=arial color=blue size=14]blue 14pt arial').html(),
+      u'<p style="color: blue; font-size: 14pt; font-family: arial;">blue 14pt arial</p>' 
+    )
+
+    self.assertEqual (
+      self.bbcode.parse ('[p class=whatever]text[/p]').html(),
+      u'<p>text</p>' 
+    )
+    return
+
+  def testColorTag (self):
+    self.assertEqual (
+      self.bbcode.parse ('[color=#0000ff]blue[/color]').html(),
+      u'<span style="color: #0000ff;">blue</span>'
+    )
+    return
+
+  def testSizeTag (self):
+    self.assertEqual (
+      self.bbcode.parse ('[size=12]12pt font[/size]').html(),
+      u'<span style="font-size: 12pt;">12pt font</span>'
+    )
+    return
+
+  def testEmail(self):
+    self.assertEqual (
+      self.bbcode.parse ('[email]asdf@asdf.com[/email]').html(),
+      u'<a href="mailto:asdf@asdf.com">asdf@asdf.com</a>'
+    )
+
+    self.assertEqual (
+      self.bbcode.parse ('[email=john@smith.com]John Smith[/email]').html(),
+      u'<a href="mailto:john@smith.com">John Smith</a>'
+    )
+    return
+
+  def testImgTag (self):
+    self.assertEqual (
+      self.bbcode.parse ('[img]http://www.codigomanso.com/image.jpg[/img]').html(),
+      u'<img src="http://www.codigomanso.com/image.jpg" />'
+    )
+
+    self.assertEqual (
+      self.bbcode.parse ('[img="This is the ALT of the image"]http://www.codigomanso.com/image.jpg[/img]').html(),
+      u'<img alt="This is the ALT of the image" src="http://www.codigomanso.com/image.jpg" />'
+    )
+
+    self.assertEqual (
+      self.bbcode.parse ('[img=320x200]http://www.codigomanso.com/image.jpg[/img]').html(),
+      u'<img height="200" src="http://www.codigomanso.com/image.jpg" width="320" />'
+    )
+
+    self.assertEqual (
+      self.bbcode.parse ('[img=320x200 title="Image Test"]http://www.codigomanso.com/image.jpg[/img]').html(),
+      u'<img height="200" src="http://www.codigomanso.com/image.jpg" title="Image Test" width="320" />'
+    )
+
+    self.assertEqual (
+      self.bbcode.parse ('[img="whatever" width=320 height="212" title="Image Test"]http://www.codigomanso.com/image.jpg[/img]').html(),
+      u'<img alt="whatever" height="212" src="http://www.codigomanso.com/image.jpg" title="Image Test" width="320" />'
+    )
+    return
+
+  def testGoogleURL (self):
+    self.assertEqual (
+      self.bbcode.parse ('[google]asdf[/google]').html(),
+      u'<a href="http://www.google.com/search?q=asdf">asdf</a>'
+    )
+    self.assertEqual (
+      self.bbcode.parse ('[google]Tom Hanks[/google]').html(),
+      u'<a href="http://www.google.com/search?q=Tom+Hanks">Tom Hanks</a>'
+    )
+    return
+    
+  def testWikipediaURL (self):
+    self.assertEqual (
+      self.bbcode.parse ('[wikipedia]Tom Hanks[/wikipedia]').html(),
+      u'<a href="http://www.wikipedia.org/wiki/Tom_Hanks">Tom Hanks</a>'
+    )
+
+    self.assertEqual (
+      self.bbcode.parse ('[wikipedia language=en]Tom Hanks[/wikipedia]').html(),
+      u'<a href="http://en.wikipedia.org/wiki/Tom_Hanks">Tom Hanks</a>'
+    )
+    
+    self.assertEqual (
+      self.bbcode.parse ('[wikipedia lang=es]Tom Hanks[/wikipedia]').html(),
+      u'<a href="http://es.wikipedia.org/wiki/Tom_Hanks">Tom Hanks</a>'
+    )
+    
+    self.assertEqual (
+      self.bbcode.parse ('[wikipedia=es]Tom Hanks[/wikipedia]').html(),
+      u'<a href="http://es.wikipedia.org/wiki/Tom_Hanks">Tom Hanks</a>'
+    )
+    return
+    
+  def testListTags (self):
+    self.assertEqual (
+      self.bbcode.parse ('[ul][li]item 1[/li][li]item 2[/li][/ul]').html(),
+      u'<ul><li>item 1</li><li>item 2</li></ul>'
+    )
+    
+    self.assertEqual (
+      self.bbcode.parse ('[ol][li]item 1[/li][li]item 2[/li][/ol]').html(),
+      u'<ol><li>item 1</li><li>item 2</li></ol>'
+    )
+
+    self.assertEqual (
+      self.bbcode.parse ('[list][li]item 1[/li][li]item 2[/li][/list]').html(),
+      u'<ul><li>item 1</li><li>item 2</li></ul>'
+    )
+
+    self.assertEqual (
+      self.bbcode.parse ('[list][*]item 1[*]item 2[/list]').html(),
+      u'<ul><li>item 1</li><li>item 2</li></ul>'
+    )
+
+    self.assertEqual (
+      self.bbcode.parse ('[list=1][li]item 1[/li][li]item 2[/li][/list]').html(),
+      u'<ol type="1"><li>item 1</li><li>item 2</li></ol>'
+    )
+    return
+
+  def testInvalidCode (self):
+    self.assertEqual (self.bbcode.parse ('[invalid]valid text[/invalid]').html(), 'valid text')
+    self.assertEqual (
+      self.bbcode.parse ('[b]bold and [i]italics[/b]').html(),
+      '<b>bold and <i>italics</i></b>'
+    )
+    self.assertEqual (
+      self.bbcode.parse ('[/b]invalid[/b][/p]').html(),
+      'invalid'
+    )
+    self.assertEqual (
+      self.bbcode.parse ('[p][b]bold').html(),
+      '<p><b>bold</b></p>'
+    )
+    self.assertEqual (
+      self.bbcode.parse ('[p][b]a <b>').html(),
+      '<p><b>a &lt;b&gt;</b></p>'
+    )
+
+    self.assertEqual (
+      self.bbcode.parse ('[ol][li]item 1[li]item 2[/li][/ol]').html(),
+      u'<ol><li>item 1</li><li>item 2</li></ol>'
+    )
+
+    self.assertEqual (
+      self.bbcode.parse ('[b]\[b\] stands for [b]bold[/b]').html(),
+      u'<b>[b] stands for </b><b>bold</b>'
+    )
+    return
+
+  def testEscapedBrackets (self):
+    self.assertEqual (
+      self.bbcode.parse ('\[b\]not bold\[/b\]').html(),
+      u'[b]not bold[/b]'
+    )
+
+    self.assertEqual (
+      self.bbcode.parse ('[b]\[b\] stands for bold[/b]').html(),
+      u'<b>[b] stands for bold</b>'
+    )
+
+    self.assertEqual (
+      self.bbcode.parse ('\[b\][b]stands for bold[/b]').html(),
+      u'[b]<b>stands for bold</b>'
+    )
+
+    self.assertEqual (
+      self.bbcode.parse ('\[b\][b]stands for bold[/b] just like <b> in HTML').html(),
+      u'[b]<b>stands for bold</b> just like &lt;b&gt; in HTML'
+    )
+
+  def testBigExample (self):
+    inputText = """check this out
+
+    [h1 class=circle]heading[/h1]
+
+    [p size=14 color=blue font="verdana, Times New Roman"]This is [b] bold [/b] and this [i]italic[/i] and this is [color=red]red[/color] and this is [color="red"]also red[/color].
+    [/p]
+
+    fix [b][i]bold [font=verdana][size=12]and[/size][/font] italic[/b]
+    [img]http://www.codigomanso.com/b.jpg[/img]
+    [url]http://www.codigomanso.com/[/url]
+    [url=http://www.codigomanso.com/]Codigo Manso[/url]
+    [uRl link=http://www.codigomanso.com title="Codigo Manso Blog"]Codigo Manso[/url]
+
+    [ul]
+     [Li]item 1[/Li]
+     [li]item 2[/LI]
+    [/UL]
+
+    [list=1 ]
+     [*]item 1
+     [*]item 2
+    [/list]
+
+    [table class="big"]
+      [tr]
+        [th]big[/th]
+      [/tr]
+    [/table]
+    [invalid class="extra"]whatever[/invalid]"""
+
+    out = self.bbcode.parse (inputText).html(allowClassAttr = True)
+    self.assertEquals (out, '''check this out
+
+    <h1 class="circle">heading</h1>
+
+    <p style="color: blue; font-size: 14pt; font-family: verdana, Times New Roman;">This is <b> bold </b> and this <i>italic</i> and this is <span style="color: red;">red</span> and this is <span style="color: red;">also red</span>.
+    </p>
+
+    fix <b><i>bold <span style="font-family: verdana;"><span style="font-size: 12pt;">and</span></span> italic</i></b>
+    <img src="http://www.codigomanso.com/b.jpg" />
+    <a href="http://www.codigomanso.com/">http://www.codigomanso.com/</a>
+    <a href="http://www.codigomanso.com/">Codigo Manso</a>
+    <a href="http://www.codigomanso.com" title="Codigo Manso Blog">Codigo Manso</a>
+
+    <ul>
+     <li>item 1</li>
+     <li>item 2</li>
+    </ul>
+
+    <ol type="1">
+     <li>item 1
+     </li><li>item 2
+    </li></ol>
+
+    <table class="big">
+      <tr>
+        <th>big</th>
+      </tr>
+    </table>
+    whatever''')
+
+
+  def testBBCodeDumper (self):
+    self.assertEquals (
+      self.bbcode.parse ('[b]bold[/b]').bbcode(),
+      '[b]bold[/b]'
+    )
+
+    self.assertEquals (
+      self.bbcode.parse ('[color=red]text in red[/color]').bbcode(),
+      '[color=red]text in red[/color]'
+    )
+    self.assertEquals (
+      self.bbcode.parse ('[p][color=red]text in red').bbcode(),
+      '[p][color=red]text in red[/color][/p]'
+    )
+
+    self.assertEquals (
+      self.bbcode.parse ('This [b][i]code[/b] will be fixed[/invalid]').bbcode(),
+      'This [b][i]code[/i][/b] will be fixed'
+    )
+
+    self.assertEquals (
+      self.bbcode.parse ('\[[url]http://www.codigomanso.com/en[/url]\]').bbcode(),
+      "\[[url]http://www.codigomanso.com/en[/url]\]"
+    )
+
+  def performanceTest(self):
+    '''
+    This test checks the performance of parse and html operations
+
+    To run this test type:
+      > python test.py BBCodeTests.performanceTest
+    '''
+    inputText = """check this out
+
+    [h1 class=circle]heading[/h1]
+
+    [p size=14 color=blue font="verdana, Times New Roman"]This is [b] bold [/b] and this [i]italic[/i] and this is [color=red]red[/color] and this is [color="red"]also red[/color].
+    [/p]
+
+    fix [b][i]bold [font=verdana][size=12]and[/size][/font] italic[/b]
+    [img]http://www.codigomanso.com/b.jpg[/img]
+    [url]http://www.codigomanso.com/[/url]
+    [url=http://www.codigomanso.com/]Codigo Manso[/url]
+    [uRl link=http://www.codigomanso.com title="Codigo Manso Blog"]Codigo Manso[/url]
+
+    [ul]
+     [Li]item 1[/Li]
+     [li]item 2[/LI]
+    [/UL]
+
+    [list=1 ]
+     [*]item 1
+     [*]item 2
+    [/list]
+
+    [table class="big"]
+      [tr]
+        [th]big[/th]
+      [/tr]
+    [/table]
+    [invalid class="extra"]whatever[/invalid]"""
+
+    import time
+    start = time.time()
+
+    for i in range(0, 12):
+      inputText += inputText
+
+    print "len(inputText) = %.2f MB  (took %.2f seconds)" % (len(inputText)/(1024.0*1024.0), time.time() - start)
+    
+    bbcode = bbcodeparser()
+    start = time.time()
+    bbcode.parse (inputText)
+    total = (time.time() - start)
+    print "time (bbcode.parse()) = %f" % total
+    print "  >> %.2f chars/second" % (len(inputText) / total)
+
+    start = time.time()
+    bbcode.html(doDeepCopy = False)
+    total = (time.time() - start)
+    print "time (bbcode.html()) = %f" % total
+    print "  >> %.2f chars/second" % (len(inputText) / total)
+    return
+    
+  def testCodeBuilder (self):
+    bbcode = bbcodebuilder ()
+    self.assertEquals (bbcode.b ('bold'), u'[b]bold[/b]')
+    self.assertEquals (bbcode.color ('this goes in red', 'red'), u'[color=red]this goes in red[/color]')
+    self.assertEquals (bbcode.url ('Google', 'http://www.google.com'), u'[url=http://www.google.com]Google[/url]')
+    self.assertEquals (bbcode.alist('item 1', 'item 2'), u"[list=a]\n  [*]item 1\n  [*]item 2\n[/list]")
+    return
+
+if __name__ == '__main__':
+    unittest.main()
+
+
+

fanficdownloader/epubutils.py

@@ -42,6 +42,7 @@ def get_update_data(inputio,
             
     oldcover = None
     calibrebookmark = None
+    logfile = None
     # Looking for pre-existing cover.
     for item in contentdom.getElementsByTagName("reference"):
         if item.getAttribute("type") == "cover":
@@ -96,6 +97,11 @@ def get_update_data(inputio,
             if( item.getAttribute("media-type") == "application/xhtml+xml" ):
                 href=relpath+item.getAttribute("href")
                 #print("---- item href:%s path part: %s"%(href,get_path_part(href)))
+                if re.match(r'.*/log_page\.x?html',href):
+                    try:
+                        logfile = epub.read(href).decode("utf-8")
+                    except:
+                        pass # corner case I bumped into while testing.
                 if re.match(r'.*/(file|chapter)\d+\.x?html',href):
                     if getsoups:
                         soup = bs.BeautifulSoup(epub.read(href).decode("utf-8"))
@@ -136,7 +142,7 @@ def get_update_data(inputio,
                     
     for k in images.keys():
         print("\tlongdesc:%s\n\tData len:%s\n"%(k,len(images[k])))
-    return (source,filecount,soups,images,oldcover,calibrebookmark)
+    return (source,filecount,soups,images,oldcover,calibrebookmark,logfile)
 
 def get_path_part(n):
     relpath = os.path.dirname(n)

fanficdownloader/geturls.py

@@ -25,23 +25,51 @@ from gziphttp import GZipProcessor
 
 import adapters
 
-def get_urls_from_page(url):
-    
-    opener = u2.build_opener(u2.HTTPCookieProcessor(),GZipProcessor())
-    soup = BeautifulSoup(opener.open(url).read())
-    
+def get_urls_from_page(url,config=None):
+
     normalized = set() # normalized url
     retlist = [] # orig urls.
-    config = ConfigParser.SafeConfigParser()
+    if not config:
+        config = ConfigParser.SafeConfigParser()
+
+    data = None
+    
+    # special stuff to log into archiveofourown.org, if possible.
+    # Unlike most that show the links to 'adult' stories, but protect
+    # them, AO3 doesn't even show them if not logged in.  Only works
+    # with saved user/pass--not going to prompt for list.
+    if 'archiveofourown.org' in url:
+        ao3adapter = adapters.getAdapter(config,"http://www.archiveofourown.org/works/0","EPUB")
+        if ao3adapter.getConfig("username"):
+            if ao3adapter.getConfig("is_adult"):
+                addurl = "?view_adult=true"
+            else:
+                addurl=""
+            # just to get an authenticity_token.
+            data = ao3adapter._fetchUrl(url+addurl)
+            # login the session.
+            ao3adapter.performLogin(url,data)
+            # get the list page with logged in session.
+            data = ao3adapter._fetchUrl(url)
+
+    if not data:
+        opener = u2.build_opener(u2.HTTPCookieProcessor(),GZipProcessor())
+        data = opener.open(url).read()
+    
+    soup = BeautifulSoup(data)
     
     for a in soup.findAll('a'):
         if a.has_key('href'):
             href = form_url(url,a['href'])
-            # lots of eFiction sites use similar 'are you old enough' javascript links.
-            if 'javascript' in a['href'] and 'viewstory.php' in a['href']:
-                m = re.search(r"'(?P<sid>(view)?story\.php\?(sid|psid|no|story|stid)=\d+)",a['href'])
+            # this (should) catch normal story links, some javascript
+            # 'are you old enough' links, and 'Report This' links.
+            # The 'normalized' set prevents duplicates.
+            if 'story.php' in a['href']:
+                #print("trying:%s"%a['href'])
+                m = re.search(r"(?P<sid>(view)?story\.php\?(sid|psid|no|story|stid)=\d+)",a['href'])
                 if m != None:
                     href = form_url(url,m.group('sid'))
+                    
             try:
                 href = href.replace('&index=1','')
                 adapter = adapters.getAdapter(config,href,"EPUB")

fanficdownloader/story.py

@@ -201,6 +201,7 @@ class Story:
         self.cover=None # *href* of new cover image--need to create html.
         self.oldcover=None # (oldcoverhtmlhref,oldcoverhtmltype,oldcoverhtmldata,oldcoverimghref,oldcoverimgtype,oldcoverimgdata)
         self.calibrebookmark=None # cheesy way to carry calibre bookmark file forward across update.
+        self.logfile=None # cheesy way to carry log file forward across update.
 
     def setMetadata(self, key, value):
         ## still keeps < < and &
@@ -223,13 +224,15 @@ class Story:
         
     def getMetadata(self, key, removeallentities=False, doreplacements=True):
         value = None
-        if self.getLists().has_key(key):
-            value = ', '.join(self.getList(key))
-        if self.metadata.has_key(key):
+        if self.isList(key):
+            value = ', '.join(self.getList(key, removeallentities))
+        elif self.metadata.has_key(key):
             value = self.metadata[key]
             if value:
                 if key == "numWords":
                     value = commaGroups(value)
+                if key == "numChapters":
+                    value = commaGroups("%d"%value)
                 if key == "dateCreated":
                     value = value.strftime("%Y-%m-%d %H:%M:%S")
                 if key == "datePublished" or key == "dateUpdated":
@@ -247,46 +250,94 @@ class Story:
         All single value *and* list value metadata as strings.
         '''
         allmetadata = {}
+        
+        # special handling for authors/authorUrls
+        authlinkhtml="<a class='authorlink' href='%s'>%s</a>"
+        if 'author' in self.listables.keys(): # more than one author, assume multiple authorUrl too.
+            htmllist=[]
+            for i, v in enumerate(self.listables['author']):
+                aurl = self.listables['authorUrl'][i]                    
+                auth = v
+                # make sure doreplacements & removeallentities are honored.
+                if doreplacements:
+                    aurl=self.doReplacments(aurl)
+                    auth=self.doReplacments(auth)
+                if removeallentities:
+                    aurl=removeAllEntities(aurl)
+                    auth=removeAllEntities(auth)
+                
+                htmllist.append(authlinkhtml%(aurl,auth))
+            self.setMetadata('authorHTML',', '.join(htmllist))
+        else:
+           self.setMetadata('authorHTML',authlinkhtml%(self.getMetadata('authorUrl', removeallentities, doreplacements),
+                                                       self.getMetadata('author', removeallentities, doreplacements)))
+            
         for k in self.metadata.keys():
             allmetadata[k] = self.getMetadata(k, removeallentities, doreplacements)
         for l in self.listables.keys():
             allmetadata[l] = self.getMetadata(l, removeallentities, doreplacements)
-
+                
         return allmetadata
 
     # just for less clutter in adapters.
     def extendList(self,listname,l):
         for v in l:
-            self.addToList(listname,v)
+            self.addToList(listname,v.strip())
     
     def addToList(self,listname,value):
         if value==None:
             return
         value = conditionalRemoveEntities(value)
-        if not self.listables.has_key(listname):
+        if not self.isList(listname):
             self.listables[listname]=[]
         # prevent duplicates.
         if not value in self.listables[listname]:
             self.listables[listname].append(value)
 
-    def getList(self,listname):
-        if not self.listables.has_key(listname):
-            return []
-        return filter( lambda x : x!=None and x!='' ,
-                       map(self.doReplacments,self.listables[listname]) )
+    def getList(self,listname, removeallentities=False, doreplacements=True):
+        retlist = []
+        if not self.isList(listname):
+            retlist = [self.getMetadata(listname,removeallentities=removeallentities)]
+        else:
+            retlist = self.listables[listname]
 
-    def getLists(self):
+        if doreplacements:
+            retlist = filter( lambda x : x!=None and x!='' ,
+                              map(self.doReplacments,retlist) )
+        if removeallentities:
+            retlist = filter( lambda x : x!=None and x!='' ,
+                              map(removeAllEntities,retlist) )
+
+        return retlist
+
+    def getLists(self, removeallentities=False):
         lsts = {}
         for ln in self.listables.keys():
-            lsts[ln] = self.getList(ln)
+            lsts[ln] = self.getList(ln, removeallentities)
         return lsts
+
+    def isList(self,listname):
+        return self.listables.has_key(listname)
     
-    def addChapter(self, title, html):
+    def addChapter(self, title, html, configurable=None):
+        if configurable and \
+                configurable.getConfig('strip_chapter_numbers') and \
+                configurable.getConfig('chapter_title_strip_pattern'):
+            title = re.sub(configurable.getConfig('chapter_title_strip_pattern'),"",title)
         self.chapters.append( (title,html) )
 
-    def getChapters(self):
+    def getChapters(self, configurable=None):
         "Chapters will be tuples of (title,html)"
-        return self.chapters
+        retval = []
+        if configurable and \
+                configurable.getConfig('add_chapter_numbers') and \
+                configurable.getConfig('chapter_title_add_pattern'):
+            for index, (title,html) in enumerate(self.chapters):
+                retval.append( (string.Template(configurable.getConfig('chapter_title_add_pattern')).substitute({'index':index+1,'title':title}),html) ) 
+        else:
+            retval = self.chapters
+            
+        return retval
 
     def formatFileName(self,template,allowunsafefilename=True):
         values = origvalues = self.getAllMetadata()

fanficdownloader/writers/base_writer.py

@@ -50,6 +50,7 @@ class BaseStoryWriter(Configurable):
             'genre',
             'language',
             'characters',
+            'ships',
             'series',
             'status',
             'datePublished',
@@ -81,6 +82,7 @@ class BaseStoryWriter(Configurable):
             'status':'Status',
             'series':'Series',
             'characters':'Characters',
+            'ships':'Relationships',
             'datePublished':'Published',
             'dateUpdated':'Updated',
             'dateCreated':'Packaged',
@@ -104,12 +106,19 @@ class BaseStoryWriter(Configurable):
             }
         self.story.setMetadata('formatname',self.getFormatName())
         self.story.setMetadata('formatext',self.getFormatExt())
-        
-        for tag in self.getConfigList("extratags"):
-            self.story.addToList("extratags",tag)
 
-    def getMetadata(self,key):
-        return stripHTML(self.story.getMetadata(key))
+        ## Look for config parameter, split and add each to metadata field.
+        for (config,metadata) in [("extratags","extratags"),
+                                  ("extracategories","category"),
+                                  ("extragenres","genre"),
+                                  ("extracharacters","characters"),
+                                  ("extraships","ships"),
+                                  ("extrawarnings","warnings")]:
+            for val in self.getConfigList(config):
+                self.story.addToList(metadata,val)
+
+    def getMetadata(self,key, removeallentities=False):
+        return stripHTML(self.story.getMetadata(key, removeallentities))
 
     def getOutputFileName(self):
         if self.getConfig('zip_output'):
@@ -134,7 +143,7 @@ class BaseStoryWriter(Configurable):
         names as Story.metadata, but ENTRY should use label and value.
         """
         if self.getConfig("include_titlepage"):
-            self._write(out,START.substitute(self.story.metadata))
+            self._write(out,START.substitute(self.story.getAllMetadata()))
 
             if WIDE_ENTRY==None:
                 WIDE_ENTRY=ENTRY
@@ -166,7 +175,7 @@ class BaseStoryWriter(Configurable):
                 else:
                     self._write(out, entry)
 
-            self._write(out,END.substitute(self.story.metadata))
+            self._write(out,END.substitute(self.story.getAllMetadata()))
 
     def writeTOCPage(self, out, START, ENTRY, END):
         """
@@ -175,14 +184,14 @@ class BaseStoryWriter(Configurable):
         names as Story.metadata, but ENTRY should use index and chapter.
         """
         # Only do TOC if there's more than one chapter and it's configured.
-        if len(self.story.getChapters()) > 1 and self.getConfig("include_tocpage") and not self.metaonly :
-            self._write(out,START.substitute(self.story.metadata))
+        if len(self.story.getChapters(self)) > 1 and self.getConfig("include_tocpage") and not self.metaonly :
+            self._write(out,START.substitute(self.story.getAllMetadata()))
 
-            for index, (title,html) in enumerate(self.story.getChapters()):
+            for index, (title,html) in enumerate(self.story.getChapters(self)):
                 if html:
                     self._write(out,ENTRY.substitute({'chapter':title, 'index':"%04d"%(index+1)}))
 
-            self._write(out,END.substitute(self.story.metadata))
+            self._write(out,END.substitute(self.story.getAllMetadata()))
 
     # if no outstream is given, write to file.
     def writeStory(self,outstream=None, metaonly=False, outfilename=None, forceOverwrite=False):
@@ -252,7 +261,7 @@ class BaseStoryWriter(Configurable):
         if close:
             outstream.close()
 
-    def getTags(self):
+    def getTags(self, removeallentities=False):
         # set to avoid duplicates subject tags.
         subjectset = set()
         
@@ -265,10 +274,10 @@ class BaseStoryWriter(Configurable):
             if entry in self.getConfigList("include_subject_tags") and \
                     entry not in self.story.getLists() and \
                     self.story.getMetadata(entry):
-                subjectset.add(self.getMetadata(entry))
+                subjectset.add(self.getMetadata(entry, removeallentities))
                 
         # listables all go into dc:subject tags, but only if they are configured.
-        for (name,lst) in self.story.getLists().iteritems():
+        for (name,lst) in self.story.getLists(removeallentities).iteritems():
             if name in self.getConfigList("include_subject_tags"):
                 for tag in lst:
                     subjectset.add(tag)

fanficdownloader/writers/writer_epub.py

@@ -52,7 +52,7 @@ class EpubWriter(BaseStoryWriter):
 <link href="stylesheet.css" type="text/css" charset="UTF-8" rel="stylesheet"/>
 </head>
 <body>
-<h3><a href="${storyUrl}">${title}</a> by <a href="${authorUrl}">${author}</a></h3>
+<h3><a href="${storyUrl}">${title}</a> by ${authorHTML}</h3>
 <div>
 ''')
 
@@ -79,7 +79,7 @@ ${value}<br />
 <link href="stylesheet.css" type="text/css" charset="UTF-8" rel="stylesheet"/>
 </head>
 <body>
-<h3><a href="${storyUrl}">${title}</a> by <a href="${authorUrl}">${author}</a></h3>
+<h3><a href="${storyUrl}">${title}</a> by ${authorHTML}</h3>
 <table class="full">
 ''')
 
@@ -140,6 +140,101 @@ ${value}<br />
 </html>
 ''')
 
+        self.EPUB_LOG_PAGE_START = string.Template('''<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head>
+<title>Update Log</title>
+<link href="stylesheet.css" type="text/css" charset="UTF-8" rel="stylesheet"/>
+</head>
+<body>
+<h3>Update Log</h3>
+''')
+
+        self.EPUB_LOG_ENTRY = string.Template('''
+<b>${label}:</b> <span id="${id}">${value}</span>
+''')
+                          
+        self.EPUB_LOG_PAGE_END = string.Template('''
+</body>
+</html>
+''')
+
+    def writeLogPage(self, out):
+        """
+       XXX 
+
+        
+        Write the log page, but only include entries that there's
+        metadata for.  START, ENTRY and END are expected to already by
+        string.Template().  START and END are expected to use the same
+        names as Story.metadata, but ENTRY should use id, label and value.
+        """
+        if self.getConfig("include_logpage"):
+
+            # if there's a self.story.logfile, there's an existing log
+            # to add to.
+            if self.story.logfile:
+                print("existing logfile found, appending")
+                print("existing data:%s"%self._getLastLogData(self.story.logfile))
+                replace_string = "</body>" # "</h3>"
+                self._write(out,self.story.logfile.replace(replace_string,self._makeLogEntry(self._getLastLogData(self.story.logfile))+replace_string))
+            else:
+                # otherwise, write a new one.
+                self._write(out,self.EPUB_LOG_PAGE_START.substitute(self.story.getAllMetadata()))
+                self._write(out,self._makeLogEntry())
+                self._write(out,self.EPUB_LOG_PAGE_END.substitute(self.story.getAllMetadata()))
+
+    # self parsing instead of Soup because it should be simple and not
+    # worth the overhead.
+    def _getLastLogData(self,logfile):
+        """
+        Make a dict() of the most recent(last) log entry for each piece of metadata.
+        Switch rindex to index to search from top instead of bottom.
+        """
+        values = {}
+        for entry in self.getConfigList("logpage_entries"):
+            if entry in self.validEntries:
+                try:
+                    # <span id="dateUpdated">1975-04-15</span>
+                    span = '<span id="%s">'%entry
+                    idx = logfile.rindex(span)+len(span)
+                    values[entry] = logfile[idx:logfile.index('</span>',idx)]
+                except Exception, e:
+                    #print("e:%s"%e)
+                    pass
+
+        return values
+        
+    def _makeLogEntry(self, oldvalues={}):
+        retval = "<p class='log_entry'>"
+
+        for entry in self.getConfigList("logpage_entries"):
+            if entry in self.validEntries:
+                val = self.story.getMetadata(entry)
+                if val and ( entry not in oldvalues or val != oldvalues[entry] ):
+                    if self.hasConfig(entry+"_label"):
+                        label=self.getConfig(entry+"_label")
+                    else:
+                        print("Using fallback label for %s_label"%entry)
+                        label=self.titleLabels[entry]
+
+                    retval = retval + self.EPUB_LOG_ENTRY.substitute({'id':entry,
+                                                                      'label':label,
+                                                                      'value':val})
+            else:
+                # could be useful for introducing extra text, but
+                # mostly it makes it easy to tell when you get the
+                # keyword wrong.
+                retval = retval + entry
+                
+        retval = retval + "</p><hr />"
+        
+        if self.getConfig('replace_hr'):
+            retval = retval.replace("<hr />","<div class='center'>* * *</div>")
+            
+        return retval
+                
     def writeStoryImpl(self, out):
 
         ## Python 2.5 ZipFile is rather more primative than later
@@ -185,7 +280,7 @@ ${value}<br />
 
         uniqueid= 'fanficdownloader-uid:%s-u%s-s%s' % (
             self.getMetadata('site'),
-            self.getMetadata('authorId'),
+            self.story.getList('authorId')[0],
             self.getMetadata('storyId'))
         
         contentdom = getDOMImplementation().createDocument(None, "package", None)
@@ -206,9 +301,15 @@ ${value}<br />
             metadata.appendChild(newTag(contentdom,"dc:title",text=self.getMetadata('title')))
 
         if self.getMetadata('author'):
-            metadata.appendChild(newTag(contentdom,"dc:creator",
-                                        attrs={"opf:role":"aut"},
-                                        text=self.getMetadata('author')))
+            if self.story.isList('author'):
+                for auth in self.story.getList('author'):
+                    metadata.appendChild(newTag(contentdom,"dc:creator",
+                                                attrs={"opf:role":"aut"},
+                                                text=auth))
+            else:
+                metadata.appendChild(newTag(contentdom,"dc:creator",
+                                            attrs={"opf:role":"aut"},
+                                            text=self.getMetadata('author')))
 
         metadata.appendChild(newTag(contentdom,"dc:contributor",text="fanficdownloader [http://fanficdownloader.googlecode.com]",attrs={"opf:role":"bkp"}))
         metadata.appendChild(newTag(contentdom,"dc:rights",text=""))
@@ -340,10 +441,15 @@ div { margin: 0pt; padding: 0pt; }
         if self.getConfig("include_titlepage"):
             items.append(("title_page","OEBPS/title_page.xhtml","application/xhtml+xml","Title Page"))
             itemrefs.append("title_page")
-        if len(self.story.getChapters()) > 1 and self.getConfig("include_tocpage") and not self.metaonly :
+        if len(self.story.getChapters(self)) > 1 and self.getConfig("include_tocpage") and not self.metaonly :
             items.append(("toc_page","OEBPS/toc_page.xhtml","application/xhtml+xml","Table of Contents"))
             itemrefs.append("toc_page")
-        for index, (title,html) in enumerate(self.story.getChapters()):
+
+        if self.getConfig("include_logpage"):
+            items.append(("log_page","OEBPS/log_page.xhtml","application/xhtml+xml","Update Log"))
+            itemrefs.append("log_page")
+            
+        for index, (title,html) in enumerate(self.story.getChapters(self)):
             if html:
                 i=index+1
                 items.append(("file%04d"%i,
@@ -433,7 +539,7 @@ div { margin: 0pt; padding: 0pt; }
         del tocncxdom
 
         # write stylesheet.css file.
-        outputepub.writestr("OEBPS/stylesheet.css",self.EPUB_CSS.substitute(self.story.metadata)) 
+        outputepub.writestr("OEBPS/stylesheet.css",self.EPUB_CSS.substitute(self.story.getAllMetadata())) 
 
         # write title page.
         if self.getConfig("titlepage_use_table"):
@@ -474,7 +580,14 @@ div { margin: 0pt; padding: 0pt; }
             outputepub.writestr("OEBPS/toc_page.xhtml",tocpageIO.getvalue())
         tocpageIO.close()
 
-        for index, (title,html) in enumerate(self.story.getChapters()):
+        # write log page.
+        logpageIO = StringIO.StringIO()
+        self.writeLogPage(logpageIO)
+        if logpageIO.getvalue(): # will be false if no log page.
+            outputepub.writestr("OEBPS/log_page.xhtml",logpageIO.getvalue())
+        logpageIO.close()
+        
+        for index, (title,html) in enumerate(self.story.getChapters(self)):
             if html:
                 logging.debug('Writing chapter text for: %s' % title)
                 fullhtml = self.EPUB_CHAPTER_START.substitute({'chapter':title, 'index':index+1}) + html + self.EPUB_CHAPTER_END.substitute({'chapter':title, 'index':index+1})

fanficdownloader/writers/writer_html.py

@@ -43,7 +43,7 @@ ${output_css}
 </style>
 </head>
 <body>
-<h1><a href="${storyUrl}">${title}</a> by <a href="${authorUrl}">${author}</a></h1>
+<h1><a href="${storyUrl}">${title}</a> by ${authorHTML}</h1>
 ''')
 
         self.HTML_TITLE_PAGE_START = string.Template('''
@@ -82,7 +82,7 @@ ${output_css}
 
     def writeStoryImpl(self, out):
 
-        self._write(out,self.HTML_FILE_START.substitute(self.story.metadata))
+        self._write(out,self.HTML_FILE_START.substitute(self.story.getAllMetadata()))
 
         self.writeTitlePage(out,
                             self.HTML_TITLE_PAGE_START,
@@ -94,10 +94,10 @@ ${output_css}
                           self.HTML_TOC_ENTRY,
                           self.HTML_TOC_PAGE_END)
 
-        for index, (title,html) in enumerate(self.story.getChapters()):
+        for index, (title,html) in enumerate(self.story.getChapters(self)):
             if html:
                 logging.debug('Writing chapter text for: %s' % title)
                 self._write(out,self.HTML_CHAPTER_START.substitute({'chapter':title, 'index':"%04d"%(index+1)}))
                 self._write(out,html)
 
-        self._write(out,self.HTML_FILE_END.substitute(self.story.metadata))
+        self._write(out,self.HTML_FILE_END.substitute(self.story.getAllMetadata()))

fanficdownloader/writers/writer_mobi.py

@@ -43,7 +43,7 @@ class MobiWriter(BaseStoryWriter):
 <title>${title} by ${author}</title>
 </head>
 <body>
-<h3><a href="${storyUrl}">${title}</a> by <a href="${authorUrl}">${author}</a></h3>
+<h3><a href="${storyUrl}">${title}</a> by ${authorHTML}</h3>
 <div>
 ''')
 
@@ -69,7 +69,7 @@ ${value}<br />
 <title>${title} by ${author}</title>
 </head>
 <body>
-<h3><a href="${storyUrl}">${title}</a> by <a href="${authorUrl}">${author}</a></h3>
+<h3><a href="${storyUrl}">${title}</a> by ${authorHTML}</h3>
 <table class="full">
 ''')
 
@@ -169,7 +169,7 @@ ${value}<br />
         #     files.append(tocpageIO.getvalue())
         # tocpageIO.close()
 
-        for index, (title,html) in enumerate(self.story.getChapters()):
+        for index, (title,html) in enumerate(self.story.getChapters(self)):
             if html:
                 logging.debug('Writing chapter text for: %s' % title)
                 fullhtml = self.MOBI_CHAPTER_START.substitute({'chapter':title, 'index':index+1}) + html + self.MOBI_CHAPTER_END.substitute({'chapter':title, 'index':index+1})

fanficdownloader/writers/writer_txt.py

@@ -114,7 +114,7 @@ End file.
         
         wrapout = KludgeStringIO()
         
-        wrapout.write(self.TEXT_FILE_START.substitute(self.story.metadata))
+        wrapout.write(self.TEXT_FILE_START.substitute(self.story.getAllMetadata()))
 
         self.writeTitlePage(wrapout,
                             self.TEXT_TITLE_PAGE_START,
@@ -133,13 +133,13 @@ End file.
         
         self._write(out,self.lineends(self.wraplines(towrap)))
 
-        for index, (title,html) in enumerate(self.story.getChapters()):
+        for index, (title,html) in enumerate(self.story.getChapters(self)):
             if html:
                 logging.debug('Writing chapter text for: %s' % title)
                 self._write(out,self.lineends(self.wraplines(removeAllEntities(self.TEXT_CHAPTER_START.substitute({'chapter':title, 'index':index+1})))))
                 self._write(out,self.lineends(html2text(html,wrap_width=self.wrap_width)))
 
-        self._write(out,self.lineends(self.wraplines(self.TEXT_FILE_END.substitute(self.story.metadata))))
+        self._write(out,self.lineends(self.wraplines(self.TEXT_FILE_END.substitute(self.story.getAllMetadata()))))
 
     def wraplines(self, text):
         

index.html

@@ -54,27 +54,17 @@
 	      much easier. </p>
 	  </div>
 	  <!-- put announcements here, h3 is a good title size. -->
-	  <h3>New Sites</h3>
+	  <h3>New Site / Features / Fixes</h3>
 	  <p>
-	    
-	    New sites ncisfic.com and national-library.net added.
-	    <br />Thanks, Ida!
-	    
-	  </p>
-	  <h3>New Fixes</h3>
-	  <p>
-
-	    Fixes to support www.fanfiktion.de and archiveofourown.org username and password.
-	    
+	    New site merlinfic.dtwins.co.uk, thanks Ida.
 	  </p>
 	  <p>
-
-	    <b>Note:</b> As part of the archiveofourown.org fix, the
-	    configuration section for AO3 is now
-	    [archiveofourown.org], not [www.archiveofourown.org].
-	    Please change your personal configuration accordingly, if
-	    you have customized your settings for archiveofourown.org.
-	    
+	    New feature options strip_chapter_numbers and add_chapter_numbers can be turned on to
+	    normalize numbers in front of chapter titles.
+	  </p>
+	  <p>
+	    Fixed BBCode in fimfiction.net story summaries.  Also improved handling of html
+	    in story summaries.
 	  </p>
 	  <p>
 	    Questions?  Check out our 
@@ -84,7 +74,7 @@
 	    If you have any problems with this application, please
 	    report them in
 	    the <a href="http://groups.google.com/group/fanfic-downloader">FanFictionDownLoader Google Group</a>.  The
-	    <a href="http://4-4-17.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
+	    <a href="http://4-4-24.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
 	  </p>
 	  <div id='error'>
 	    {{ error_message }}
@@ -465,6 +455,92 @@
 	    Use the URL of the story's first chapter, such as
 	    <br /><a href="http://national-library.net/viewstory.php?storyid=1234">http://national-library.net/viewstory.php?storyid=1234</a>
 	  </dd>
+	  <dt>dark-solace.org</dt>
+	  <dd>
+	    Use the URL of the story's first chapter, such as
+	    <br /><a href="http://dark-solace.org/elysian/viewstory.php?sid=1234">http://dark-solace.org/elysian/viewstory.php?sid=1234</a>
+	  </dd>
+	  <dt>pretendercentre.com</dt>
+	  <dd>
+	    Use the URL of the story's first chapter, such as
+	    <br /><a href="http://pretendercentre.com/missingpieces/viewstory.php?sid=1234">http://pretendercentre.com/missingpieces/viewstory.php?sid=1234</a>
+	  </dd>
+	  <dt>themasque.net</dt>
+	  <dd>
+	    Use the URL of the story's first chapter, such as
+	    <br /><a href="http://themasque.net/wiktt/efiction/viewstory.php?sid=1234">http://themasque.net/wiktt/efiction/viewstory.php?sid=1234</a>
+	  </dd>
+	  <dt>finestories.com</dt>
+	  <dd>
+	    Use the URL of the story's first chapter, such as
+	    <br /><a href="http://finestories.com/s/1234">http://finestories.com/s/1234</a>
+	  </dd>
+	  <dt>www.hpfanficarchive.com</dt>
+	  <dd>
+	    Use the URL of the story's first chapter, such as
+	    <br /><a href="http://www.hpfanficarchive.com/stories/viewstory.php?sid=1234">http://www.hpfanficarchive.com/stories/viewstory.php?sid=1234</a>
+	  </dd>
+	  <dt>svufiction.com</dt>
+	  <dd>
+	    Use the URL of the story's first chapter, such as
+	    <br /><a href="http://svufiction.com/viewstory.php?sid=1234">http://svufiction.com/viewstory.php?sid=1234</a>
+	  </dd>
+	  <dt>www.twilightarchives.com</dt>
+	  <dd>
+	    Use the URL of the story's first chapter, such as
+	    <br /><a href="http://www.twilightarchives.com/read/1234">http://www.twilightarchives.com/read/1234</a>
+	  </dd>
+	  <dt>www.wizardtales.net</dt>
+	  <dd>
+	    Use the URL of the story's first chapter, such as
+	    <br /><a href="http://www.wizardtales.net/viewstory.php?sid=1234">http://www.wizardtales.net/viewstory.php?sid=1234</a>
+	  </dd>
+	  <dt>grangerenchanted.com</dt>
+	  <dd>
+	    Use the URL of the story's chapter list, such as
+	    <br /><a href="http://grangerenchanted.com/enchant/viewstory.php?sid=1234">http://grangerenchanted.com/enchant/viewstory.php?sid=1234</a> or
+	    <br /><a href="http://malfoymanor.grangerenchanted.com/enchant/viewstory.php?sid=1234">http://grangerenchanted.com/enchant/viewstory.php?sid=1234</a>
+	  </dd>
+	  <dt>hlfiction.net</dt>
+	  <dd>
+	    Use the URL of the story's first chapter, such as
+	    <br /><a href="http://hlfiction.net/viewstory.php?sid=1234">http://hlfiction.net/viewstory.php?sid=1234</a>
+	  </dd>
+	  <dt>nha.magical-worlds.us</dt>
+	  <dd>
+	    Use the URL of the story's chapter list, such as
+	    <br /><a href="http://nha.magical-worlds.us/viewstory.php?sid=1234">http://nha.magical-worlds.us/viewstory.php?sid=1234</a>
+	  </dd>
+	  <dt>www.dracoandginny.com</dt>
+	  <dd>
+	    Use the URL of the story's chapter list, such as
+	    <br /><a href="http://www.dracoandginny.com/viewstory.php?sid=1234">http://www.dracoandginny.com/viewstory.php?sid=1234</a>
+	  </dd>
+	  <dt>www.scarvesandcoffee.net</dt>
+	  <dd>
+	    Use the URL of the story's chapter list, such as
+	    <br /><a href="http://www.scarvesandcoffee.net/viewstory.php?sid=1234">http://www.scarvesandcoffee.net/viewstory.php?sid=1234</a>
+	  </dd>
+	  <dt>www.thepetulantpoetess.com</dt>
+	  <dd>
+	    Use the URL of the story's chapter list, such as
+	    <br /><a href="http://www.thepetulantpoetess.com/viewstory.php?sid=1234">http://www.thepetulantpoetess.com/viewstory.php?sid=1234</a>
+	  </dd>
+	  <dt>www.wolverineandrogue.com</dt>
+	  <dd>
+	    Use the URL of the story's chapter list, such as
+	    <br /><a href="http://www.wolverineandrogue.com/wrfa/viewstory.php?sid=1234">http://www.wolverineandrogue.com/wrfa/viewstory.php?sid=1234</a>
+	  </dd>
+	  <dt>www.sinful-desire.org</dt>
+	  <dd>
+	    Use the URL of the story's chapter list, such as
+	    <br /><a href="http://www.sinful-desire.org/archive/viewstory.php?sid=1234">http://www.sinful-desire.org/archive/viewstory.php?sid=1234</a>
+	  </dd>
+	  <dt>merlinfic.dtwins.co.uk</dt>
+	  <dd>
+	    Use the URL of the story's chapter list, such as
+	    <br /><a href="http://merlinfic.dtwins.co.uk/viewstory.php?sid=1234">http://merlinfic.dtwins.co.uk/viewstory.php?sid=1234</a>
+	  </dd>
 	</dl>
 	<p>
 	  A few additional things to know, which will make your life substantially easier:

plugin-defaults.ini

@@ -19,8 +19,7 @@
 ## overridden at several levels
 
 ## Some sites also require the user to confirm they are adult for
-## adult content.  In commandline version, this should go in your
-## personal.ini, not defaults.ini.
+## adult content.  Uncomment by removing '#' in front of is_adult.
 #is_adult:true
 
 ## All available titlepage_entries and the label used for them:
@@ -43,6 +42,7 @@ category_label:Category
 genre_label:Genre
 language_label:Language
 characters_label:Characters
+ships_label:Relationships
 series_label:Series
 ## Completed/In-Progress
 status_label:Status
@@ -75,7 +75,7 @@ version_label:FFDL Version
 ## You can include extra text or HTML that will be included as-is in
 ## the title page. Eg: titlepage_entries: ...,<br />,summary,<br />,...
 ## All current formats already include title and author.
-titlepage_entries: series,category,genre,language,characters,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,description
+titlepage_entries: series,category,genre,language,characters,ships,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,description
 
 ## Try to collect series name and number of this story in series.
 ## Some sites (ab)use 'series' for reading lists and personal
@@ -101,11 +101,20 @@ include_tocpage: true
 
 ## entries to make epub subjects and calibre tags
 ## lastupdate creates two tags: "Last Update Year/Month: %Y/%m" and "Last Update: %Y/%m/%d"
-include_subject_tags: extratags, genre, category, characters, status
+include_subject_tags: extratags, genre, category, characters, ships, status
 
 ## extra tags (comma separated) to include, primarily for epub.
 extratags: FanFiction
 
+## extra categories, genres, characters, ships and warnings can be
+## configured.  Used primarily for sites that are dedicated to a genre
+## or 'ship and so don't included it for every story.
+#extracategories:
+#extragenres:
+#extracharacters:
+#extraships:
+#extrawarnings:
+
 ## number of seconds to sleep between calls to the story site.  May by
 ## useful if pulling large numbers of stories or if the site is slow.
 #slow_down_sleep_time:0.5
@@ -143,6 +152,45 @@ extratags: FanFiction
 # ${category} => Buffy:? [tT]he Vampire Slayer => BuffyCover
 # ${category} => Star Trek => StarTrekCover
 
+## If set false, the summary will have all html stripped.
+## Both this and include_images must be true to get images in the
+## summary.
+keep_summary_html:true
+
+## Don't like the numbers at the start of chapter titles on some
+## sites?  You can use strip_chapter_numbers to strip them off.  Just
+## want to make them all look the same?  Strip them off, then add them
+## back on with add_chapter_numbers.  Don't like the way it strips
+## numbers or adds them back?  See chapter_title_strip_pattern and
+## chapter_title_add_pattern.
+strip_chapter_numbers:false
+add_chapter_numbers:false
+
+## (Two versions of chapter_title_strip_pattern are shown below.  You
+## should only have one uncommented.)
+## This version will remove the leading number from:
+## "1." => ""
+## "1. The Beginning" => "The Beginning"
+## "1: Start" => "Start"
+## "2, Chapter the second" => "Chapter the second"
+## etc
+chapter_title_strip_pattern:^[0-9]+[\.: -]+
+
+## This version will strip all of the above *plus* remove 'Chapter 1':
+## "Chapter 1" => ""
+## "1. Chapter 1" => ""
+## "1. Chapter 1, Bob's First Clue" => "Bob's First Clue"
+## "Chapter 2 - Pirates Place" => "Pirates Place"
+## etc
+#chapter_title_strip_pattern:^([0-9]+[\.: -]+)?(Chapter *[0-9]+[\.:, -]*)?
+
+## Uses a python template substitution.  The ${index} is the 'chapter'
+## number and ${title} is the chapter title, after applying
+## chapter_title_strip_pattern.  Those are the only variables available.
+## "The Beginning" => "1. The Beginning" 
+chapter_title_add_pattern:${index}. ${title}
+
+
 ## Each output format has a section that overrides [defaults]
 [html]
 
@@ -183,9 +231,24 @@ windows_eol: true
 [epub]
 
 ## epub carries the TOC in metadata.
-## mobi generated from epub will have a TOC at the end.
+## mobi generated from epub by calibre will have a TOC at the end.
 include_tocpage: false
 
+## include a Update Log page before the story text.  If included, the
+## log will be updated each time the epub is an all the metadata
+## fields that have changed since the last update (typically
+## dateUpdated,numChapters,numWords at a minimum) will be shown.
+## Great for tracking when chapters came out and when the description,
+## etc changed.
+include_logpage: false
+
+## items to include in the log page Empty metadata entries, or those
+## that haven't changed since the last update, will *not* appear, even
+## if in the list.  You can include extra text or HTML that will be
+## included as-is in each log entry. Eg: logpage_entries: ...,<br />,
+## summary,<br />,...
+logpage_entries: dateCreated,datePublished,dateUpdated,numChapters,numWords,status,title,author,description,category,genre,rating,warnings
+
 ## epub->mobi conversions typically don't like tables.
 titlepage_use_table: false
 
@@ -200,10 +263,12 @@ background_color: ffffff
 ## Allow customization of CSS.  Make sure to keep at least one space
 ## at the start of each line and to escape % to %%.  Also need
 ## background_color to be in the same section, if included in CSS.
+## 'adobe-text-layout: optimizeSpeed;' prevents hyphenation on newer Nooks
 output_css: 
  body { background-color: #%(background_color)s;
         text-align: justify;
-        margin: 2%%; }
+        margin: 2%%;
+	adobe-text-layout: optimizeSpeed; }
  pre { font-size: x-small; }
  sml { font-size: small; }
  h1 { text-align: center; }
@@ -230,11 +295,6 @@ output_css:
 ## stories.  Images will be converted to jpg for size if possible.
 #include_images:false
 
-## If not set, the summary will have all html stripped for safety.
-## Both this and include_images must be true to get images in the
-## summary.
-#keep_summary_html:false
-
 ## If set, the first image found will be made the cover image.  If
 ## keep_summary_html is true, any images in summary will be before any
 ## in chapters.
@@ -287,6 +347,11 @@ nook_img_fix:true
 ## URLs like: http://test1.com?sid=12345
 [test1.com]
 extratags: FanFiction,Testing
+extracategories:Fafner
+extragenres:Romance,Fluff
+extracharacters:Reginald Smythe-Smythe,Mokona
+extraships:Smythe-Smythe/Mokona
+extrawarnings:Extreme Bogosity
 
 ## If necessary, you can define [<site>:<format>] sections to
 ## customize the formats differently for the same site.  Overrides
@@ -313,6 +378,11 @@ extratags: FanFiction,Testing,HTML
 #is_adult:true
 
 [ashwinder.sycophanthex.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+extracharacters:Severus Snape,Hermione Granger
+extraships:Severus Snape/Hermione Granger
+
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -321,6 +391,9 @@ extratags: FanFiction,Testing,HTML
 #password:yourpassword
 
 [castlefans.org]
+## Site dedicated to these categories/characters/ships
+extracategories:Castle
+
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -343,7 +416,25 @@ extratags: FanFiction,Testing,HTML
 ## cover image.  This lets you exclude them.
 cover_exclusion_regexp:/images/.*?ribbon.gif
 
+[dark-solace.org]
+## Site dedicated to these categories/characters/ships
+extracategories:Buffy: The Vampire Slayer
+extracharacters:Buffy, Spike
+extraships:Spike/Buffy
+
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
 [dramione.org]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+extracharacters:Draco Malfoy,Hermione Granger
+extraships:Draco Malfoy/Hermione Granger
+
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -357,6 +448,9 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 #is_adult:true
 
 [erosnsappho.sycophanthex.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
@@ -367,6 +461,9 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 cover_exclusion_regexp:/images/.*?ribbon.gif
 
 [fanfiction.mugglenet.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -380,6 +477,10 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 #is_adult:true
 
 [fanfiction.portkey.org]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+extraships:Harry Potter/Hermione Granger
+
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
@@ -393,23 +494,93 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 #username:YourName
 #password:yourpassword
 
+[finestories.com]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+[grangerenchanted.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+extracharacters:Hermione Granger
+
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
+[hlfiction.net]
+## Site dedicated to these categories/characters/ships
+extracategories:Highlander
+
+[ksarchive.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Star Trek
+extracharacters:Kirk,Spock
+extraships:Kirk/Spock
+
 [lumos.sycophanthex.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+[merlinfic.dtwins.co.uk]
+## Site dedicated to these categories/characters/ships
+extracategories:Merlin
+
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
 [national-library.net]
+## Site dedicated to these categories/characters/ships
+extracategories:West Wing
 
 [ncisfic.com]
+## Site dedicated to these categories/characters/ships
+extracategories:NCIS
 
 [nfacommunity.com]
+## Site dedicated to these categories/characters/ships
+extracategories:NCIS
+
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+[nha.magical-worlds.us]
+## Site dedicated to these categories/characters/ships
+extracategories:Buffy: The Vampire Slayer
+extracharacters:Willow
+
 [occlumency.sycophanthex.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+extracharacters:Severus Snape
+
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -418,6 +589,9 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 #password:yourpassword
 
 [onedirectionfanfiction.com]
+## Site dedicated to these categories/characters/ships
+extracategories:One Direction
+
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -431,12 +605,28 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 #is_adult:true
 
 [ponyfictionarchive.net]
+## Site dedicated to these categories/characters/ships
+extracategories:My Little Pony: Friendship is Magic
+
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+[pretendercentre.com]
+## Site dedicated to these categories/characters/ships
+extracategories:The Pretender
+
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
 [samdean.archive.nu]
+## Site dedicated to these categories/characters/ships
+extracategories:Supernatural
+extracharacters:Sam,Dean
+extraships:Sam/Dean
 
 [sg1-heliopolis.com]
 ## Some sites do not require a login, but do require the user to
@@ -445,10 +635,45 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 #is_adult:true
 
 [stargate-atlantis.org]
+## Site dedicated to these categories/characters/ships
+extracategories:Stargate: Atlantis
+
+[svufiction.com]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
 
 [thehexfiles.net]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+extracharacters:Draco Malfoy,Harry Potter
+extraships:Harry Potter/Draco Malfoy
+
+[themasque.net]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
 
 [thequidditchpitch.org]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -462,14 +687,40 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 #is_adult:true
 
 [www.adastrafanfic.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Star Trek
+
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+[www.dracoandginny.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+extracharacters:Draco Malfoy,Ginny Weasley
+extraships:Draco Malfoy/Ginny Weasley
+
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
 [www.thealphagate.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Stargate: SG-1
 
 [www.checkmated.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -484,6 +735,10 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 #is_adult:true
 
 [www.dokuga.com]
+## Site dedicated to these categories/characters/ships
+extracategories:InuYasha
+extracharacters:Sesshoumaru,Kagome
+extraships:Sesshoumaru/Kagome
 
 [www.fanfiction.net]
 ## fanfiction.net's 'cover' images are really just tiny thumbnails.
@@ -501,6 +756,9 @@ never_make_cover: true
 [www.ficbook.net]
 
 [www.fictionalley.org]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
@@ -519,6 +777,9 @@ extratags:
 #password:yourpassword
 
 [www.fimfiction.net]
+## Site dedicated to these categories/characters/ships
+extracategories:My Little Pony: Friendship is Magic
+
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
@@ -530,18 +791,33 @@ extratags:
 #fail_on_password: false
 
 [www.harrypotterfanfiction.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
 [www.hpfandom.net]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+[www.hpfanficarchive.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
 [www.ik-eternal.net]
+## Site dedicated to these categories/characters/ships
+extracategories:InuYasha
+extracharacters:InuYasha,Kagome
+extraships:InuYasha/Kagome
+
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -555,6 +831,9 @@ extratags:
 #is_adult:true
 
 [www.libraryofmoria.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Lord of the Rings
+
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
@@ -563,6 +842,9 @@ extratags:
 [www.mediaminer.org]
 
 [www.midnightwhispers.ca]
+## Site dedicated to these categories/characters/ships
+extracategories:Queer as Folk
+
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
@@ -573,8 +855,14 @@ extratags:
 cover_exclusion_regexp:/stories/999/images/.*?_trophy.png
 
 [www.ncisfiction.com]
+## Site dedicated to these categories/characters/ships
+extracategories:NCIS
 
 [www.phoenixsong.net]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+extraships:Harry Potter/Ginny Weasley
+
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -583,16 +871,43 @@ cover_exclusion_regexp:/stories/999/images/.*?_trophy.png
 #password:yourpassword
 
 [www.potionsandsnitches.net]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
 
 [www.prisonbreakfic.net]
+## Site dedicated to these categories/characters/ships
+extracategories:Prison Break
+
+[www.scarvesandcoffee.net]
+## Site dedicated to these categories/characters/ships
+extracategories:Glee
+extracharacters:Kurt Hummel,Blaine Anderson
+
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+[www.sinful-desire.org]
+## Site dedicated to these categories/characters/ships
+extracategories:Supernatural
+
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
 
 [www.siye.co.uk]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+extracharacters:Harry Potter,Ginny Weasley
+extraships:Harry Potter/Ginny Weasley
 
 [www.squidge.org/peja]
 # www.squidge.org/peja calls it Fandom <shrug>
 category_label:Fandom
 # Remove numWords -- www.squidge.org/peja word counts are inaccurate
-titlepage_entries: series,category,genre,language,characters,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,site,description
+titlepage_entries: series,category,genre,language,characters,ships,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,site,description
 
 [www.squidge.org/peja:txt]
 ## Add URLs since there aren't links.
@@ -600,6 +915,16 @@ titlepage_entries: series,category,genre,language,characters,status,datePublishe
 titlepage_entries: series,category,genre,language,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,site,storyUrl, authorUrl, description
 
 [www.storiesofarda.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Lord of the Rings
+
+[www.thepetulantpoetess.com]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
 
 [www.thewriterscoffeeshop.com]
 ## Some sites require login (or login for some rated stories) The
@@ -630,7 +955,14 @@ collect_series: false
 #username:YourName
 #password:yourpassword
 
+[www.twilightarchives.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Twilight
+
 [www.twilighted.net]
+## Site dedicated to these categories/characters/ships
+extracategories:Twilight
+
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -642,6 +974,9 @@ collect_series: false
 collect_series: false
 
 [www.twiwrite.net]
+## Site dedicated to these categories/characters/ships
+extracategories:Twilight
+
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -653,6 +988,11 @@ collect_series: false
 collect_series: false
 
 [www.walkingtheplank.org]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+extracharacters:Severus Snape,Harry Potter
+extraships:Severus Snape/Harry Potter
+
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
@@ -660,7 +1000,28 @@ collect_series: false
 
 [www.whofic.com]
 
+[www.wizardtales.net]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
+[www.wolverineandrogue.com]
+## Site dedicated to these categories/characters/ships
+extracategories:X-Men Movie
+extracharacters:Wolverine,Rogue
+
 [www.wraithbait.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Stargate: Atlantis
+
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.

plugin-example.ini

@@ -1,14 +1,25 @@
 ## This is an example of what your personal configuration might look
-## like.
+## like.  Uncomment options by removing the '#' in front of them.
 
 [defaults]
 ## Some sites also require the user to confirm they are adult for
-## adult content.  In commandline version, this should go in your
-## personal.ini, not defaults.ini.
+## adult content.  Uncomment by removing '#' in front of is_adult.
 #is_adult:true
 
-## include images from img tags in the body and summary of
-## stories.  Images will be converted to jpg for size if possible.
+## Don't like the numbers at the start of chapter titles on some
+## sites?  You can use strip_chapter_numbers to strip them off.  Just
+## want to make them all look the same?  Strip them off, then add them
+## back on with add_chapter_numbers.  Don't like the way it strips
+## numbers or adds them back?  See chapter_title_strip_pattern and
+## chapter_title_add_pattern.
+#strip_chapter_numbers:true
+#add_chapter_numbers:true
+
+[epub]
+## include images from img tags in the body and summary of stories.
+## Images will be converted to jpg for size if possible.  Images work
+## in epub format only.  To get mobi or other format with images,
+## download as epub and use Calibre to convert.
 #include_images:true
 
 ## If not set, the summary will have all html stripped for safety.
@@ -29,18 +40,10 @@
 ## space.
 #grayscale_images: false
 
-## Try to collect series name and number of this story in series.
-## Some sites (ab)use 'series' for reading lists and personal
-## collections.  This lets us turn it on and off by site without
-## keeping a lengthy titlepage_entries per site and prevents it
-## updating in the plugin.
-## Turn off in [defaults] or [overrides] to prevent all sites from
-## updating series column.
-## default is true
-#collect_series: false
 
 ## Most common, I expect will be using this to save username/passwords
-## for different sites.
+## for different sites.  Here are a few examples.
+
 [www.twilighted.net]
 #username:YourPenname
 #password:YourPassword

readme.txt

@@ -1,3 +1,21 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Other code contributed by Pau Sanchez(bbcodeutils).
+
 To use, do:
 
 python downloader.py [-f (epub|html|txt)] <url>