Change FFDL to store settings in library db, not json file.

svufiction.com, twilightarchives.com, and wizardtales.net

app.yaml

@@ -1,6 +1,6 @@
 # ffd-retief-hrd fanfictiondownloader
 application: fanfictiondownloader
-version: 4-4-16
+version: 4-4-21
 runtime: python27
 api_version: 1
 threadsafe: true

calibre-plugin/__init__.py

@@ -27,8 +27,8 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
     description         = 'UI plugin to download FanFiction stories from various sites.'
     supported_platforms = ['windows', 'osx', 'linux']
     author              = 'Jim Miller'
-    version             = (1, 5, 33)
-    minimum_calibre_version = (0, 8, 30)
+    version             = (1, 6, 0)
+    minimum_calibre_version = (0, 8, 57)
 
     #: This field defines the GUI plugin class that contains all the code
     #: that actually does something. Its format is module_path:class_name

calibre-plugin/common_utils.py

@@ -11,9 +11,11 @@ import os
 from PyQt4 import QtGui
 from PyQt4.Qt import (Qt, QIcon, QPixmap, QLabel, QDialog, QHBoxLayout,
                       QTableWidgetItem, QFont, QLineEdit, QComboBox,
-                      QVBoxLayout, QDialogButtonBox, QStyledItemDelegate, QDateTime)
+                      QVBoxLayout, QDialogButtonBox, QStyledItemDelegate, QDateTime,
+                      QTextEdit,
+                      QListWidget, QAbstractItemView)
 from calibre.constants import iswindows
-from calibre.gui2 import gprefs, error_dialog, UNDEFINED_QDATETIME
+from calibre.gui2 import gprefs, error_dialog, UNDEFINED_QDATETIME, info_dialog
 from calibre.gui2.actions import menu_action_unique_name
 from calibre.gui2.keyboard import ShortcutConfig
 from calibre.utils.config import config_dir
@@ -445,3 +447,97 @@ class DateDelegate(QStyledItemDelegate):
             model.setData(index, UNDEFINED_QDATETIME, Qt.EditRole)
         else:
             model.setData(index, QDateTime(val), Qt.EditRole)
+            
+class PrefsViewerDialog(SizePersistedDialog):
+
+    def __init__(self, gui, namespace):
+        SizePersistedDialog.__init__(self, gui, 'Prefs Viewer dialog')
+        self.setWindowTitle('Preferences for: '+namespace)
+
+        self.gui = gui
+        self.db = gui.current_db
+        self.namespace = namespace
+        self._init_controls()
+        self.resize_dialog()
+
+        self._populate_settings()
+
+        if self.keys_list.count():
+            self.keys_list.setCurrentRow(0)
+
+    def _init_controls(self):
+        layout = QVBoxLayout(self)
+        self.setLayout(layout)
+
+        ml = QHBoxLayout()
+        layout.addLayout(ml, 1)
+
+        self.keys_list = QListWidget(self)
+        self.keys_list.setSelectionMode(QAbstractItemView.SingleSelection)
+        self.keys_list.setFixedWidth(150)
+        self.keys_list.setAlternatingRowColors(True)
+        ml.addWidget(self.keys_list)
+        self.value_text = QTextEdit(self)
+        self.value_text.setTabStopWidth(24)
+        self.value_text.setReadOnly(True)
+        ml.addWidget(self.value_text, 1)
+
+        button_box = QDialogButtonBox(QDialogButtonBox.Ok)
+        button_box.accepted.connect(self.accept)
+        self.clear_button = button_box.addButton('Clear', QDialogButtonBox.ResetRole)
+        self.clear_button.setIcon(get_icon('trash.png'))
+        self.clear_button.setToolTip('Clear all settings for this plugin')
+        self.clear_button.clicked.connect(self._clear_settings)
+        layout.addWidget(button_box)
+
+    def _populate_settings(self):
+        self.keys_list.clear()
+        ns_prefix = self._get_ns_prefix()
+        keys = sorted([k[len(ns_prefix):] for k in self.db.prefs.iterkeys()
+                       if k.startswith(ns_prefix)])
+        for key in keys:
+            self.keys_list.addItem(key)
+        self.keys_list.setMinimumWidth(self.keys_list.sizeHintForColumn(0))
+        self.keys_list.currentRowChanged[int].connect(self._current_row_changed)
+
+    def _current_row_changed(self, new_row):
+        if new_row < 0:
+            self.value_text.clear()
+            return
+        key = unicode(self.keys_list.currentItem().text())
+        val = self.db.prefs.get_namespaced(self.namespace, key, '')
+        self.value_text.setPlainText(self.db.prefs.to_raw(val))
+        
+    def _get_ns_prefix(self):
+        return 'namespaced:%s:'% self.namespace
+        
+    def _clear_settings(self):
+        from calibre.gui2.dialogs.confirm_delete import confirm
+        message = '<p>Are you sure you want to clear your settings in this library for this plugin?</p>' \
+                  '<p>Any settings in other libraries or stored in a JSON file in your calibre plugins ' \
+                  'folder will not be touched.</p>' \
+                  '<p>You must restart calibre afterwards.</p>'
+        if not confirm(message, self.namespace+'_clear_settings', self):
+            return
+        ns_prefix = self._get_ns_prefix()
+        keys = [k for k in self.db.prefs.iterkeys() if k.startswith(ns_prefix)]
+        for k in keys:
+            del self.db.prefs[k]
+        self._populate_settings()
+        d = info_dialog(self, 'Settings deleted', 
+                        '<p>All settings for this plugin in this library have been cleared.</p>'
+                        '<p>Please restart calibre now.</p>',
+                        show_copy_button=False)
+        b = d.bb.addButton(_('Restart calibre now'), d.bb.AcceptRole)
+        b.setIcon(QIcon(I('lt.png')))
+        d.do_restart = False
+        def rf():
+            d.do_restart = True
+        b.clicked.connect(rf)
+        d.set_details('')
+        d.exec_()
+        b.clicked.disconnect()
+        self.close()
+        if d.do_restart:
+            self.gui.quit(restart=True)
+        

calibre-plugin/config.py

@@ -4,7 +4,7 @@ from __future__ import (unicode_literals, division, absolute_import,
                         print_function)
 
 __license__   = 'GPL v3'
-__copyright__ = '2011, Jim Miller'
+__copyright__ = '2012, Jim Miller'
 __docformat__ = 'restructuredtext en'
 
 import traceback, copy
@@ -22,115 +22,118 @@ from calibre_plugins.fanfictiondownloader_plugin.dialogs \
 from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.adapters import getConfigSections
 
 from calibre_plugins.fanfictiondownloader_plugin.common_utils \
-    import ( get_library_uuid, KeyboardConfigDialog )
+    import ( get_library_uuid, KeyboardConfigDialog, PrefsViewerDialog )
 
 from calibre.gui2.complete import MultiCompleteLineEdit
 
-# This is where all preferences for this plugin will be stored
+PREFS_NAMESPACE = 'FanFictionDownLoaderPlugin'
+PREFS_KEY_SETTINGS = 'settings'
+
+# Set defaults used by all.  Library specific settings continue to
+# take from here.
+default_prefs = {}
+default_prefs['personal.ini'] = get_resources('plugin-example.ini')
+
+default_prefs['updatemeta'] = True
+default_prefs['updatecover'] = False
+default_prefs['updateepubcover'] = False
+default_prefs['keeptags'] = False
+default_prefs['urlsfromclip'] = True
+default_prefs['updatedefault'] = True
+default_prefs['fileform'] = 'epub'
+default_prefs['collision'] = OVERWRITE
+default_prefs['deleteotherforms'] = False
+default_prefs['adddialogstaysontop'] = False
+default_prefs['includeimages'] = False
+default_prefs['lookforurlinhtml'] = False
+default_prefs['injectseries'] = False
+
+default_prefs['send_lists'] = ''
+default_prefs['read_lists'] = ''
+default_prefs['addtolists'] = False
+default_prefs['addtoreadlists'] = False
+default_prefs['addtolistsonread'] = False
+
+default_prefs['gcnewonly'] = False
+default_prefs['gc_site_settings'] = {}
+default_prefs['allow_gc_from_ini'] = True
+
+default_prefs['countpagesstats'] = []
+
+default_prefs['errorcol'] = ''
+default_prefs['custom_cols'] = {}
+
+def set_library_config(library_config):
+    get_gui().current_db.prefs.set_namespaced(PREFS_NAMESPACE,
+                                              PREFS_KEY_SETTINGS,
+                                              library_config)
+    
+def get_library_config():
+    db = get_gui().current_db
+    library_id = get_library_uuid(db)
+    library_config = None
+    # Check whether this is a configuration needing to be migrated
+    # from json into database.  If so: get it, set it, rename it in json.
+    if library_id in old_prefs:
+        print("get prefs from old_prefs")
+        library_config = old_prefs[library_id]
+        set_library_config(library_config)
+        old_prefs["migrated to library db %s"%library_id] = old_prefs[library_id]
+        del old_prefs[library_id]
+
+    if library_config is None:
+        print("get prefs from db")
+        library_config = db.prefs.get_namespaced(PREFS_NAMESPACE, PREFS_KEY_SETTINGS,
+                                                 copy.deepcopy(default_prefs))
+    return library_config
+
+# This is where all preferences for this plugin *were* stored
 # Remember that this name (i.e. plugins/fanfictiondownloader_plugin) is also
 # in a global namespace, so make it as unique as possible.
 # You should always prefix your config file name with plugins/,
 # so as to ensure you dont accidentally clobber a calibre config file
-all_prefs = JSONConfig('plugins/fanfictiondownloader_plugin')
-
-# Set defaults used by all.  Library specific settings continue to
-# take from here.
-all_prefs.defaults['personal.ini'] = get_resources('plugin-example.ini')
-
-all_prefs.defaults['updatemeta'] = True
-all_prefs.defaults['updatecover'] = False
-all_prefs.defaults['updateepubcover'] = False
-all_prefs.defaults['keeptags'] = False
-all_prefs.defaults['urlsfromclip'] = True
-all_prefs.defaults['updatedefault'] = True
-all_prefs.defaults['fileform'] = 'epub'
-all_prefs.defaults['collision'] = OVERWRITE
-all_prefs.defaults['deleteotherforms'] = False
-all_prefs.defaults['adddialogstaysontop'] = False
-all_prefs.defaults['includeimages'] = False
-all_prefs.defaults['lookforurlinhtml'] = False
-all_prefs.defaults['injectseries'] = False
-
-all_prefs.defaults['send_lists'] = ''
-all_prefs.defaults['read_lists'] = ''
-all_prefs.defaults['addtolists'] = False
-all_prefs.defaults['addtoreadlists'] = False
-all_prefs.defaults['addtolistsonread'] = False
-
-all_prefs.defaults['gcnewonly'] = False
-all_prefs.defaults['gc_site_settings'] = {}
-all_prefs.defaults['allow_gc_from_ini'] = True
-
-all_prefs.defaults['custom_cols'] = {}
-
-# The list of settings to copy from all_prefs or the previous library
-# when config is called for the first time on a library.
-copylist = ['personal.ini',
-            'updatemeta',
-            'updatecover',
-            'updateepubcover',
-            'keeptags',
-            'urlsfromclip',
-            'updatedefault',
-            'fileform',
-            'collision',
-            'deleteotherforms',
-            'adddialogstaysontop',
-            'includeimages',
-            'lookforurlinhtml',
-            'injectseries',
-            'gcnewonly',
-            'gc_site_settings',
-            'allow_gc_from_ini']
+old_prefs = JSONConfig('plugins/fanfictiondownloader_plugin')
 
 # fake out so I don't have to change the prefs calls anywhere.  The
 # Java programmer in me is offended by op-overloading, but it's very
 # tidy.
 class PrefsFacade():
-    def __init__(self,all_prefs):
-        self.all_prefs = all_prefs
-        self.lastlibid = None
-
-    def _get_copylist_prefs(self,frompref):
-        return filter( lambda x : x[0] in copylist, frompref.items() )
+    def __init__(self,default_prefs):
+        self.default_prefs = default_prefs
+        self.libraryid = None
+        self.current_prefs = None
         
     def _get_prefs(self):
         libraryid = get_library_uuid(get_gui().current_db)
-        if libraryid not in self.all_prefs:
-            if self.lastlibid == None:
-                self.all_prefs[libraryid] = dict(self._get_copylist_prefs(self.all_prefs))
-            else:
-                self.all_prefs[libraryid] = dict(self._get_copylist_prefs(self.all_prefs[self.lastlibid]))
-                
-        self.lastlibid = libraryid
-            
-        return self.all_prefs[libraryid]
-
-    def _save_prefs(self,prefs):
-        libraryid = get_library_uuid(get_gui().current_db)
-        self.all_prefs[libraryid] = prefs
+        if self.current_prefs == None or self.libraryid != libraryid:
+            print("self.current_prefs == None(%s) or self.libraryid != libraryid(%s)"%(self.current_prefs == None,self.libraryid != libraryid))
+            self.libraryid = libraryid
+            self.current_prefs = get_library_config()
+        return self.current_prefs
         
     def __getitem__(self,k):            
         prefs = self._get_prefs()
         if k not in prefs:
-            # pulls from all_prefs.defaults automatically if not set
-            # in all_prefs
-            return self.all_prefs[k]
+            # pulls from default_prefs.defaults automatically if not set
+            # in default_prefs
+            return self.default_prefs[k]
         return prefs[k]
 
     def __setitem__(self,k,v):
         prefs = self._get_prefs()
         prefs[k]=v
-        self._save_prefs(prefs)
+        # self._save_prefs(prefs)
 
-    # to be avoided--can cause unexpected results as possibly ancient
-    # all_pref settings may be pulled.
     def __delitem__(self,k):
         prefs = self._get_prefs()
-        del prefs[k]
-        self._save_prefs(prefs)
+        if k in prefs:
+            del prefs[k]
 
-prefs = PrefsFacade(all_prefs)
+    def save_to_db(self):
+        set_library_config(self._get_prefs())
+        
+
+prefs = PrefsFacade(default_prefs)
     
 class ConfigWidget(QWidget):
 
@@ -164,7 +167,12 @@ class ConfigWidget(QWidget):
         if 'Generate Cover' not in plugin_action.gui.iactions:
             self.generatecover_tab.setEnabled(False)
 
-        self.columns_tab = ColumnsTab(self, plugin_action)
+        self.countpages_tab = CountPagesTab(self, plugin_action)
+        tab_widget.addTab(self.countpages_tab, 'Count Pages')
+        if 'Count Pages' not in plugin_action.gui.iactions:
+            self.countpages_tab.setEnabled(False)
+
+        self.columns_tab = CustomColumnsTab(self, plugin_action)
         tab_widget.addTab(self.columns_tab, 'Custom Columns')
 
         self.other_tab = OtherTab(self, plugin_action)
@@ -217,7 +225,27 @@ class ConfigWidget(QWidget):
         prefs['gc_site_settings'] = gc_site_settings
         prefs['allow_gc_from_ini'] = self.generatecover_tab.allow_gc_from_ini.isChecked()
 
+        # Count Pages tab
+        countpagesstats = []
+        
+        if self.countpages_tab.pagecount.isChecked():
+            countpagesstats.append('PageCount')
+        if self.countpages_tab.wordcount.isChecked():
+            countpagesstats.append('WordCount')
+        if self.countpages_tab.fleschreading.isChecked():
+            countpagesstats.append('FleschReading')
+        if self.countpages_tab.fleschgrade.isChecked():
+            countpagesstats.append('FleschGrade')
+        if self.countpages_tab.gunningfog.isChecked():
+            countpagesstats.append('GunningFog')
+            
+        prefs['countpagesstats'] = countpagesstats
+        
         # Custom Columns tab
+        # error column
+        prefs['errorcol'] = unicode(self.columns_tab.errorcol.itemData(self.columns_tab.errorcol.currentIndex()).toString())
+
+        # cust cols
         colsmap = {}
         for (col,combo) in self.columns_tab.custcol_dropdowns.iteritems():
             val = unicode(combo.itemData(combo.currentIndex()).toString())
@@ -226,6 +254,8 @@ class ConfigWidget(QWidget):
                 #print("colsmap[%s]:%s"%(col,colsmap[col]))
         prefs['custom_cols'] = colsmap
 
+        prefs.save_to_db()
+
     def edit_shortcuts(self):
         self.save_settings()
         # Force the menus to be rebuilt immediately, so we have all our actions registered
@@ -546,13 +576,61 @@ class GenerateCoverTab(QWidget):
         self.gcnewonly.setChecked(prefs['gcnewonly'])
         self.l.addWidget(self.gcnewonly)
 
-        self.allow_gc_from_ini = QCheckBox('Allow generate_cover_settings from personal.ini to override.',self)
+        self.allow_gc_from_ini = QCheckBox('Allow generate_cover_settings from personal.ini to override',self)
         self.allow_gc_from_ini.setToolTip("The personal.ini parameter generate_cover_settings allows you to choose a GC setting based on metadata rather than site, but it's much more complex.<br \>generate_cover_settings is ignored when this is off.")
         self.allow_gc_from_ini.setChecked(prefs['allow_gc_from_ini'])
         self.l.addWidget(self.allow_gc_from_ini)
             
         self.l.insertStretch(-1)
         
+class CountPagesTab(QWidget):
+
+    def __init__(self, parent_dialog, plugin_action):
+        self.parent_dialog = parent_dialog
+        self.plugin_action = plugin_action
+        QWidget.__init__(self)
+        
+        self.l = QVBoxLayout()
+        self.setLayout(self.l)
+
+        label = QLabel('These settings provide integration with the Count Pages Plugin.  Count Pages can automatically update custom columns with page, word and reading level statistics.  You have to create and configure the columns in Count Pages first.')
+        label.setWordWrap(True)
+        self.l.addWidget(label)
+        self.l.addSpacing(5)
+
+        label = QLabel('If any of the settings below are checked, when stories are added or updated, the Count Pages Plugin will be called to update the checked statistics.')
+        label.setWordWrap(True)
+        self.l.addWidget(label)
+        self.l.addSpacing(5)
+        
+        # 'PageCount', 'WordCount', 'FleschReading', 'FleschGrade', 'GunningFog'
+        self.pagecount = QCheckBox('PageCount',self)
+        self.pagecount.setToolTip('Which column and algorithm to use are configured in Count Pages.')
+        self.pagecount.setChecked('PageCount' in prefs['countpagesstats'])
+        self.l.addWidget(self.pagecount)
+            
+        self.wordcount = QCheckBox('WordCount',self)
+        self.wordcount.setToolTip('Which column and algorithm to use are configured in Count Words.\nWill overwrite word count from FFDL metadata if set to update the same custom column.')
+        self.wordcount.setChecked('WordCount' in prefs['countpagesstats'])
+        self.l.addWidget(self.wordcount)
+
+        self.fleschreading = QCheckBox('FleschReading',self)
+        self.fleschreading.setToolTip('Which column and algorithm to use are configured in Count Pages.')
+        self.fleschreading.setChecked('FleschReading' in prefs['countpagesstats'])
+        self.l.addWidget(self.fleschreading)
+        
+        self.fleschgrade = QCheckBox('Fleschgrade',self)
+        self.fleschgrade.setToolTip('Which column and algorithm to use are configured in Count Pages.')
+        self.fleschgrade.setChecked('Fleschgrade' in prefs['countpagesstats'])
+        self.l.addWidget(self.fleschgrade)
+        
+        self.gunningfog = QCheckBox('GunningFog',self)
+        self.gunningfog.setToolTip('Which column and algorithm to use are configured in Count Pages.')
+        self.gunningfog.setChecked('GunningFog' in prefs['countpagesstats'])
+        self.l.addWidget(self.gunningfog)
+        
+        self.l.insertStretch(-1)
+        
 class OtherTab(QWidget):
 
     def __init__(self, parent_dialog, plugin_action):
@@ -580,6 +658,12 @@ class OtherTab(QWidget):
         reset_confirmation_button.clicked.connect(self.reset_dialogs)
         self.l.addWidget(reset_confirmation_button)
         
+        view_prefs_button = QPushButton('&View library preferences...', self)
+        view_prefs_button.setToolTip(_(
+                    'View data stored in the library database for this plugin'))
+        view_prefs_button.clicked.connect(self.view_prefs)
+        self.l.addWidget(view_prefs_button)
+        
         self.l.insertStretch(-1)
         
     def reset_dialogs(self):
@@ -591,6 +675,10 @@ class OtherTab(QWidget):
                     _('Confirmation dialogs have all been reset'),
                     show=True,
                     show_copy_button=False)
+        
+    def view_prefs(self):
+        d = PrefsViewerDialog(self.plugin_action.gui, PREFS_NAMESPACE)
+        d.exec_()
 
 permitted_values = {
     'int' : ['numWords','numChapters'],
@@ -661,13 +749,15 @@ titleLabels = {
     'version':'FFDL Version'
     }
 
-class ColumnsTab(QWidget):
+class CustomColumnsTab(QWidget):
 
     def __init__(self, parent_dialog, plugin_action):
         self.parent_dialog = parent_dialog
         self.plugin_action = plugin_action
         QWidget.__init__(self)
         
+        custom_columns = self.plugin_action.gui.library_view.model().custom_columns
+
         self.l = QVBoxLayout()
         self.setLayout(self.l)
 
@@ -678,8 +768,6 @@ class ColumnsTab(QWidget):
         
         self.custcol_dropdowns = {}
 
-        custom_columns = self.plugin_action.gui.library_view.model().custom_columns
-
         for key, column in custom_columns.iteritems():
 
             if column['datatype'] in permitted_values:
@@ -707,4 +795,26 @@ class ColumnsTab(QWidget):
         
         self.l.insertStretch(-1)
 
+        self.l.addSpacing(5)
+        label = QLabel("Special column:")
+        label.setWordWrap(True)
+        self.l.addWidget(label)
+        self.l.addSpacing(5)
+        
+        horz = QHBoxLayout()
+        label = QLabel("Update/Overwrite Error Column:")
+        tooltip="When an update or overwrite of an existing story fails, record the reason in this column.\n(Text and Long Text columns only.)"
+        label.setToolTip(tooltip)
+        horz.addWidget(label)
+        self.errorcol = QComboBox(self)
+        self.errorcol.setToolTip(tooltip)
+        self.errorcol.addItem('',QVariant('none'))
+        for key, column in custom_columns.iteritems():
+            if column['datatype'] in ('text','comments'):
+                self.errorcol.addItem(column['name'],QVariant(key))
+        self.errorcol.setCurrentIndex(self.errorcol.findData(QVariant(prefs['errorcol'])))
+        horz.addWidget(self.errorcol)
+        self.l.addLayout(horz)
+        
+        
         #print("prefs['custom_cols'] %s"%prefs['custom_cols'])

calibre-plugin/dialogs.py

@@ -371,7 +371,7 @@ class AuthorTableWidgetItem(ReadOnlyTableWidgetItem):
 
     #Qt uses a simple < check for sorting items, override this to use the sortKey
     def __lt__(self, other):
-        return self.sort_key < other.sort_key
+        return self.sort_key.lower() < other.sort_key.lower()
 
 class UpdateExistingDialog(SizePersistedDialog):
     def __init__(self, gui, header, prefs, icon, books,
@@ -623,7 +623,7 @@ class StoryListTableWidget(QTableWidget):
         title_cell.setData(Qt.UserRole, QVariant(row))
         self.setItem(row, 1, title_cell)
         
-        self.setItem(row, 2, AuthorTableWidgetItem(book['author'], book['author_sort']))
+        self.setItem(row, 2, AuthorTableWidgetItem(", ".join(book['author']), ", ".join(book['author_sort'])))
         
         url_cell = ReadOnlyTableWidgetItem(book['url'])
         #url_cell.setData(Qt.UserRole, QVariant(book['url']))

calibre-plugin/ffdl_plugin.py

@@ -19,6 +19,7 @@ from PyQt4.Qt import (QApplication, QMenu, QToolButton)
 from PyQt4.Qt import QPixmap, Qt
 from PyQt4.QtCore import QBuffer
 
+from calibre.constants import numeric_version as calibre_version
 
 from calibre.ptempfile import PersistentTemporaryFile, PersistentTemporaryDirectory, remove_dir
 from calibre.ebooks.metadata import MetaInformation, authors_to_string
@@ -193,10 +194,10 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                                                             shortcut_name='Get URLs from Selected Books',
                                                             triggered=self.get_list_urls)
 
-            self.get_list_action = self.create_menu_item_ex(self.menu, 'Get Story URLs from Web Page', image='view.png',
-                                                            unique_name='Get Story URLs from Web Page',
-                                                            shortcut_name='Get Story URLs from Web Page',
-                                                            triggered=self.get_urls_from_page)
+            self.get_list_url_action = self.create_menu_item_ex(self.menu, 'Get Story URLs from Web Page', image='view.png',
+                                                                unique_name='Get Story URLs from Web Page',
+                                                                shortcut_name='Get Story URLs from Web Page',
+                                                                triggered=self.get_urls_from_page)
 
             self.menu.addSeparator()
             self.config_action = create_menu_action_unique(self, self.menu, '&Configure Plugin', shortcut=False,
@@ -205,11 +206,11 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                                                            shortcut_name='Configure FanFictionDownLoader',
                                                            triggered=partial(do_user_config,parent=self.gui))
             
-            self.config_action = create_menu_action_unique(self, self.menu, '&About Plugin', shortcut=False,
-                                                           image= 'images/icon.png',
-                                                           unique_name='About FanFictionDownLoader',
-                                                           shortcut_name='About FanFictionDownLoader',
-                                                           triggered=self.about)
+            self.about_action = create_menu_action_unique(self, self.menu, '&About Plugin', shortcut=False,
+                                                          image= 'images/icon.png',
+                                                          unique_name='About FanFictionDownLoader',
+                                                          shortcut_name='About FanFictionDownLoader',
+                                                          triggered=self.about)
             
             # Before we finalize, make sure we delete any actions for menus that are no longer displayed
             for menu_id, unique_name in self.old_actions_unique_map.iteritems():
@@ -480,7 +481,7 @@ make_firstimage_cover:true
 
         book['all_metadata'] = story.getAllMetadata(removeallentities=True)
         book['title'] = story.getMetadata("title", removeallentities=True)
-        book['author_sort'] = book['author'] = story.getMetadata("author", removeallentities=True)
+        book['author_sort'] = book['author'] = story.getList("author", removeallentities=True)
         book['publisher'] = story.getMetadata("site")
         book['tags'] = writer.getTags() # getTags could be moved up into adapter now.  Adapter didn't used to know the fileform
         book['comments'] = stripHTML(story.getMetadata("description")) #, removeallentities=True) comments handles entities better.
@@ -496,6 +497,7 @@ make_firstimage_cover:true
         book['password'] = adapter.password
 
         book['icon'] = 'plus.png'
+        book['status'] = 'Add'
         if story.getMetadataRaw('datePublished'):
             # should only happen when an adapter is broken, but better to
             # fail gracefully.
@@ -504,6 +506,7 @@ make_firstimage_cover:true
         
         if collision in (CALIBREONLY):
             book['icon'] = 'metadata.png'
+            book['status'] = 'Meta'
         
         # Dialogs should prevent this case now.
         if collision in (UPDATE,UPDATEALWAYS) and fileform != 'epub':
@@ -520,20 +523,30 @@ make_firstimage_cover:true
         # only care about collisions when not ADDNEW
         elif collision != ADDNEW:
             # 'new' book from URL.  collision handling applies.
-            print("from URL")
+            print("from URL(%s)"%url)
 
-            # find dups
-            mi = MetaInformation(story.getMetadata("title", removeallentities=True),
-                                 [story.getMetadata("author", removeallentities=True)]) # author is a list.
-            identicalbooks = db.find_identical_books(mi)
-            ## removed for being overkill.
-            # for ib in identicalbooks:
-            #     # only *really* identical if URL matches, too.
-            #     # XXX make an option?
-            #     if self._get_story_url(db,ib) == url:
-            #         identicalbooks.append(ib)
-            #print("identicalbooks:%s"%identicalbooks)
+            # try to find by identifier url first.
+            searchstr = 'identifiers:"=url:%s"'%url.replace(":","|")
+            identicalbooks = db.search_getting_ids(searchstr, None)
+            if len(identicalbooks) < 1:
+                # find dups
+                authlist = story.getList("author", removeallentities=True)
+                if len(authlist) > 100 and calibre_version < (0, 8, 61):
+                    ## should be fixed from 0.8.61 on.  In the
+                    ## meantime, if it matches the title *and* first
+                    ## 100 authors, I'm prepared to assume it's a
+                    ## match.
+                    print("reduce author list to 100 only when calibre < 0.8.61")
+                    authlist = authlist[:100]
+                mi = MetaInformation(story.getMetadata("title", removeallentities=True),
+                                     authlist)
+                identicalbooks = db.find_identical_books(mi)
+                if len(identicalbooks) > 0:
+                    print("existing found by title/author(s)")
 
+            else:
+                print("existing found by identifier URL")
+                
             if collision == SKIP and identicalbooks:
                 raise NotGoingToDownload("Skipping duplicate story.","list_remove.png")
 
@@ -544,12 +557,12 @@ make_firstimage_cover:true
             if collision == CALIBREONLY and not identicalbooks:
                 collision = ADDNEW
                 options['collision'] = ADDNEW
-            #     raise NotGoingToDownload("Not updating Calibre Metadata, no existing book to update.","search_delete_saved.png")
 
             if len(identicalbooks)>0:
                 book_id = identicalbooks.pop()
                 book['calibre_id'] = book_id
                 book['icon'] = 'edit-redo.png'
+                book['status'] = 'Update'
 
         if book_id != None and collision != ADDNEW:
             if collision in (CALIBREONLY):
@@ -653,6 +666,22 @@ make_firstimage_cover:true
                                        label_text='None of the URLs/stories given can be/need to be downloaded.'
                                        )
             d.exec_()
+
+            
+            custom_columns = self.gui.library_view.model().custom_columns
+            if prefs['errorcol'] != '' and prefs['errorcol'] in custom_columns:
+                label = custom_columns[prefs['errorcol']]['label']
+                ## if error column and all bad.
+                self.previous = self.gui.library_view.currentIndex()
+                LoopProgressDialog(self.gui,
+                                   book_list,
+                                   partial(self._update_bad_book, label=label, options=options, db=self.gui.current_db),
+                                   partial(self._update_books_completed, options=options, showlist=False),
+                                   init_label="Updating calibre for BAD FanFiction stories...",
+                                   win_title="Update calibre for BAD FanFiction stories",
+                                   status_prefix="Updated")
+            
+            
             return
             
         func = 'arbitrary_n'
@@ -682,7 +711,16 @@ make_firstimage_cover:true
                 (options['updatemeta'] and book['good']):
             self._update_metadata(db, book['calibre_id'], book, mi, options)
 
-    def _update_books_completed(self, book_list, options={}):
+    def _update_bad_book(self,book,db=None,label='errorcol',
+                         options={'fileform':'epub',
+                                  'collision':ADDNEW,
+                                  'updatemeta':True,
+                                  'updateepubcover':True},):
+        if book['calibre_id']:
+            print("add/update bad %s %s %s"%(book['title'],book['url'],book['comment']))
+            db.set_custom(book['calibre_id'], book['comment'], label=label, commit=True)
+
+    def _update_books_completed(self, book_list, options={}, showlist=True):
         
         add_list = filter(lambda x : x['good'] and x['added'], book_list)
         add_ids = [ x['calibre_id'] for x in add_list ]
@@ -694,7 +732,7 @@ make_firstimage_cover:true
             self.gui.library_view.model().books_added(len(add_list))
             self.gui.library_view.model().refresh_ids(add_ids)
 
-        if update_ids:
+        if len(update_list):
             self.gui.library_view.model().refresh_ids(update_ids)
 
         current = self.gui.library_view.currentIndex()
@@ -706,7 +744,7 @@ make_firstimage_cover:true
         
         self.gui.status_bar.show_message(_('Finished Adding/Updating %d books.'%(len(update_list) + len(add_list))), 3000)
             
-        if len(update_list) + len(add_list) != len(book_list):
+        if showlist and (len(update_list) + len(add_list) != len(book_list)):
             d = DisplayStoryListDialog(self.gui,
                                        'Updates completed, final status',
                                        prefs,
@@ -718,7 +756,13 @@ make_firstimage_cover:true
     
         print("all done, remove temp dir.")
         remove_dir(options['tdir'])
-        
+
+        all_ids = add_ids
+        all_ids.extend(update_ids)
+        if 'Count Pages' in self.gui.iactions and len(prefs['countpagesstats']) and len(all_ids):
+            cp_plugin = self.gui.iactions['Count Pages']
+            cp_plugin.count_statistics(all_ids,prefs['countpagesstats'])
+
     def download_list_completed(self, job, options={}):
         if job.failed:
             self.gui.job_exception(job, dialog_title='Failed to Download Stories')
@@ -726,30 +770,90 @@ make_firstimage_cover:true
 
         self.previous = self.gui.library_view.currentIndex()
         db = self.gui.current_db
+        # if display_story_list(self.gui,
+        #                       'Downloads finished, confirm to update Calibre',
+        #                       prefs,
+        #                       self.qaction.icon(),
+        #                       job.result,
+        #                       label_text='Stories will not be added or updated in Calibre without confirmation.',
+        #                       offer_skip=True):
 
-        if display_story_list(self.gui,
-                              'Downloads finished, confirm to update Calibre',
-                              prefs,
-                              self.qaction.icon(),
-                              job.result,
-                              label_text='Stories will not be added or updated in Calibre without confirmation.',
-                              offer_skip=True):
-            
-            book_list = job.result
-            good_list = filter(lambda x : x['good'], book_list)
-            total_good = len(good_list)
+        # payload = (job.statistics_cols_map, book_statistics_map)
+        # all_ids = set(book_statistics_map.keys())
+        # msg = '<p>Count Pages plugin found <b>%d statistics(s)</b>. ' % len(all_ids) + \
+        #       'Proceed with updating columns in your library?'
+        # self.gui.proceed_question(self._update_database_columns,
+        #         payload, job.details,
+        #         'Count log', 'Count complete', msg,
+        #         show_copy_button=False)
 
-            self.gui.status_bar.show_message(_('Adding/Updating %s books.'%total_good))
+        book_list = job.result
+        good_list = filter(lambda x : x['good'], book_list)
+        bad_list = filter(lambda x : x['calibre_id'] and not x['good'], book_list)
+        payload = (good_list, bad_list, options)
+        
+        msg = '''
+<p>FFDL found <b>%s</b> good and <b>%s</b> bad updates.</p>
+<p>See log for details.</p>
+<p>Proceed with updating your library?</p>
+'''%(len(good_list),len(bad_list))
 
-            if total_good > 0:
+        htmllog='<html><body><table border="1"><tr><th>Status</th><th>Title</th><th>Author</th><th>URL</th><th>Comment</th></tr>'
+        for book in good_list:
+            if 'status' in book:
+                status = book['status']
+            else:
+                status = 'Good'
+            htmllog = htmllog + '<tr><td>' + '</td><td>'.join([status,book['title'],", ".join(book['author']),book['url'],book['comment']]) + '</td></tr>'
+        
+        for book in bad_list:
+            if 'status' in book:
+                status = book['status']
+            else:
+                status = 'Bad'
+            htmllog = htmllog + '<tr><td>' + '</td><td>'.join([status,book['title'],", ".join(book['author']),book['url'],book['comment']]) + '</td></tr>'
+        
+        htmllog = htmllog + '</table></body></html>'
+        
+        self.gui.proceed_question(self._do_download_list_update,
+                payload, htmllog,
+                'FFDL log', 'FFDL download complete', msg,
+                show_copy_button=False)
+        
+        
+    def _do_download_list_update(self, payload):
+        
+        (good_list,bad_list,options) = payload
+        total_good = len(good_list)
+
+        self.gui.status_bar.show_message(_('Adding/Updating %s books.'%total_good))
+
+        if total_good > 0:
+            LoopProgressDialog(self.gui,
+                               good_list,
+                               partial(self._update_book, options=options, db=self.gui.current_db),
+                               partial(self._update_books_completed, options=options),
+                               init_label="Updating calibre for FanFiction stories...",
+                               win_title="Update calibre for FanFiction stories",
+                               status_prefix="Updated")
+
+        total_bad = len(bad_list)
+
+        if total_bad > 0:
+            custom_columns = self.gui.library_view.model().custom_columns
+            if prefs['errorcol'] != '' and prefs['errorcol'] in custom_columns:
+                self.gui.status_bar.show_message(_('Adding/Updating %s BAD books.'%total_bad))
+                label = custom_columns[prefs['errorcol']]['label']
+                ## if error column and all bad.
                 LoopProgressDialog(self.gui,
-                                   good_list,
-                                   partial(self._update_book, options=options, db=self.gui.current_db),
-                                   partial(self._update_books_completed, options=options),
-                                   init_label="Updating calibre for FanFiction stories...",
-                                   win_title="Update calibre for FanFiction stories",
+                                   bad_list,
+                                   partial(self._update_bad_book, label=label, options=options, db=self.gui.current_db),
+                                   partial(self._update_books_completed, options=options, showlist=False),
+                                   init_label="Updating calibre for BAD FanFiction stories...",
+                                   win_title="Update calibre for BAD FanFiction stories",
                                    status_prefix="Updated")
 
+                
     def _add_or_update_book(self,book,options,prefs,mi=None):
         db = self.gui.current_db
         
@@ -771,6 +875,7 @@ make_firstimage_cover:true
             book['comment'] = "Adding format to book failed for some reason..."
             book['good']=False
             book['icon']='dialog_error.png'
+            book['status'] = 'Error'
 
         if prefs['deleteotherforms']:
             fmts = db.formats(book['calibre_id'], index_is_id=True).split(',')
@@ -812,9 +917,11 @@ make_firstimage_cover:true
 
         # set author link if found.  All current adapters have authorUrl, except anonymous on AO3.
         if 'authorUrl' in book['all_metadata']:
-            autid=db.get_author_id(book['author'])
-            db.set_link_field_for_author(autid, unicode(book['all_metadata']['authorUrl']),
-                                         commit=False, notify=False)
+            authurls = book['all_metadata']['authorUrl'].split(", ")
+            for i, auth in enumerate(book['author']):
+                autid=db.get_author_id(auth)
+                db.set_link_field_for_author(autid, unicode(authurls[i]),
+                                             commit=False, notify=False)
             
         db.set_metadata(book_id,mi)
 
@@ -850,8 +957,13 @@ make_firstimage_cover:true
                 db.set_custom(book_id, val, label=label, commit=False)
 
         db.commit()
-        
+
         if 'Generate Cover' in self.gui.iactions and (book['added'] or not prefs['gcnewonly']):
+            
+            # force a refresh if generating cover so complex composite
+            # custom columns are current and correct
+            db.refresh_ids([book_id])
+                
             gc_plugin = self.gui.iactions['Generate Cover']
             setting_name = None
             if prefs['allow_gc_from_ini']:
@@ -866,7 +978,7 @@ make_firstimage_cover:true
                 for line in adapter.getConfig('generate_cover_settings').splitlines():
                     if "=>" in line:
                         (template,regexp,setting) = map( lambda x: x.strip(), line.split("=>") )
-                        value = Template(template).substitute(book['all_metadata']).encode('utf8')
+                        value = Template(template).safe_substitute(book['all_metadata']).encode('utf8')
                         print("%s(%s) => %s => %s"%(template,value,regexp,setting))
                         if re.search(regexp,value):
                             setting_name = setting
@@ -888,7 +1000,11 @@ make_firstimage_cover:true
                 print("Running Generate Cover with settings %s."%setting_name)
                 realmi = db.get_metadata(book_id, index_is_id=True)
                 gc_plugin.generate_cover_for_book(realmi,saved_setting_name=setting_name)
-        
+
+        ## if error column set.
+        if prefs['errorcol'] != '' and prefs['errorcol'] in custom_columns:
+            label = custom_columns[prefs['errorcol']]['label']
+            db.set_custom(book['calibre_id'], '', label=label, commit=True) # book['comment']
 
     def _get_clean_reading_lists(self,lists):
         if lists == None or lists.strip() == "" :
@@ -951,19 +1067,19 @@ make_firstimage_cover:true
                         message="<p>You configured FanFictionDownLoader to automatically update Reading List '%s', but you don't have a list of that name?</p>"%l
                         confirm(message,'fanfictiondownloader_no_reading_list_%s'%l, self.gui)
 
-    def _find_existing_book_id(self,db,book,matchurl=True):
-        mi = MetaInformation(book["title"],[book["author"]]) # author is a list.
-        identicalbooks = db.find_identical_books(mi)
-        if matchurl: # only *really* identical if URL matches, too.
-            for ib in identicalbooks:
-                if self._get_story_url(db,ib) == book['url']:
-                    return ib
-        if identicalbooks:
-            return identicalbooks.pop()
-        return None
+    # def _find_existing_book_id(self,db,book,matchurl=True):
+    #     mi = MetaInformation(book["title"],book["author"]) # author is a list.
+    #     identicalbooks = db.find_identical_books(mi)
+    #     if matchurl: # only *really* identical if URL matches, too.
+    #         for ib in identicalbooks:
+    #             if self._get_story_url(db,ib) == book['url']:
+    #                 return ib
+    #     if identicalbooks:
+    #         return identicalbooks.pop()
+    #     return None
     
     def _make_mi_from_book(self,book):
-        mi = MetaInformation(book['title'],[book['author']]) # author is a list.
+        mi = MetaInformation(book['title'],book['author']) # author is a list.
         mi.set_identifiers({'url':book['url']})
         mi.publisher = book['publisher']
         mi.tags = book['tags']
@@ -1000,8 +1116,7 @@ make_firstimage_cover:true
         book['good'] = True
         book['calibre_id'] = None
         book['title'] = 'Unknown'
-        book['author'] = 'Unknown'
-        book['author_sort'] = 'Unknown'
+        book['author_sort'] = book['author'] = ['Unknown'] # list
         book['begin'] = None
         book['end'] = None
             
@@ -1017,8 +1132,7 @@ make_firstimage_cover:true
         book['good'] = good
         book['calibre_id'] = idval
         book['title'] = 'Unknown'
-        book['author'] = 'Unknown'
-        book['author_sort'] = 'Unknown'
+        book['author_sort'] = book['author'] = ['Unknown'] # list
         book['begin'] = None
         book['end'] = None
             
@@ -1034,7 +1148,7 @@ make_firstimage_cover:true
         book['good'] = True
         book['calibre_id'] = mi.id
         book['title'] = mi.title
-        book['author'] = authors_to_string(mi.authors)
+        book['author'] = mi.authors
         book['author_sort'] = mi.author_sort
         book['comment'] = ''
         book['url'] = ""
@@ -1049,6 +1163,7 @@ make_firstimage_cover:true
             book['comment'] = "No story URL found."
             book['good'] = False
             book['icon'] = 'search_delete_saved.png'
+            book['status'] = 'Not Found'
         else:
             # get normalized url or None.
             book['url'] = self._is_good_downloader_url(url)
@@ -1057,6 +1172,7 @@ make_firstimage_cover:true
                 book['comment'] = "URL is not a valid story URL."
                 book['good'] = False
                 book['icon']='dialog_error.png'
+                book['status'] = 'Bad URL'
     
     def _get_story_url(self, db, book_id):
         identifiers = db.get_identifiers(book_id,index_is_id=True)

calibre-plugin/jobs.py

@@ -177,6 +177,7 @@ def do_download_for_worker(book,options):
         book['good']=False
         book['comment']=unicode(e)
         book['icon']='dialog_error.png'
+        book['status'] = 'Error'
         print("Exception: %s:%s"%(book,unicode(e)))
         traceback.print_exc()
 

defaults.ini

@@ -66,7 +66,9 @@ extratags_label:Extra Tags
 version_label:FFDL Version
 
 ## items to include in the title page
-## Empty entries will *not* appear, even if in the list.
+## Empty metadata entries will *not* appear, even if in the list.
+## You can include extra text or HTML that will be included as-is in
+## the title page. Eg: titlepage_entries: ...,<br />,summary,<br />,...
 ## All current formats already include title and author.
 titlepage_entries: series,category,genre,language,characters,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,description
 
@@ -311,6 +313,19 @@ extratags: FanFiction,Testing,HTML
 
 [archive.skyehawke.com]
 
+[archiveofourown.org]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
 [ashwinder.sycophanthex.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -342,6 +357,14 @@ extratags: FanFiction,Testing,HTML
 ## cover image.  This lets you exclude them.
 cover_exclusion_regexp:/images/.*?ribbon.gif
 
+[dark-solace.org]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
 [dramione.org]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -392,18 +415,47 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 #username:YourName
 #password:yourpassword
 
+[finestories.com]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+[grangerenchanted.com]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
+[hlfiction.net]
+
 [lumos.sycophanthex.com]
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+[national-library.net]
+
+[ncisfic.com]
+
 [nfacommunity.com]
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+[nha.magical-worlds.us]
+
 [occlumency.sycophanthex.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -425,12 +477,56 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 ## personal.ini, not defaults.ini.
 #is_adult:true
 
+[ponyfictionarchive.net]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+[pretendercentre.com]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
 [samdean.archive.nu]
 
+[sg1-heliopolis.com]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
 [stargate-atlantis.org]
 
+[svufiction.com]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
 [thehexfiles.net]
 
+[themasque.net]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
 [thequidditchpitch.org]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -452,12 +548,6 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 
 [www.thealphagate.com]
 
-[www.archiveofourown.org]
-## Some sites do not require a login, but do require the user to
-## confirm they are adult for adult content.  In commandline version,
-## this should go in your personal.ini, not defaults.ini.
-#is_adult:true
-
 [www.checkmated.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -479,6 +569,14 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 ## Comment this out or change it to false to use them anyway.
 never_make_cover: true
 
+[www.fanfiktion.de]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
 [www.ficbook.net]
 
 [www.fictionalley.org]
@@ -525,6 +623,8 @@ extratags:
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+[www.hpfanficarchive.com]
+
 [www.ik-eternal.net]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -614,6 +714,8 @@ collect_series: false
 #username:YourName
 #password:yourpassword
 
+[www.twilightarchives.com]
+
 [www.twilighted.net]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -644,6 +746,19 @@ collect_series: false
 
 [www.whofic.com]
 
+[www.wizardtales.net]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
 [www.wraithbait.com]
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,

downloader.py

@@ -16,8 +16,6 @@
 #
 
 import logging
-## XXX cli option for logging level.
-logging.basicConfig(level=logging.DEBUG,format="%(levelname)s:%(filename)s(%(lineno)d):%(message)s")
 
 import sys, os
 from os.path import normpath, expanduser, isfile, join
@@ -74,9 +72,18 @@ def main():
    parser.add_option("-l", "--list",
                      action="store_true", dest="list",
                      help="Get list of valid story URLs from page given.",)
+   parser.add_option("-d", "--debug",
+                     action="store_true", dest="debug",
+                     help="Show debug output while downloading.",)
    
    (options, args) = parser.parse_args()
 
+   if options.debug:
+       logging.basicConfig(level=logging.DEBUG,format="%(levelname)s:%(filename)s(%(lineno)d):%(message)s")
+   else:
+       logging.basicConfig(level=logging.INFO,format="%(levelname)s:%(filename)s(%(lineno)d):%(message)s")
+
+   
    if len(args) != 1:
        parser.error("incorrect number of arguments")
 
@@ -141,11 +148,16 @@ def main():
        ## Check for include_images and absence of PIL, give warning.
        if adapter.getConfig('include_images'):
            try:
-               import Image
+               from calibre.utils.magick import Image
+               logging.debug("Using calibre.utils.magick")
            except:
-               print "You have include_images enabled, but Python Image Library(PIL) isn't found.\nImages will be included full size in original format.\nContinue? (y/n)?"
-               if not sys.stdin.readline().strip().lower().startswith('y'):
-                   return
+               try:
+                   import Image
+                   logging.debug("Using PIL")
+               except:
+                   print "You have include_images enabled, but Python Image Library(PIL) isn't found.\nImages will be included full size in original format.\nContinue? (y/n)?"
+                   if not sys.stdin.readline().strip().lower().startswith('y'):
+                       return
                
 
        ## three tries, that's enough if both user/pass & is_adult needed,

fanficdownloader/adapters/__init__.py

@@ -76,6 +76,22 @@ import adapter_destinysgatewaycom
 import adapter_ncisfictioncom
 import adapter_stargateatlantisorg
 import adapter_thealphagatecom
+import adapter_fanfiktionde
+import adapter_ponyfictionarchivenet
+import adapter_sg1heliopoliscom
+import adapter_ncisficcom
+import adapter_nationallibrarynet
+import adapter_themasquenet
+import adapter_pretendercentrecom
+import adapter_darksolaceorg
+import adapter_finestoriescom
+import adapter_hpfanficarchivecom
+import adapter_svufictioncom
+import adapter_twilightarchivescom
+import adapter_wizardtalesnet
+import adapter_nhamagicalworldsus
+import adapter_hlfictionnet
+import adapter_grangerenchantedcom
 
 ## This bit of complexity allows adapters to be added by just adding
 ## importing.  It eliminates the long if/else clauses we used to need
@@ -113,6 +129,11 @@ def getAdapter(config,url,fileform=None):
 
     logging.debug("site:"+domain)
     cls = getClassFor(domain)
+    if not cls and domain.startswith("www."):
+        domain = domain.replace("www.","")
+        logging.debug("trying site:without www: "+domain)
+        cls = getClassFor(domain)
+        fixedurl = fixedurl.replace("http://www.","http://")
     if not cls:
         logging.debug("trying site:www."+domain)
         cls = getClassFor("www."+domain)

fanficdownloader/adapters/adapter_archiveofourownorg.py

@@ -42,6 +42,9 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
                                # utf8) are really windows-1252.
 							   
 							   
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
@@ -60,13 +63,54 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
     @staticmethod # must be @staticmethod, don't remove it.
     def getSiteDomain():
         # The site domain.  Does have www here, if it uses it.
-        return 'www.archiveofourown.org'
+        return 'archiveofourown.org'
+
+    @classmethod
+    def getAcceptDomains(cls):
+        return ['www.archiveofourown.org','archiveofourown.org']
 
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/works/123456"
 
     def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/works/")+r"\d+(/chapters/\d+)?/?$"
+        return re.escape("http://")+"(www\.)?"+re.escape(self.getSiteDomain()+"/works/")+r"\d+(/chapters/\d+)?/?$"
+        
+    ## Login
+    def needToLoginCheck(self, data):
+        if 'This work is only available to registered users of the Archive.' in data \
+                or "The password or user name you entered doesn't match our records" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url, data):
+
+        params = {}
+        if self.password:
+            params['user_session[login]'] = self.username
+            params['user_session[password]'] = self.password
+        else:
+            params['user_session[login]'] = self.getConfig("username")
+            params['user_session[password]'] = self.getConfig("password")
+        params['user_session[remember_me]'] = '1'
+        params['commit'] = 'Log in'
+        #params['utf8'] = u'✓'#u'\x2713' # gets along with out it, and it confuses the encoder.
+        params['authenticity_token'] = data.split('input name="authenticity_token" type="hidden" value="')[1].split('" /></div>')[0]
+
+        loginUrl = 'http://' + self.getSiteDomain() + '/user_sessions'
+        logging.info("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['user_session[login]']))
+    
+        d = self._postUrl(loginUrl, params)
+        #logging.info(d)
+    
+        if "Successfully logged in" not in d : #Member Account
+            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['user_session[login]']))
+            raise exceptions.FailedToLogin(url,params['user_session[login]'])
+            return False
+        else:
+            return True
 
     ## Getting the chapter list and the meta data, plus 'is adult' checking.
     def extractChapterUrlsAndMetadata(self):
@@ -76,13 +120,14 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
         else:
             addurl=""
 
-        meta = self.url+addurl
+        metaurl = self.url+addurl
         url = self.url+'/navigate'+addurl
-        logging.debug("URL: "+meta)
+        logging.info("url: "+url)
+        logging.info("metaurl: "+metaurl)
 
         try:
             data = self._fetchUrl(url)
-            meta = self._fetchUrl(meta)
+            meta = self._fetchUrl(metaurl)
 
             if "This work could have adult content. If you proceed you have agreed that you are willing to see such content." in meta:
                 raise exceptions.AdultCheckRequired(self.url)
@@ -92,11 +137,16 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
                 raise exceptions.StoryDoesNotExist(self.meta)
             else:
                 raise e
+                
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url,data)
+            data = self._fetchUrl(url)
+            meta = self._fetchUrl(metaurl)
             
         # use BeautifulSoup HTML parser to make everything easier to find.
         soup = bs.BeautifulSoup(data)
         metasoup = bs.BeautifulSoup(meta)
-        # print data
 
         # Now go hunting for all the meta data and the chapter list.
         
@@ -105,14 +155,15 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
         self.story.setMetadata('title',a.string)
 		
         # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"^/users/\w+/pseuds/\w+"))
-        if a == None: # ao3 allows for author 'Anonymous' with no author link.
+        alist = soup.findAll('a', href=re.compile(r"^/users/\w+/pseuds/\w+"))
+        if len(alist) < 1: # ao3 allows for author 'Anonymous' with no author link.
             self.story.setMetadata('author','Anonymous')
             self.story.setMetadata('authorUrl',self.url)
         else:
-            self.story.setMetadata('authorId',a['href'].split('/')[2])
-            self.story.setMetadata('authorUrl','http://'+self.host+a['href'])
-            self.story.setMetadata('author',a.text)
+            for a in alist:
+                self.story.addToList('authorId',a['href'].split('/')[2])
+                self.story.addToList('authorUrl','http://'+self.host+a['href'])
+                self.story.addToList('author',a.text)
 
         # Find the chapters:
         chapters=soup.findAll('a', href=re.compile(r'/works/'+self.story.getMetadata('storyId')+"/chapters/\d+$"))

fanficdownloader/adapters/adapter_darksolaceorg.py

@@ -0,0 +1,279 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return DarkSolaceOrgAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class DarkSolaceOrgAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+		
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/elysian/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','dksl')
+
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","Buffy the Vampire Slayer")
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%d %B %Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'dark-solace.org'
+        
+    @classmethod
+    def getAcceptDomains(cls):
+        return ['www.dark-solace.org','dark-solace.org']
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/elysian/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://")+"(www\.)?"+re.escape(self.getSiteDomain()+"/elysian/viewstory.php?sid=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'This story contains adult content not suitable for children' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['rememberme'] = '1'
+        params['sid'] = ''
+        params['intent'] = ''
+        params['submit'] = 'Submit'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/elysian/user.php'
+        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "User Account Page" not in d : #Member Account
+            logging.info("Failed to login to URL %s as %s, or have no authorization to access the story" % (loginUrl, params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            addurl="&ageconsent=ok"
+            self.performLogin(url)
+            data = self._fetchUrl(url+addurl)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title and author
+        a = soup.find('div', {'id' : 'pagetitle'})
+        
+        aut = a.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',aut['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/elysian/'+aut['href'])
+        self.story.setMetadata('author',aut.string)
+        aut.extract()
+        
+        self.story.setMetadata('title',a.string[:(len(a.string)-3)])
+
+        # Find the chapters:
+        chapters=soup.find('select', {'name' : 'chapter'})
+        if chapters != None:
+            for chapter in chapters.findAll('option'):
+                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/elysian/viewstory.php?sid='+self.story.getMetadata('storyId')+'&chapter='+chapter['value']))
+        else:
+            self.chapterUrls.append((self.story.getMetadata('title'),url))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+        
+        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
+        
+        for list in asoup.findAll('div', {'class' : re.compile('listbox\s+')}):
+            a = list.find('a', href=re.compile(r'viewstory.php\?sid='))
+            if a != None:
+                if 'viewstory.php?sid='+self.story.getMetadata('storyId') in a['href']:
+                    break
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+                
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = list.findAll('span', {'class' : 'classification'})
+        for labelspan in labels:
+            label = labelspan.text
+            value = labelspan.nextSibling
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not (defaultGetattr(value,'class') == 'classification' or "Chapters: " in stripHTML(value)):
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value[:len(value)-2])
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'categories.php\?catid=\d+'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                for char in value.string.split(', '):
+                    if not 'None' in char:
+                        self.story.addToList('characters',char)
+
+            if 'Genre' in label:
+                for genre in value.string.split(', '):
+                    if not 'None' in genre:
+                        self.story.addToList('genre',genre)
+
+            if 'Warnings' in label:
+                for warning in value.string.split(', '):
+                    if not 'None' in warning:
+                        self.story.addToList('warnings',warning)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = list.find('a', href=re.compile(r"series.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/elysian/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            i=1
+            for a in storyas:
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if ('viewstory.php?sid='+self.story.getMetadata('storyId')) in a['href']:
+                        self.setSeries(series_name, i)
+                        break
+                    i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_dramioneorg.py

@@ -62,7 +62,7 @@ class DramioneOrgAdapter(BaseSiteAdapter):
 
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%dth %B %Y"
+        self.dateformat = "%d %B %Y"
             
     @staticmethod # must be @staticmethod, don't remove it.
     def getSiteDomain():
@@ -228,11 +228,11 @@ class DramioneOrgAdapter(BaseSiteAdapter):
                     self.story.setMetadata('status', 'In-Progress')
 
             if 'Published' in label:
-                value=value.replace('st','th').replace('nd','th').replace('rd','th')
+                value=re.sub(r"(\d+)(st|nd|rd|th)",r"\1",value)
                 self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
             
             if 'Updated' in label:
-                value=value.replace('st','th').replace('nd','th').replace('rd','th')
+                value=re.sub(r"(\d+)(st|nd|rd|th)",r"\1",value)
                 self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
 
         try:

fanficdownloader/adapters/adapter_erosnsapphosycophanthexcom.py

@@ -227,13 +227,15 @@ class ErosnSapphoSycophantHexComAdapter(BaseSiteAdapter):
 
             # use BeautifulSoup HTML parser to make everything easier to find.
             seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
             i=1
             for a in storyas:
-                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                    self.setSeries(series_name, i)
-                    break
-                i+=1
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        break
+                    i+=1
             
         except:
             # I find it hard to care if the series parsing fails

fanficdownloader/adapters/adapter_fanfictionnet.py

@@ -19,6 +19,7 @@ import time
 import logging
 import re
 import urllib2
+from urllib import unquote_plus
 import time
 
 from .. import BeautifulSoup as bs
@@ -87,7 +88,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
             else:
                 raise e
             
-        if "Unable to locate story with id of " in data:
+        if "Unable to locate story" in data:
             raise exceptions.StoryDoesNotExist(url)
 
         # some times "Chapter not found...", sometimes "Chapter text not found..."
@@ -122,86 +123,23 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         self.story.setMetadata('authorUrl','http://'+self.host+a['href'])
         self.story.setMetadata('author',a.string)
 
+        ## Pull some additional data from html.
+
+        ## ffnet shows category two ways
+        ## 1) class(Book, TV, Game,etc) >> category(Harry Potter, Sailor Moon, etc)
+        ## 2) cat1_cat2_Crossover
+        ## For 1, use the second link.
+        ## For 2, fetch the crossover page and pull the two categories from there.
+
+        categories = soup.findAll('a',{'class':'xcontrast_txt'})
+        if len(categories) > 1:
+            self.story.addToList('category',stripHTML(categories[-1]))
+        elif 'Crossover' in categories[0]['href']:
+            caturl = "http://%s%s"%(self.getSiteDomain(),categories[0]['href'])
+            catsoup = bs.BeautifulSoup(self._fetchUrl(caturl))
+            for a in catsoup.findAll('a',href=re.compile(r"^/crossovers/")):
+                self.story.addToList('category',stripHTML(a))
             
-        # start by finding a script towards the bottom that has a
-        # bunch of useful stuff in it.
-            
-        # var storyid = 6577076;
-        # var chapter = 1;
-        # var chapters = 17;
-        # var words = 42787;
-        # var userid = 2645830;
-        # var title = 'The+Invitation';
-        # var title_t = 'The Invitation';
-        # var summary = 'Dudley Dursley would be the first to say he lived a very normal life. But what happens when he gets invited to his cousin Harry Potter\'s wedding? Will Dudley get the courage to apologize for the torture he caused all those years ago? Harry/Ginny story.';
-        # var categoryid = 224;
-        # var cat_title = 'Harry Potter';
-        # var datep = '12-21-10';
-        # var dateu = '04-06-11';
-        # var author = 'U n F a b u l o u s M e';
-
-        for script in soup.findAll('script', src=None):
-            if not script:
-                continue
-            if not script.string:
-                continue
-            if 'var storyid' in script.string:
-                for line in script.string.split('\n'):
-                    m = re.match(r"^ +var ([^ ]+) = '?(.*?)'?;\r?$",line)
-                    if m == None : continue
-                    var,value = m.groups()
-                    # remove javascript escaping from values.
-                    value = re.sub(r'\\(.)',r'\1',value)
-                    #print var,value
-                    if 'words' in var:
-                        self.story.setMetadata('numWords', value)
-                    if 'title_t' in var:
-                        self.story.setMetadata('title', value)
-                    if 'summary' in var:
-                        self.setDescription(url,value)
-                        #self.story.setMetadata('description', value)
-                    if 'datep' in var:
-                        self.story.setMetadata('datePublished',makeDate(value, '%m-%d-%y'))
-                    if 'dateu' in var:
-                        self.story.setMetadata('dateUpdated',makeDate(value, '%m-%d-%y'))
-                    if 'cat_title' in var:
-                        if "Crossover" in value:
-                            value = re.sub(r' Crossover$','',value)
-                            for c in value.split(' and '):
-                                self.story.addToList('category',c)
-                                # Screws up when the category itself
-                                # contains ' and '.  But that's rare
-                                # and the only alternative is to find
-                                # the 'Crossover' category URL and
-                                # parse that page to search for <a>
-                                # with href /crossovers/(name)/(num)/
-				# <a href="/crossovers/Harry_Potter/224/">Harry Potter</a>
-				# <a href="/crossovers/Naruto/1402/">Naruto</a>
-                        else:
-                            self.story.addToList('category',value)
-                break # for script in soup.findAll('script', src=None):
-            
-        # Find the chapter selector 
-        select = soup.find('select', { 'name' : 'chapter' } )
-    	 
-        if select is None:
-    	   # no selector found, so it's a one-chapter story.
-    	   self.chapterUrls.append((self.story.getMetadata('title'),url))
-        else:
-            allOptions = select.findAll('option')
-            for o in allOptions:
-                url = u'http://%s/s/%s/%s/' % ( self.getSiteDomain(),
-                                            self.story.getMetadata('storyId'),
-                                            o['value'])
-                # just in case there's tags, like <i> in chapter titles.
-                title = u"%s" % o
-                title = re.sub(r'<[^>]+>','',title)
-                self.chapterUrls.append((title,url))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        ## Pull some additional data from html.  Find Rating and look around it.
-
         a = soup.find('a', href='http://www.fictionratings.com/')
         rating = a.string
         if 'Fiction' in rating: # if rating has 'Fiction ', strip that out for consistency with past.
@@ -212,6 +150,14 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         # after Rating, the same bit of text containing id:123456 contains
         # Complete--if completed.
         gui_table1i = soup.find(id="gui_table1i")
+
+        self.story.setMetadata('title', stripHTML(gui_table1i.find('b'))) # title appears to be only(or at least first) bold tag in gui_table1i
+
+        summarydiv = gui_table1i.find('div',{'style':'margin-top:2px'})
+        if summarydiv:
+            self.setDescription(url,stripHTML(summarydiv))
+            
+        
         metatext = stripHTML(gui_table1i.find('div', {'style':'color:gray;'})).replace('Hurt/Comfort','Hurt-Comfort')
         metalist = metatext.split(" - ")
         #print("metatext:(%s)"%metalist)
@@ -236,19 +182,54 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
             self.story.extendList('genre',genrelist)
             metalist=metalist[1:]
 
+        donechars = False
+        while len(metalist) > 0:
+            if  metalist[0].startswith('Reviews') or metalist[0].startswith('Chapters') or metalist[0].startswith('Status') or metalist[0].startswith('id:') or metalist[0].startswith('Favs:') or metalist[0].startswith('Follows:'):
+                pass
+            elif  metalist[0].startswith('Updated'):
+                self.story.setMetadata('dateUpdated',makeDate(metalist[0].split(':')[1].strip(), '%m-%d-%y'))
+            elif  metalist[0].startswith('Published'):
+                self.story.setMetadata('datePublished',makeDate(metalist[0].split(':')[1].strip(), '%m-%d-%y'))
+            elif  metalist[0].startswith('Words'):
+                self.story.setMetadata('numWords',metalist[0].split(':')[1].strip())
+            elif not donechars:
+                self.story.extendList('characters',metalist[0].split('&'))
+                donechars = True
+            metalist=metalist[1:]
+
         # next might be characters, otherwise Reviews, Updated, Published, Words
-        if not ( metalist[0].startswith('Reviews') or metalist[0].startswith('Updated') or metalist[0].startswith('Published') or metalist[0].startswith('Words') ):
-            self.story.extendList('characters',metalist[0].split('&'))
+        # if not ( metalist[0].startswith('Reviews') or metalist[0].startswith('Updated') or metalist[0].startswith('Published') or metalist[0].startswith('Words') or metalist[0].startswith('Chapters') ):
+        #     self.story.extendList('characters',metalist[0].split('&'))
         
         if 'Status: Complete' in metatext:
             self.story.setMetadata('status', 'Completed')
         else:
             self.story.setMetadata('status', 'In-Progress')
 
-        img = soup.find('img',{'class':'cimage'})
-        if img:
-            self.story.addImgUrl(self,url,img['src'],self._fetchUrlRaw,cover=True)
+        if self.getConfig('include_images'):
+            img = soup.find('img',{'class':'cimage'})
+            if img:
+                self.story.addImgUrl(self,url,img['src'],self._fetchUrlRaw,cover=True)
             
+        # Find the chapter selector 
+        select = soup.find('select', { 'name' : 'chapter' } )
+    	 
+        if select is None:
+    	   # no selector found, so it's a one-chapter story.
+    	   self.chapterUrls.append((self.story.getMetadata('title'),url))
+        else:
+            allOptions = select.findAll('option')
+            for o in allOptions:
+                url = u'http://%s/s/%s/%s/' % ( self.getSiteDomain(),
+                                            self.story.getMetadata('storyId'),
+                                            o['value'])
+                # just in case there's tags, like <i> in chapter titles.
+                title = u"%s" % o
+                title = re.sub(r'<[^>]+>','',title)
+                self.chapterUrls.append((title,url))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
         return
 
     def getChapterText(self, url):

fanficdownloader/adapters/adapter_fanfiktionde.py

@@ -0,0 +1,199 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return FanFiktionDeAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class FanFiktionDeAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["utf8",
+                       "Windows-1252"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/s/'+self.story.getMetadata('storyId') + '/1')
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','ffde')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%d.%m.%Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.fanfiktion.de'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/s/46ccbef30000616306614050"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/s/")+r"\w+(/\d+)?$"
+        
+        ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Diese Geschichte wurde als entwicklungsbeeintr' in data \
+                or 'There is no such account on our website' in data \
+                or "Noch kein registrierter Benutzer?" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self,url):
+        params = {}
+
+        if self.password:
+            params['nickname'] = self.username
+            params['passwd'] = self.password
+        else:
+            params['nickname'] = self.getConfig("username")
+            params['passwd'] = self.getConfig("password")
+        params['savelogindata'] = '1'
+        params['a'] = 'l'
+        params['submit'] = 'Login...'
+
+        loginUrl = 'https://ssl.fanfiktion.de/'
+        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['nickname']))
+        d = self._postUrl(loginUrl,params)
+    
+        if "Login erfolgreich" not in d : #Member Account
+            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['nickname']))
+            raise exceptions.FailedToLogin(url,params['nickname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        url = self.url
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+                
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+            
+        if "Uhr ist diese Geschichte nur nach einer" in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Auserhalb der Zeit von 23:00 Uhr bis 04:00 Uhr ist diese Geschichte nur nach einer erfolgreichen Altersverifikation zuganglich.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'/s/'+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        head = soup.find('div', {'style' : 'width:85%;float:left;'})
+        a = head.find('a')
+        self.story.setMetadata('authorId',a['href'].split('/')[2])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.find('select').findAll('option'):
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/s/'+self.story.getMetadata('storyId')+'/'+chapter['value']))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+        self.story.setMetadata('language','German')
+
+        #find metadata on the story page
+        self.story.setMetadata('datePublished', makeDate(head.text.split('erstellt: ')[1].split('\n')[0], self.dateformat))
+        
+        self.story.setMetadata('dateUpdated', makeDate(head.text.split('letztes Update: ')[1].split('\n')[0], self.dateformat))
+        
+        for genre in head.text.split('   ')[3].split('/')[0].split(', '):
+            self.story.addToList('genre',genre)
+            
+        if 'fertiggestellt' in head.text:
+            self.story.setMetadata('status', 'Completed')
+        else:
+            self.story.setMetadata('status', 'In Progress')
+
+        #find metadata on the author's page
+        asoup = bs.BeautifulSoup(self._fetchUrl("http://"+self.getSiteDomain()+"?a=q&a1=v&t=nickdetailsstories&lbi=stories&ar=0&nick="+self.story.getMetadata('authorId')))
+        tr=asoup.findAll('tr')
+        for i in range(1,len(tr)):
+            a = tr[i].find('a')
+            if a['href'] == '/s/'+self.story.getMetadata('storyId'):
+                break
+        self.setDescription(url,a['onmouseover'].split("', '")[1])
+        
+        td = tr[i].findAll('td')
+        self.story.addToList('category',td[1].string)
+        self.story.setMetadata('rating', td[4].string)
+        self.story.setMetadata('numWords', td[5].string)
+ 
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+        time.sleep(0.5) ## ffde has "floodlock" protection
+
+        soup = bs.BeautifulSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'storytext'})
+        for a in div.findAll('script'):
+            a.extract()
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_ficbooknet.py

@@ -94,7 +94,7 @@ class FicBookNetAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('h1')
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         logging.debug("Title: (%s)"%self.story.getMetadata('title'))
         
         # Find authorid and URL from... author url.

fanficdownloader/adapters/adapter_finestoriescom.py

@@ -0,0 +1,293 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return FineStoriesComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class FineStoriesComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url
+        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2].split(':')[0])
+        if 'storyInfo' in self.story.getMetadata('storyId'):
+            self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/s/storyInfo.php?id='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','fnst')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%Y-%m-%d"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'finestories.com'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/s/10537 http://"+self.getSiteDomain()+"/s/10537:4010 http://"+self.getSiteDomain()+"/library/storyInfo.php?id=10537"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain())+r"/(s|library)?/(storyInfo.php\?id=)?\d+(:\d+)?(;\d+)?$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Free Registration' in data \
+                or "Invalid Password!" in data \
+                or "Invalid User Name!" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['theusername'] = self.username
+            params['thepassword'] = self.password
+        else:
+            params['theusername'] = self.getConfig("username")
+            params['thepassword'] = self.getConfig("password")
+        params['rememberMe'] = '1'
+        params['page'] = 'http://finestories.com/'
+        params['submit'] = 'Login'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/login.php'
+        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['theusername']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "My Account" not in d : #Member Account
+            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['theusername']))
+            raise exceptions.FailedToLogin(url,params['theusername'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'/s/'+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.text)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"/a/\w+"))
+        self.story.setMetadata('authorId',a['href'].split('/')[2])
+        self.story.setMetadata('authorUrl','http://'+self.host+a['href'])
+        self.story.setMetadata('author',a.text)
+
+        # Find the chapters:
+        chapters = soup.findAll('a', href=re.compile(r'/s/'+self.story.getMetadata('storyId')+":\d+$"))
+        if len(chapters) != 0:
+            for chapter in chapters:
+                # just in case there's tags, like <i> in chapter titles.
+                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+chapter['href']))
+        else:
+            self.chapterUrls.append((self.story.getMetadata('title'),'http://'+self.host+'/s/'+self.story.getMetadata('storyId')))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # surprisingly, the detailed page does not give enough details, so go to author's page
+        
+        skip=0
+        i=0
+        while i == 0:
+            asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')+"&skip="+str(skip)))
+        
+            a = asoup.findAll('td', {'class' : 'lc2'})
+            for lc2 in a:
+                if lc2.find('a')['href'] == '/s/'+self.story.getMetadata('storyId'):
+                    i=1
+                    break
+                if a[len(a)-1] == lc2:
+                    skip=skip+10
+        
+        for cat in lc2.findAll('div', {'class' : 'typediv'}):
+            self.story.addToList('category',cat.text)
+        
+        self.story.setMetadata('numWords', lc2.findNext('td', {'class' : 'num'}).text)
+        
+        lc4 = lc2.findNext('td', {'class' : 'lc4'})
+        
+        
+        try:
+            a = lc4.find('a', href=re.compile(r"/library/show_series.php\?id=\d+"))
+            i = a.parent.text.split('(')[1].split(')')[0]
+            self.setSeries(a.text, i)
+        except:
+            pass
+        try:
+            a = lc4.find('a', href=re.compile(r"/library/universe.php\?id=\d+"))
+            self.story.addToList("category",a.text)
+        except:
+            pass
+            
+        for a in lc4.findAll('span', {'class' : 'help'}):
+            a.extract()
+            
+        self.setDescription('http://'+self.host+'/s/'+self.story.getMetadata('storyId'),lc4.text.split('[More Info')[0])
+            
+        for b in lc4.findAll('b'):
+            label = b.text
+            value = b.nextSibling
+            
+            if 'For Age' in label:
+                self.story.setMetadata('rating', value)
+                
+            if 'Tags' in label:
+                for genre in value.split(', '):
+                    self.story.addToList('genre',genre)
+                    
+            if 'Posted' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value.split('/ (')[0]), self.dateformat))
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value.split('/ (')[0]), self.dateformat))
+                
+            if 'Concluded' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value.split('/ (')[0]), self.dateformat))
+                
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value.split('/ (')[0]), self.dateformat))
+                
+        status = lc4.find('span', {'class' : 'ab'})
+        if  status != None:
+            self.story.setMetadata('status', 'In-Progress')
+            if "Last Activity" in status.text:
+                self.story.setMetadata('dateUpdated', makeDate(status.text.split('Activity: ')[1].split(')')[0], self.dateformat))
+        else:
+            self.story.setMetadata('status', 'Completed')
+
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+        
+        # some big chapters are split over several pages
+        pager = div.find('span', {'class' : 'pager'})
+        if pager != None:
+            urls=pager.findAll('a')
+            urls=urls[:len(urls)-1]
+            
+            
+            for ur in urls:
+                soup = bs.BeautifulSoup(self._fetchUrl("http://"+self.getSiteDomain()+ur['href']),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+                div1 = soup.find('div', {'id' : 'story'})
+                
+                # appending next section
+                last=div.findAll('p')
+                next=div1.find('span', {'class' : 'conTag'}).nextSibling
+            
+                last[len(last)-1]=last[len(last)-1].append(next)
+                div.append(div1)
+            
+            
+            
+        # removing all the left-over stuff    
+        for a in div.findAll('span'):
+            a.extract()
+            
+        for a in div.findAll('h1'):
+            a.extract()
+        for a in div.findAll('h2'):
+            a.extract()
+        for a in div.findAll('h3'):
+            a.extract()
+        for a in div.findAll('h4'):
+            a.extract()
+        for a in div.findAll('br'):
+            a.extract()
+        for a in div.findAll('div', {'class' : 'date'}):
+            a.extract()
+            
+        a = div.find('form')
+        if a != None:
+            b = a.nextSibling
+            while b != None:
+                a.extract()
+                a=b
+                b=b.nextSibling
+        
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_grangerenchantedcom.py

@@ -0,0 +1,302 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return GrangerEnchantedCom
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class GrangerEnchantedCom(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        self.story.setMetadata('section',self.parsedUrl.path.split('/',)[1])
+             
+        # normalized story URL.
+        if "malfoymanor" in self.parsedUrl.netloc:
+            self._setURL('http://malfoymanor.' + self.getSiteDomain() + '/themanor/viewstory.php?sid='+self.story.getMetadata('storyId'))
+            self.story.addToList("category","The Manor")
+        else:
+            self._setURL('http://' + self.getSiteDomain() + '/enchant/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','gech')
+
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","Harry Potter")
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%d/%b/%Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'grangerenchanted.com'
+        
+    @classmethod
+    def getAcceptDomains(cls):
+        return ['grangerenchanted.com','malfoymanor.grangerenchanted.com']
+
+    def getSiteExampleURLs(self):
+        return "http://grangerenchanted.com/enchant/viewstory.php?sid=1234 http://malfoymanor.grangerenchanted.com/themanor/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return r"http://(malfoymanor.)?grangerenchanted.com/(enchant|themanor)?/viewstory.php\?sid=\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Registered Users Only' in data \
+                or 'There is no such account on our website' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['cookiecheck'] = '1'
+        params['submit'] = 'Submit'
+    
+        if "enchant" in self.story.getMetadata('section'):
+            loginUrl = 'http://grangerenchanted.com/enchant/user.php?action=login'
+        else:
+            loginUrl = 'http://malfoymanor.grangerenchanted.com/themanor/user.php?action=login'
+        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "Member Account" not in d : #Member Account
+            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=1"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid & error in url.
+                addurl = addurl.replace("&","&")
+                url = self.url+'&index=1'+addurl
+                logging.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+self.story.getMetadata('section')+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=4'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+self.story.getMetadata('section')+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            i=1
+            for a in storyas:
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        break
+                    i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story1'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_hlfictionnet.py

@@ -0,0 +1,233 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return HLFictionNetAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class HLFictionNetAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','hlf')
+
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","Highlander")
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%m/%d/%y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'hlfiction.net'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title and author
+        a = soup.find('div', {'id' : 'pagetitle'})
+        
+        aut = a.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',aut['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+aut['href'])
+        self.story.setMetadata('author',aut.string)
+        aut.extract()
+        
+        self.story.setMetadata('title',a.string[:(len(a.string)-3)])
+
+        # Find the chapters:
+        chapters=soup.find('select')
+        if chapters != None:
+            for chapter in chapters.findAll('option'):
+                # just in case there's tags, like <i> in chapter titles. 
+                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/viewstory.php?sid='+self.story.getMetadata('storyId')+'&chapter='+chapter['value']))
+        else:
+            self.chapterUrls.append((self.story.getMetadata('title'),url))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+        
+        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
+        
+        for list in asoup.findAll('div', {'class' : re.compile('listbox\s+')}):
+            a = list.find('a')
+            if ('viewstory.php?sid='+self.story.getMetadata('storyId')) in a['href']:
+                break
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = list.findAll('span', {'class' : 'classification'})
+        for labelspan in labels:
+            label = labelspan.string
+            value = labelspan.nextSibling
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'classification':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value[:len(value)-2])
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'categories.php\?catid=\d+'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                for char in value.string.split(', '):
+                    if not 'None' in char:
+                        self.story.addToList('characters',char)
+
+            if 'Genre' in label:
+                for genre in value.string.split(', '):
+                    if not 'None' in genre:
+                        self.story.addToList('genre',genre)
+
+            if 'Warnings' in label:
+                for warning in value.string.split(', '):
+                    if not 'None' in warning:
+                        self.story.addToList('warnings',warning)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = list.find('a', href=re.compile(r"series.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            i=1
+            for a in storyas:
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if ('viewstory.php?sid='+self.story.getMetadata('storyId')) in a['href']:
+                        self.setSeries(series_name, i)
+                        break
+                    i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_hpfanficarchivecom.py

@@ -0,0 +1,221 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return HPFanficArchiveComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class HPFanficArchiveComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/stories/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','hpffa')
+
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","Harry Potter")
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%B %d, %Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.hpfanficarchive.com'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/stories/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/stories/viewstory.php?sid=")+r"\d+$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/stories/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/stories/'+chapter['href']))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) # XXX
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/stories/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            i=1
+            for a in storyas:
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        break
+                    i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+        
+
+        soup = bs.BeautifulSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_ksarchivecom.py

@@ -197,7 +197,13 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
                 svalue = ""
                 while not defaultGetattr(value,'class') == 'label':
                     svalue += str(value)
-                    value = value.nextSibling
+                    # poor HTML(unclosed <p> for one) can cause run on
+                    # over the next label.
+                    if '<span class="label">' in svalue:
+                        svalue = svalue[0:svalue.find('<span class="label">')]
+                        break
+                    else:
+                        value = value.nextSibling
                 self.setDescription(url,svalue)
                 #self.story.setMetadata('description',stripHTML(svalue))
 
@@ -211,7 +217,9 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
                 cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
                 catstext = [cat.string for cat in cats]
                 for cat in catstext:
-                    if cat.string.strip() in ('Poetry','Essays'):
+                    # ran across one story with an empty <a href="browse.php?type=categories&amp;catid=1"></a>
+                    # tag in the desc once.
+                    if cat and cat.strip() in ('Poetry','Essays'):
                         self.story.addToList('category',cat.string)
 
             if 'Characters' in label:
@@ -274,13 +282,15 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
 
             # use BeautifulSoup HTML parser to make everything easier to find.
             seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
             i=1
             for a in storyas:
-                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                    self.setSeries(series_name, i)
-                    break
-                i+=1
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        break
+                    i+=1
             
         except:
             # I find it hard to care if the series parsing fails

fanficdownloader/adapters/adapter_libraryofmoriacom.py

@@ -222,13 +222,15 @@ class LibraryOfMoriaComAdapter(BaseSiteAdapter):
 
             # use BeautifulSoup HTML parser to make everything easier to find.
             seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
             i=1
             for a in storyas:
-                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                    self.setSeries(series_name, i)
-                    break
-                i+=1
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        break
+                    i+=1
             
         except:
             # I find it hard to care if the series parsing fails

fanficdownloader/adapters/adapter_midnightwhispersca.py

@@ -260,10 +260,12 @@ class MidnightwhispersCaAdapter(BaseSiteAdapter): # XXX
             storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
             i=1
             for a in storyas:
-                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                    self.setSeries(series_name, i)
-                    break
-                i+=1
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        break
+                    i+=1
             
         except:
             # I find it hard to care if the series parsing fails

fanficdownloader/adapters/adapter_mugglenetcom.py

@@ -303,13 +303,15 @@ class MuggleNetComAdapter(BaseSiteAdapter): # XXX
 
             # use BeautifulSoup HTML parser to make everything easier to find.
             seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
             i=1
             for a in storyas:
-                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                    self.setSeries(series_name, i)
-                    break
-                i+=1
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        break
+                    i+=1
             
         except:
             # I find it hard to care if the series parsing fails

fanficdownloader/adapters/adapter_nationallibrarynet.py

@@ -0,0 +1,209 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return NationalLibraryNetAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class NationalLibraryNetAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only storyid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?storyid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','ntlb')
+
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","West Wing")
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%m-%d-%y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        return 'national-library.net'
+        
+    @classmethod
+    def getAcceptDomains(cls):
+        return ['www.national-library.net','national-library.net']
+
+    def getSiteExampleURLs(self):
+        return "ONLY the stories archived on or after June 17, 2006 and that are hosted on the website: http://"+self.getSiteDomain()+"/viewstory.php?storyid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://")+"(www\.)?"+re.escape(self.getSiteDomain()+"/viewstory.php?storyid=")+r"\d+$"
+
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('h1')
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"authorresults.php\?author=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for p in soup.findAll('p'):
+            chapters = p.findAll('a', href=re.compile(r'viewstory.php\?storyid='+self.story.getMetadata('storyId')+"&chapnum=\d+$"))
+            if len(chapters) > 0:
+                for chapter in chapters:
+                    # just in case there's tags, like <i> in chapter titles.
+                    self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']))
+                break
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+        self.story.setMetadata('status', 'Completed')
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('b')
+        for x in range(2,len(labels)):
+            value = labels[x].nextSibling
+            label = labels[x].string
+            
+            if 'Summary' in label:
+                self.setDescription(url,value)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rating' in label:
+                self.story.setMetadata('rating', stripHTML(value.nextSibling))
+
+            if 'Word Count' in label:
+                self.story.setMetadata('numWords', value.string)
+
+            if 'Category' in label:
+                for cat in value.string.split(', '):
+                    self.story.addToList('category',cat)
+            if 'Crossover Shows' in label:
+                for cat in value.string.split(', '):
+                    if "No Show" not in cat:
+                        self.story.addToList('category',cat)
+
+            if 'Character' in label:
+                for char in value.string.split(', '):
+                    self.story.addToList('characters',char)
+                    
+            if 'Warnings' in label:
+                for warning in value.string.split(', '):
+                    self.story.addToList('warnings',warning)
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Series' in label:
+                self.setSeries(stripHTML(value.nextSibling), value.nextSibling.nextSibling.string[2:])
+                
+        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
+        story=asoup.find('a', href=re.compile(r'viewstory.php\?storyid='+self.story.getMetadata('storyId')))
+            
+        a=story.findNext(text=re.compile('Genre')).parent.nextSibling.string.split(', ')
+        for genre in a:
+            self.story.setMetadata('genre', genre)
+        
+        a=story.findNext(text=re.compile('Archived'))
+        self.story.setMetadata('datePublished', makeDate(stripHTML(a.parent.nextSibling), self.dateformat))
+        self.story.setMetadata('dateUpdated', makeDate(stripHTML(a.parent.nextSibling), self.dateformat))
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div')
+        
+        # bit messy since higly inconsistent
+        for p in soup.findAll('p', {'align' : 'center'}):
+            p.extract()
+        p = soup.findAll('p')
+        for x in range(0,3):
+            p[x].extract()
+        if "Chapters: " in stripHTML(p[3]):
+            p[3].extract()
+        for x in range(len(p)-2,len(p)-1):
+            p[x].extract()
+
+        for p in soup.findAll('h1'):
+            p.extract()
+        for p in soup.findAll('h3'):
+            p.extract()
+        for p in soup.findAll('a'):
+            p.extract()
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_ncisficcom.py

@@ -0,0 +1,216 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return NCISFicComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class NCISFicComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only storyid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?storyid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','ncisf')
+
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","NCIS")
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%m-%d-%y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        return 'ncisfic.com'
+        
+    @classmethod
+    def getAcceptDomains(cls):
+        return ['www.ncisfic.com','ncisfic.com']
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?storyid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://")+"(www\.)?"+re.escape(self.getSiteDomain()+"/viewstory.php?storyid=")+r"\d+$"
+
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('h1')
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"authorresults.php\?author=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for p in soup.findAll('p'):
+            chapters = p.findAll('a', href=re.compile(r'viewstory.php\?storyid='+self.story.getMetadata('storyId')+"&chapnum=\d+$"))
+            if len(chapters) > 0:
+                for chapter in chapters:
+                    # just in case there's tags, like <i> in chapter titles.
+                    self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']))
+                break
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+        self.story.setMetadata('status', 'Completed')
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('b')
+        for x in range(2,len(labels)):
+            value = labels[x].nextSibling
+            label = labels[x].string
+            
+            if 'Summary' in label:
+                self.setDescription(url,value)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rating' in label:
+                self.story.setMetadata('rating', stripHTML(value.nextSibling))
+
+            if 'Word Count' in label:
+                self.story.setMetadata('numWords', value.string)
+
+            if 'Category' in label:
+                for cat in value.string.split(', '):
+                    self.story.addToList('category',cat)
+            if 'Crossover Shows' in label:
+                for cat in value.string.split(', '):
+                    if "No Show" not in cat:
+                        self.story.addToList('category',cat)
+
+            if 'Character' in label:
+                for char in value.string.split(', '):
+                    self.story.addToList('characters',char)
+                    
+            if 'Warnings' in label:
+                for warning in value.string.split(', '):
+                    self.story.addToList('warnings',warning)
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Series' in label:
+                if "No Series" not in value.nextSibling.string:
+                    self.setSeries(stripHTML(value.nextSibling), value.nextSibling.nextSibling.string[2:])
+                
+        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
+        story=asoup.find('a', href=re.compile(r'viewstory.php\?storyid='+self.story.getMetadata('storyId')))
+        
+        a=story.findNext('font')
+        if 'Complete' in a.string:
+            self.story.setMetadata('status', 'Completed')
+        else:
+            self.story.setMetadata('status', 'In-Progress')
+            
+        a=story.findNext(text=re.compile('Genre')).parent.nextSibling.string.split(', ')
+        for genre in a:
+            self.story.setMetadata('genre', genre)
+        
+        a=story.findNext(text=re.compile('Archived'))
+        self.story.setMetadata('datePublished', makeDate(stripHTML(a.parent.nextSibling), self.dateformat))
+        self.story.setMetadata('dateUpdated', makeDate(stripHTML(a.parent.nextSibling), self.dateformat))
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div')
+        
+        # bit messy since higly inconsistent
+        for p in soup.findAll('p', {'align' : 'center'}):
+            p.extract()
+        p = soup.findAll('p')
+        for x in range(0,3):
+            p[x].extract()
+        if "Chapters: " in stripHTML(p[3]):
+            p[3].extract()
+        for x in range(len(p)-2,len(p)-1):
+            p[x].extract()
+
+        for p in soup.findAll('h1'):
+            p.extract()
+        for p in soup.findAll('h3'):
+            p.extract()
+        for p in soup.findAll('a'):
+            p.extract()
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_nfacommunitycom.py

@@ -261,13 +261,15 @@ class NfaCommunityComAdapter(BaseSiteAdapter): # XXX
 
             # use BeautifulSoup HTML parser to make everything easier to find.
             seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
             i=1
             for a in storyas:
-                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                    self.setSeries(series_name, i)
-                    break
-                i+=1
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        break
+                    i+=1
             
         except:
             # I find it hard to care if the series parsing fails

fanficdownloader/adapters/adapter_nhamagicalworldsus.py

@@ -0,0 +1,216 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return NHAMagicalWorldsUsAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class NHAMagicalWorldsUsAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+		
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','nha')
+
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","Buffy the Vampire Slayer")
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = " %m/%d/%y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'nha.magical-worlds.us'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+        url = self.url
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+            
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
+		
+        try:
+            # in case link points somewhere other than the first chapter
+            a = soup.findAll('option')[1]['value']
+            self.story.setMetadata('storyId',a.split('=',)[1])
+            url = 'http://'+self.host+'/'+a
+            soup = bs.BeautifulSoup(self._fetchUrl(url))
+        except:
+            pass
+		
+        for info in asoup.findAll('table', {'width' : '100%', 'bordercolor' : re.compile(r'#')}):
+            a = info.find('a')
+            if ('viewstory.php?sid='+self.story.getMetadata('storyId')) in a['href']:
+                self.story.setMetadata('title',a.string)
+                break
+		
+
+        # Find the chapters:
+        chapters=soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+'&chapter=\d$'))
+        if len(chapters) == 0:
+            self.chapterUrls.append((self.story.getMetadata('title'),url))
+        else:
+            for chapter in chapters:
+                # just in case there's tags, like <i> in chapter titles.
+                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d):
+            try:
+                return d.name
+            except:
+                return ""
+				
+        cats = info.findAll('a',href=re.compile('categories.php'))
+        for cat in cats:
+            self.story.addToList('category',cat.string)
+					
+        a = info.find('a', href=re.compile(r'reviews.php\?sid='+self.story.getMetadata('storyId')))
+        val = a.nextSibling
+        svalue = ""
+        while not defaultGetattr(val) == 'br':
+            val = val.nextSibling
+        val = val.nextSibling
+        while not defaultGetattr(val) == 'br':
+            svalue += str(val)
+            val = val.nextSibling
+        self.setDescription(url,svalue)
+        
+        #does not provide convenient way to get word count
+        labels = info.findAll('i')
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = stripHTML(labelspan)
+
+            if 'Rating' in label:
+                self.story.setMetadata('rating', value.split(' -')[0])
+
+            if 'Genres' in label:
+                genres = value.string.split(', ')
+                for genre in genres:
+                    if 'None' not in genre:
+                        self.story.addToList('genre',genre.split(' -')[0])
+            
+            if 'Characters' in label:
+                chars = value.string.split(', ')
+                for char in chars:
+                    if 'None' not in char:
+                        self.story.addToList('characters',char.split(' -')[0])
+
+            if 'Warnings' in label:
+                warnings = value.string.split(', ')
+                for warning in warnings:
+                    if 'None' not in warning:
+                        self.story.addToList('warnings',warning.split(' -')[0])
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(value.split(' -')[0], self.dateformat))
+            
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(value.split(' -')[0], self.dateformat))
+
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+		
+        data = self._fetchUrl(url)
+
+        soup = bs.BeautifulSoup(data, selfClosingTags=('br','hr','span','center')) # some chapters seem to be hanging up on those tags, so it is safer to close them
+        
+        story = soup.find('div', {"id" : "story"})
+
+        if None == story:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,story)

fanficdownloader/adapters/adapter_ponyfictionarchivenet.py

@@ -0,0 +1,252 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return PonyFictionArchiveNetAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class PonyFictionArchiveNetAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logging.info(self.parsedUrl.netloc)
+        # normalized story URL.
+        if "explicit" in self.parsedUrl.netloc:
+            self._setURL('http://explicit.' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+            self.dateformat = "%d/%b/%y"
+        else:
+            self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+            self.dateformat = "%d %b %Y"
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','pffa')
+
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","My Little Pony")
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'ponyfictionarchive.net'
+        
+    @classmethod
+    def getAcceptDomains(cls):
+        return ['www.ponyfictionarchive.net','ponyfictionarchive.net','explicit.ponyfictionarchive.net']
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://")+"(www\.|explicit\.)?"+re.escape(self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&warning=9"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid & error in url.
+                addurl = addurl.replace("&","&")
+                url = self.url+'&index=1'+addurl
+                logging.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        genres = soup.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+        for genre in genres:
+            self.story.addToList('genre',genre.string)
+           
+        warnings = soup.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=3'))
+        for warning in warnings:
+            self.story.addToList('warnings',warning.string)
+        
+        status = soup.find('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
+        self.story.setMetadata('status',status.string)
+        
+        section = soup.findAll('span', {'class' : 'General'})[1]            
+        
+        self.story.setMetadata('rating', section.previousSibling.previousSibling.string)
+
+        value = section.nextSibling
+        svalue = ""
+        while not defaultGetattr(value,'class') == 'label':
+            svalue += str(value)
+            value = value.nextSibling
+        self.setDescription(url,svalue)
+        
+        # <span class="label">Rated:</span> NC-17<br /> etc           
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            i=1
+            for a in storyas:
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        break
+                    i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_pretendercentrecom.py

@@ -0,0 +1,255 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return PretenderCenterComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class PretenderCenterComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/missingpieces/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','ptdc')
+
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","The Pretender")
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%d/%m/%y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'pretendercentre.com'
+        
+    @classmethod
+    def getAcceptDomains(cls):
+        return ['www.pretendercentre.com','pretendercentre.com']
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/missingpieces/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://")+"(www\.)?"+re.escape(self.getSiteDomain()+"/missingpieces/viewstory.php?sid=")+r"\d+$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=4"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid & error in url.
+                addurl = addurl.replace("&","&")
+                url = self.url+'&index=1'+addurl
+                logging.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.text)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/missingpieces/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/missingpieces/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+                
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) # XXX
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/missingpieces/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            i=1
+            for a in storyas:
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        break
+                    i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.findAll('div', {'id' : 'story'})[1]
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_sg1heliopoliscom.py

@@ -0,0 +1,255 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return SG1HeliopolisComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class SG1HeliopolisComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        self.story.setMetadata('section',self.parsedUrl.path.split('/',)[1])
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/'+self.story.getMetadata('section')+'/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','sghp')
+
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        if 'atlantis' in self.story.getMetadata('section'):
+            self.story.addToList("category","Stargate: Atlantis")
+        else:
+            self.story.addToList("category","Stargate: SG-1")
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%Y.%m.%d"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'sg1-heliopolis.com'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/archive/viewstory.php?sid=1234 http://"+self.getSiteDomain()+"/adult/viewstory.php?sid=1234 http://"+self.getSiteDomain()+"/atlantis/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return r"http://sg1-heliopolis.com/(archive|adult|atlantis)?/viewstory.php\?sid=\d+$"
+
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&warning=4"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid & error in url.
+                addurl = addurl.replace("&","&")
+                url = self.url+'&index=1'+addurl
+                logging.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+self.story.getMetadata('section')+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+self.story.getMetadata('section')+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            i=1
+            for a in storyas:
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        break
+                    i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_svufictioncom.py

@@ -0,0 +1,271 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return SVUFictionComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class SVUFictionComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','svuf')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%b %d, %Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'svufiction.com'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Registered Users Only' in data \
+                or 'There is no such account on our website' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['cookiecheck'] = '1'
+        params['submit'] = 'Submit'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
+        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "Member Account" not in d : #Member Account
+            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+    
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=6"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+                
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+                
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid & error in url.
+                addurl = addurl.replace("&","&")
+                url = self.url+'&index=1'+addurl
+                logging.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        pt = soup.find('div', {'class' : 'title'})
+        a = pt.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        if a.text != "":
+            self.story.setMetadata('title',a.text)
+        else:
+            self.story.setMetadata('title','Banner')
+        
+        # Find authorid and URL from... author url.
+        a = pt.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        rating=pt.text.split('[')[1].split(']')[0]
+        self.story.setMetadata('rating', rating)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        bottom=soup.find('div', {'class' : 'bottom'}).text
+        self.story.setMetadata('numWords', bottom.split('Word count: ')[1].split(' Read:')[0])
+        self.story.setMetadata('datePublished', makeDate(bottom.split('Published: ')[1].split(' Updated:')[0], self.dateformat))
+        self.story.setMetadata('dateUpdated', makeDate(bottom.split('Updated: ')[1], self.dateformat))
+
+        
+        content=soup.find('div', {'class' : 'content'})
+        value=content.find('h1').nextSibling
+        svalue = ""
+        while not defaultGetattr(value,'class') == 'smaller':
+            svalue += str(value)
+            value = value.nextSibling
+        self.setDescription(url,svalue)
+        
+        status = content.text.split('Completed: ')[1]
+        if 'Yes' in status:
+            self.story.setMetadata('status', 'Completed')
+        else:
+            self.story.setMetadata('status', 'In-Progress')
+        
+        cats = content.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+        for cat in cats:
+            self.story.addToList('category',cat.string)
+            
+        chars = content.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+        for char in chars:
+            self.story.addToList('characters',char.string)
+            
+        genres = content.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+        for genre in genres:
+            self.story.addToList('genre',genre.string)
+            
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            i=1
+            for a in storyas:
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        break
+                    i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_test1.py

@@ -103,8 +103,19 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
         
         self.story.setMetadata('rating','Tweenie')
         
-        self.story.setMetadata('authorId','98765')
-        self.story.setMetadata('authorUrl','http://author/url')
+
+        if self.story.getMetadata('storyId') == '673':
+            self.story.addToList('author','Author From List')
+            self.story.addToList('author','Author From List 2')
+        
+            self.story.addToList('authorId','98765')
+            self.story.addToList('authorId','98765-2')
+        
+            self.story.addToList('authorUrl','http://author/url')
+            self.story.addToList('authorUrl','http://author/url-2')
+        else:
+            self.story.setMetadata('authorId','98765')
+            self.story.setMetadata('authorUrl','http://author/url')
 
         self.story.addToList('warnings','Swearing')
         self.story.addToList('warnings','Violence')
@@ -114,10 +125,15 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
         self.story.addToList('category','Crossover')
         self.story.addToList('category',u'Puella Magi Madoka Magica/魔法少女まどか★マギカ')
         self.story.addToList('category',u'Magical Girl Lyrical Nanoha')
+
         self.story.addToList('genre','Fantasy')
         self.story.addToList('genre','SF')
         self.story.addToList('genre','Noir')
         
+        self.story.addToList('characters','Bob Smith')
+        self.story.addToList('characters','George Johnson')
+        self.story.addToList('characters','Fred Smythe')
+        
         self.chapterUrls = [(u'Prologue '+self.crazystring,self.url+"&chapter=1"),
                             ('Chapter 1, Xenos on Cinnabar',self.url+"&chapter=2"),
                             ('Chapter 2, Sinmay on Kintikin',self.url+"&chapter=3"),
@@ -170,6 +186,7 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
 <p>http://test1.com?sid=670 - Succeeds, but sleeps 2sec on each chapter</p>
 <p>http://test1.com?sid=671 - Succeeds, but sleeps 2sec metadata only</p>
 <p>http://test1.com?sid=672 - Succeeds, quick meta, sleeps 2sec chapters only</p>
+<p>http://test1.com?sid=673 - Succeeds, multiple authors</p>
 <p>Odd sid's will be In-Progress, evens complete.  sid&lt;10 will be assigned one of four languages and included in a series.</p>
 </div>
 '''

fanficdownloader/adapters/adapter_thehexfilesnet.py

@@ -190,12 +190,19 @@ class TheHexFilesNetAdapter(BaseSiteAdapter):
         logging.debug('Getting chapter text from: %s' % url)
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        for a in soup.findAll('table'):
-            a.extract()
+                                     selfClosingTags=('br','hr','img')) # otherwise soup eats the br/hr tags.
 
         if None == soup:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
     
+        # Ugh.  chapter html doesn't haven't anything useful around it to demarcate.
+        for a in soup.findAll('table'):
+            a.extract()        
+
+        for a in soup.findAll('head'):
+            a.extract()
+
+        html = soup.find('html')
+        html.name='div'
+            
         return self.utf8FromSoup(url,soup)

fanficdownloader/adapters/adapter_themasquenet.py

@@ -0,0 +1,271 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return TheMasqueNetAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class TheMasqueNetAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        if self.parsedUrl.path.split('/',)[1] == 'wiktt':
+            self.story.addToList("category","Harry Potter")
+            self.story.setMetadata('section','/wiktt/efiction/')
+            self.dateformat = "%m/%d/%Y"
+        else:
+            self.story.addToList("category","Originals")
+            self.story.setMetadata('section','/efiction/')
+            self.dateformat = "%b %d, %Y"
+            
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + self.story.getMetadata('section') + 'viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','msq')
+        
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'themasque.net'
+        
+    def getSiteExampleURLs(self):
+        return "http://themasque.net/wiktt/efiction/viewstory.php?sid=1234 http://themasque.net/efiction/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain())+"(/wiktt)?/efiction"+re.escape("/viewstory.php?sid=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Registered Users Only' in data \
+                or 'There is no such account on our website' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['cookiecheck'] = '1'
+        params['submit'] = 'Submit'
+    
+        loginUrl = 'http://' + self.getSiteDomain()  + self.story.getMetadata('section') + 'user.php?action=login'
+        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "Member Account" not in d : #Member Account
+            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=4"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid & error in url.
+                addurl = addurl.replace("&","&")
+                url = self.url+'&index=1'+addurl
+                logging.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host + self.story.getMetadata('section') + chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+                
+# summary, rated, word count, categories, characters, genre, warnings, completed, published, updated, seires
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.text
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_thewriterscoffeeshopcom.py

@@ -133,7 +133,7 @@ class TheWritersCoffeeShopComSiteAdapter(BaseSiteAdapter):
 
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -165,7 +165,13 @@ class TheWritersCoffeeShopComSiteAdapter(BaseSiteAdapter):
                 svalue = ""
                 while not defaultGetattr(value,'class') == 'label':
                     svalue += str(value)
-                    value = value.nextSibling
+                    # poor HTML(unclosed <p> for one) can cause run on
+                    # over the next label.
+                    if '<span class="label">' in svalue:
+                        svalue = svalue[0:svalue.find('<span class="label">')]
+                        break
+                    else:
+                        value = value.nextSibling
                 self.setDescription(url,svalue)
 
             if 'Rated' in label:

fanficdownloader/adapters/adapter_tthfanficorg.py

@@ -151,12 +151,33 @@ class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter):
         self.story.setMetadata('authorId',a['href'].split('/')[1].split('-')[1])
         self.story.setMetadata('authorUrl','http://'+self.host+a['href'])
         self.story.setMetadata('author',stripHTML(a))
+        authorurl = 'http://'+self.host+a['href']
 
+        ainfo = soup.find('a', href='/StoryInfo-%s-1'%self.story.getMetadata('storyId'))
+        if ainfo != None: # indicates multiple authors/contributors.
+            try:
+                # going to pull part of the meta data from author list page.
+                infourl = 'http://'+self.host+ainfo['href']
+                logging.debug("**StoryInfo** URL: "+infourl)
+                infodata = self._fetchUrl(infourl)
+                infosoup = bs.BeautifulSoup(infodata)
+
+                for a in infosoup.findAll('a',href=re.compile(r"^/Author-\d+")):
+                    self.story.addToList('authorId',a['href'].split('/')[1].split('-')[1])
+                    self.story.addToList('authorUrl','http://'+self.host+a['href'].replace("/Author-","/AuthorStories-"))
+                    self.story.addToList('author',stripHTML(a))                
+                                            
+            except urllib2.HTTPError, e:
+                if e.code == 404:
+                    raise exceptions.StoryDoesNotExist(url)
+                else:
+                    raise e
+    
         try:
-            # going to pull part of the meta data from author list page.
-            logging.debug("**AUTHOR** URL: "+self.story.getMetadata('authorUrl'))
-            authordata = self._fetchUrl(self.story.getMetadata('authorUrl'))
-            descurl=self.story.getMetadata('authorUrl')
+            # going to pull part of the meta data from *primary* author list page.
+            logging.debug("**AUTHOR** URL: "+authorurl)
+            authordata = self._fetchUrl(authorurl)
+            descurl=authorurl
             authorsoup = bs.BeautifulSoup(authordata)
             # author can have several pages, scan until we find it.
             while( not authorsoup.find('a', href=re.compile(r"^/Story-"+self.story.getMetadata('storyId'))) ):
@@ -209,7 +230,7 @@ class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter):
                 if not BtVSNonX:
                     BtVS = False # Don't add BtVS if Non-Crossover, unless it's a BtVS/AtS Non-Crossover
 
-        print("BtVS: %s BtVSNonX: %s"%(BtVS,BtVSNonX))
+        #print("BtVS: %s BtVSNonX: %s"%(BtVS,BtVSNonX))
         if BtVS:
             self.story.addToList('category','Buffy: The Vampire Slayer')
             

fanficdownloader/adapters/adapter_twilightarchivescom.py

@@ -0,0 +1,190 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return TwilightArchivesComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class TwilightArchivesComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL. http://www.twilightarchives.com/read/9353
+        self._setURL('http://' + self.getSiteDomain() + '/read/'+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','twa')
+
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","Twilight")
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%d %b %y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.twilightarchives.com'
+       
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/read/1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://" + self.getSiteDomain()+"/read/")+r"\d+(/d+)?$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('h1')
+        self.story.setMetadata('title',a.string)
+        
+        # Find the chapters:
+        chapters=soup.find('ol', {'class' : 'chapters'})
+        if chapters != None:
+            for chapter in chapters.findAll('a', href=re.compile(r'/read/'+self.story.getMetadata('storyId')+"/\d+$")):
+                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+chapter['href']))
+        else:
+            self.chapterUrls.append((self.story.getMetadata('title'),url))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+                
+# rated, genre, warnings, seires
+
+        summary = soup.find('p', {'class' : 'images'})
+        self.setDescription(url,summary)
+
+        for c in soup.findAll('h2', {'class' : 'title'}):
+            div = c.nextSibling.nextSibling
+            
+            if 'Information' in c.text:
+                for dt in div.findAll('dt'):
+                    dd=dt.nextSibling.nextSibling
+                    
+                    if 'Author' in dt.text:
+                        a=dd.find('a')
+                        self.story.setMetadata('authorId',a['href'].split('/')[2])
+                        self.story.setMetadata('authorUrl','http://'+self.host+a['href'])
+                        self.story.setMetadata('author',a.text)
+                        
+                    if 'Words' in dt.text:
+                        self.story.setMetadata('numWords', dd.text)
+                        
+                    if 'Published' in dt.text:
+                        self.story.setMetadata('datePublished', makeDate(stripHTML(dd.text), self.dateformat))
+            
+                    if 'Updated' in dt.text:
+                        self.story.setMetadata('dateUpdated', makeDate(stripHTML(dd.text), self.dateformat))
+                    
+                    if 'Status' in dt.text:
+                        if 'Complete' in dd.text:
+                            self.story.setMetadata('status', 'Completed')
+                        else:
+                            self.story.setMetadata('status', 'In-Progress')
+
+            if 'Categories' in c.text:
+                for a in div.findAll('a'):
+                    self.story.addToList('category',a.text)
+                    
+            if 'Characters' in c.text:
+                for a in div.findAll('a'):
+                    self.story.addToList('category',a.text)
+
+            if 'Series' in c.text:
+                a=div.find('a')
+                series_name = a.text
+                series_url = 'http://'+self.host+a['href']
+                
+                seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+                storyas = seriessoup.find('tbody').findAll('a', href=re.compile(r'^/read/\d+$'))
+                i=1
+                for a in storyas:
+                    if a['href'] == ('/read/'+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        break
+                    i+=1
+                    
+        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
+        a=asoup.find('tbody').find('a', href=re.compile(r'^/read/'+self.story.getMetadata('storyId')))
+        self.story.setMetadata('rating',a.parent.nextSibling.nextSibling.text)
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'class' : 'size images medium'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_wizardtalesnet.py

@@ -0,0 +1,299 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return WizardTalesNetAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class WizardTalesNetAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','wzt')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%m/%d/%Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.wizardtales.net'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Registered Users Only' in data \
+                or 'There is no such account on our website' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['cookiecheck'] = '1'
+        params['submit'] = 'Submit'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
+        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "Member Account" not in d : #Member Account
+            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=4"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid & error in url.
+                addurl = addurl.replace("&","&")
+                url = self.url+'&index=1'+addurl
+                logging.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        pt = soup.find('div', {'id' : 'pagetitle'})
+        a = pt.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.text)
+        
+        # Find authorid and URL from... author url.
+        a = pt.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        rating=pt.text.split('[')[1].split(']')[0]
+        self.story.setMetadata('rating', rating)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        content=soup.find('div',{'class' : 'content'})
+        
+        for genre in content.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')):
+            self.story.addToList('genre',genre.string)
+
+        for warning in content.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')):
+            self.story.addToList('warnings',warning.string)
+        
+        labels = content.findAll('b')
+        
+        value = labels[0].previousSibling
+        svalue = ""
+        while value != None:
+            val = value
+            value = value.previousSibling
+        while "Categories" not in val:
+            svalue += str(val)
+            val = val.nextSibling
+        self.setDescription(url,svalue)
+        
+        
+        
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', stripHTML(value).split(': ')[1].split(';')[0])
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value).split(': ')[1].split(';')[0], self.dateformat))
+            
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value).split(': ')[1], self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            i=1
+            for a in storyas:
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        break
+                    i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.findAll('div', {'id' : 'story'})[1]
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_wraithbaitcom.py

@@ -119,10 +119,11 @@ class WraithBaitComAdapter(BaseSiteAdapter):
         self.story.setMetadata('title',a.string)
         
         # Find authorid and URL from... author url.
-        a = pt.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
+        alist = pt.findAll('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        for a in alist:
+            self.story.addToList('authorId',a['href'].split('=')[1])
+            self.story.addToList('authorUrl','http://'+self.host+'/'+a['href'])
+            self.story.addToList('author',a.string)
 
         rating=pt.text.split('[')[1].split(']')[0]
         self.story.setMetadata('rating', rating)
@@ -215,8 +216,7 @@ class WraithBaitComAdapter(BaseSiteAdapter):
 
         logging.debug('Getting chapter text from: %s' % url)
 
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        soup = bs.BeautifulSoup(self._fetchUrl(url))
         
         div = soup.find('div', {'id' : 'story'})
 

fanficdownloader/adapters/adapter_yourfanfictioncom.py

@@ -36,6 +36,13 @@ class YourFanfictionComAdapter(BaseSiteAdapter):
     def __init__(self, config, url):
         BaseSiteAdapter.__init__(self, config, url)
 
+        # yourfanfiction.com blocks the default user-agent.  However,
+        # when asked, they said it was just general anti-spam, not
+        # targeted as us and offered to 'whitelist our IP'.  Clearly,
+        # that wouldn't work, but it does let me do this in good
+        # conscience:
+        self.opener.addheaders = [('User-agent', 'FFDL/1.5')]
+
         self.decode = ["Windows-1252",
                        "utf8"] # 1252 is a superset of iso-8859-1.
                                # Most sites that claim to be
@@ -234,13 +241,17 @@ class YourFanfictionComAdapter(BaseSiteAdapter):
 
             # use BeautifulSoup HTML parser to make everything easier to find.
             seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
             i=1
             for a in storyas:
-                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                    self.setSeries(series_name, i)
-                    break
-                i+=1
+                print("series a['href']:%s"%a['href'])
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        break
+                    i+=1
             
         except:
             # I find it hard to care if the series parsing fails

fanficdownloader/adapters/base_adapter.py

@@ -134,9 +134,10 @@ class BaseSiteAdapter(Configurable):
                         code=detected['encoding']
                     else:
                         continue
+                logging.debug("try code:"+code)
                 return data.decode(code)
             except:
-                logging.debug("code failed:"+code)
+                logging.info("code failed:"+code)
                 pass
         logging.info("Could not decode story, tried:%s Stripping non-ASCII."%decode)
         return "".join([x for x in data if ord(x) < 128])
@@ -159,9 +160,9 @@ class BaseSiteAdapter(Configurable):
 
     def _fetchUrlRaw(self, url, parameters=None):
         if parameters != None:
-            return self.opener.open(url,urllib.urlencode(parameters)).read()
+            return self.opener.open(url.replace(' ','%20'),urllib.urlencode(parameters)).read()
         else:
-            return self.opener.open(url).read()
+            return self.opener.open(url.replace(' ','%20')).read()
     
     # parameters is a dict()
     def _fetchUrl(self, url, parameters=None):
@@ -234,6 +235,10 @@ class BaseSiteAdapter(Configurable):
     def getStoryMetadataOnly(self):
         if not self.metadataDone:
             self.extractChapterUrlsAndMetadata()
+            
+            if not self.story.getMetadataRaw('dateUpdated'):
+                self.story.setMetadata('dateUpdated',self.story.getMetadataRaw('datePublished'))
+
             self.metadataDone = True
         return self.story
 

fanficdownloader/configurable.py

@@ -54,6 +54,17 @@ class Configurable(object):
     def addConfigSection(self,section):
         self.sectionslist.insert(0,section)
 
+    def hasConfig(self, key):
+        for section in self.sectionslist:
+            try:
+                self.config.get(section,key)
+                #print("found %s in section [%s]"%(key,section))
+                return True
+            except:
+                pass
+
+        return False
+        
     def getConfig(self, key):
         val = ""
         for section in self.sectionslist:

fanficdownloader/geturls.py

@@ -15,7 +15,7 @@
 # limitations under the License.
 #
 
-
+import re
 import urlparse
 import urllib2 as u2
 import ConfigParser
@@ -37,7 +37,17 @@ def get_urls_from_page(url):
     for a in soup.findAll('a'):
         if a.has_key('href'):
             href = form_url(url,a['href'])
+            # this (should) catch normal story links, some javascript
+            # 'are you old enough' links, and 'Report This' links.
+            # The 'normalized' set prevents duplicates.
+            if 'story.php' in a['href']:
+                #print("trying:%s"%a['href'])
+                m = re.search(r"(?P<sid>(view)?story\.php\?(sid|psid|no|story|stid)=\d+)",a['href'])
+                if m != None:
+                    href = form_url(url,m.group('sid'))
+                    
             try:
+                href = href.replace('&index=1','')
                 adapter = adapters.getAdapter(config,href,"EPUB")
                 if adapter.story.getMetadata('storyUrl') not in normalized:
                     normalized.add(adapter.story.getMetadata('storyUrl'))

fanficdownloader/story.py

@@ -20,6 +20,7 @@ import urlparse
 import string
 from math import floor
 
+import exceptions
 from htmlcleanup import conditionalRemoveEntities, removeAllEntities
 
 # Create convert_image method depending on which graphics lib we can
@@ -222,9 +223,9 @@ class Story:
         
     def getMetadata(self, key, removeallentities=False, doreplacements=True):
         value = None
-        if self.getLists().has_key(key):
+        if self.isList(key):
             value = ', '.join(self.getList(key))
-        if self.metadata.has_key(key):
+        elif self.metadata.has_key(key):
             value = self.metadata[key]
             if value:
                 if key == "numWords":
@@ -246,11 +247,33 @@ class Story:
         All single value *and* list value metadata as strings.
         '''
         allmetadata = {}
+        
+        # special handling for authors/authorUrls
+        authlinkhtml="<a class='authorlink' href='%s'>%s</a>"
+        if 'author' in self.listables.keys(): # more than one author, assume multiple authorUrl too.
+            htmllist=[]
+            for i, v in enumerate(self.listables['author']):
+                aurl = self.listables['authorUrl'][i]                    
+                auth = v
+                # make sure doreplacements & removeallentities are honored.
+                if doreplacements:
+                    aurl=self.doReplacments(aurl)
+                    auth=self.doReplacments(auth)
+                if removeallentities:
+                    aurl=removeAllEntities(aurl)
+                    auth=removeAllEntities(auth)
+                
+                htmllist.append(authlinkhtml%(aurl,auth))
+            self.setMetadata('authorHTML',', '.join(htmllist))
+        else:
+           self.setMetadata('authorHTML',authlinkhtml%(self.getMetadata('authorUrl', removeallentities, doreplacements),
+                                                       self.getMetadata('author', removeallentities, doreplacements)))
+            
         for k in self.metadata.keys():
             allmetadata[k] = self.getMetadata(k, removeallentities, doreplacements)
         for l in self.listables.keys():
             allmetadata[l] = self.getMetadata(l, removeallentities, doreplacements)
-
+                
         return allmetadata
 
     # just for less clutter in adapters.
@@ -262,23 +285,36 @@ class Story:
         if value==None:
             return
         value = conditionalRemoveEntities(value)
-        if not self.listables.has_key(listname):
+        if not self.isList(listname):
             self.listables[listname]=[]
         # prevent duplicates.
         if not value in self.listables[listname]:
             self.listables[listname].append(value)
 
-    def getList(self,listname):
-        if not self.listables.has_key(listname):
-            return []
-        return filter( lambda x : x!=None and x!='' ,
-                       map(self.doReplacments,self.listables[listname]) )
+    def getList(self,listname, removeallentities=False, doreplacements=True):
+        retlist = []
+        if not self.isList(listname):
+            retlist = [self.getMetadata(listname,removeallentities=removeallentities)]
+        else:
+            retlist = self.listables[listname]
+
+        if doreplacements:
+            retlist = filter( lambda x : x!=None and x!='' ,
+                              map(self.doReplacments,retlist) )
+        if removeallentities:
+            retlist = filter( lambda x : x!=None and x!='' ,
+                              map(removeAllEntities,retlist) )
+
+        return retlist
 
     def getLists(self):
         lsts = {}
         for ln in self.listables.keys():
             lsts[ln] = self.getList(ln)
         return lsts
+
+    def isList(self,listname):
+        return self.listables.has_key(listname)
     
     def addChapter(self, title, html):
         self.chapters.append( (title,html) )
@@ -304,6 +340,11 @@ class Story:
     # pass fetch in from adapter in case we need the cookies collected
     # as well as it's a base_story class method.
     def addImgUrl(self,configurable,parenturl,url,fetch,cover=False,coverexclusion=None):
+
+        # otherwise it saves the image in the epub even though it
+        # isn't used anywhere.
+        if cover and configurable.getConfig('never_make_cover'):
+            return
         
         url = url.strip() # ran across an image with a space in the
                           # src. Browser handled it, so we'd better, too.
@@ -339,7 +380,10 @@ class Story:
         prefix='ffdl'
         if imgurl not in self.imgurls:
             parsedUrl = urlparse.urlparse(imgurl)
-            sizes = [ int(x) for x in configurable.getConfigList('image_max_size') ]
+            try:
+                sizes = [ int(x) for x in configurable.getConfigList('image_max_size') ]
+            except Exception, e:
+                raise exceptions.FailedToDownload("Failed to parse image_max_size from personal.ini:%s\nException: %s"%(configurable.getConfigList('image_max_size'),e))
             try:
                 (data,ext,mime) = convert_image(imgurl,
                                                 fetch(imgurl),

fanficdownloader/writers/base_writer.py

@@ -134,7 +134,7 @@ class BaseStoryWriter(Configurable):
         names as Story.metadata, but ENTRY should use label and value.
         """
         if self.getConfig("include_titlepage"):
-            self._write(out,START.substitute(self.story.metadata))
+            self._write(out,START.substitute(self.story.getAllMetadata()))
 
             if WIDE_ENTRY==None:
                 WIDE_ENTRY=ENTRY
@@ -149,20 +149,24 @@ class BaseStoryWriter(Configurable):
                             TEMPLATE=WIDE_ENTRY
                         else:
                             TEMPLATE=ENTRY
-                        if self.getConfigList(entry):
+                            
+                        if self.hasConfig(entry+"_label"):
                             label=self.getConfig(entry+"_label")
                         else:
+                            print("Using fallback label for %s_label"%entry)
                             label=self.titleLabels[entry]
 
                         # If the label for the title entry is empty, use the
-                        # 'no title' option if there is one.  
+                        # 'no title' option if there is one.
                         if label == "" and NO_TITLE_ENTRY:
                            TEMPLATE= NO_TITLE_ENTRY
                            
                         self._write(out,TEMPLATE.substitute({'label':label,
                                                              'value':self.story.getMetadata(entry)}))
+                else:
+                    self._write(out, entry)
 
-            self._write(out,END.substitute(self.story.metadata))
+            self._write(out,END.substitute(self.story.getAllMetadata()))
 
     def writeTOCPage(self, out, START, ENTRY, END):
         """
@@ -172,13 +176,13 @@ class BaseStoryWriter(Configurable):
         """
         # Only do TOC if there's more than one chapter and it's configured.
         if len(self.story.getChapters()) > 1 and self.getConfig("include_tocpage") and not self.metaonly :
-            self._write(out,START.substitute(self.story.metadata))
+            self._write(out,START.substitute(self.story.getAllMetadata()))
 
             for index, (title,html) in enumerate(self.story.getChapters()):
                 if html:
                     self._write(out,ENTRY.substitute({'chapter':title, 'index':"%04d"%(index+1)}))
 
-            self._write(out,END.substitute(self.story.metadata))
+            self._write(out,END.substitute(self.story.getAllMetadata()))
 
     # if no outstream is given, write to file.
     def writeStory(self,outstream=None, metaonly=False, outfilename=None, forceOverwrite=False):
@@ -195,7 +199,7 @@ class BaseStoryWriter(Configurable):
             
         if not outstream:
             close=True
-            logging.debug("Save directly to file: %s" % outfilename)
+            logging.info("Save directly to file: %s" % outfilename)
             if self.getConfig('make_directories'):
                 path=""
                 dirs = os.path.dirname(outfilename).split('/')

fanficdownloader/writers/writer_epub.py

@@ -52,7 +52,7 @@ class EpubWriter(BaseStoryWriter):
 <link href="stylesheet.css" type="text/css" charset="UTF-8" rel="stylesheet"/>
 </head>
 <body>
-<h3><a href="${storyUrl}">${title}</a> by <a href="${authorUrl}">${author}</a></h3>
+<h3><a href="${storyUrl}">${title}</a> by ${authorHTML}</h3>
 <div>
 ''')
 
@@ -79,7 +79,7 @@ ${value}<br />
 <link href="stylesheet.css" type="text/css" charset="UTF-8" rel="stylesheet"/>
 </head>
 <body>
-<h3><a href="${storyUrl}">${title}</a> by <a href="${authorUrl}">${author}</a></h3>
+<h3><a href="${storyUrl}">${title}</a> by ${authorHTML}</h3>
 <table class="full">
 ''')
 
@@ -185,7 +185,7 @@ ${value}<br />
 
         uniqueid= 'fanficdownloader-uid:%s-u%s-s%s' % (
             self.getMetadata('site'),
-            self.getMetadata('authorId'),
+            self.story.getList('authorId')[0],
             self.getMetadata('storyId'))
         
         contentdom = getDOMImplementation().createDocument(None, "package", None)
@@ -206,9 +206,15 @@ ${value}<br />
             metadata.appendChild(newTag(contentdom,"dc:title",text=self.getMetadata('title')))
 
         if self.getMetadata('author'):
-            metadata.appendChild(newTag(contentdom,"dc:creator",
-                                        attrs={"opf:role":"aut"},
-                                        text=self.getMetadata('author')))
+            if self.story.isList('author'):
+                for auth in self.story.getList('author'):
+                    metadata.appendChild(newTag(contentdom,"dc:creator",
+                                                attrs={"opf:role":"aut"},
+                                                text=auth))
+            else:
+                metadata.appendChild(newTag(contentdom,"dc:creator",
+                                            attrs={"opf:role":"aut"},
+                                            text=self.getMetadata('author')))
 
         metadata.appendChild(newTag(contentdom,"dc:contributor",text="fanficdownloader [http://fanficdownloader.googlecode.com]",attrs={"opf:role":"bkp"}))
         metadata.appendChild(newTag(contentdom,"dc:rights",text=""))
@@ -433,7 +439,7 @@ div { margin: 0pt; padding: 0pt; }
         del tocncxdom
 
         # write stylesheet.css file.
-        outputepub.writestr("OEBPS/stylesheet.css",self.EPUB_CSS.substitute(self.story.metadata)) 
+        outputepub.writestr("OEBPS/stylesheet.css",self.EPUB_CSS.substitute(self.story.getAllMetadata())) 
 
         # write title page.
         if self.getConfig("titlepage_use_table"):

fanficdownloader/writers/writer_html.py

@@ -43,7 +43,7 @@ ${output_css}
 </style>
 </head>
 <body>
-<h1><a href="${storyUrl}">${title}</a> by <a href="${authorUrl}">${author}</a></h1>
+<h1><a href="${storyUrl}">${title}</a> by ${authorHTML}</h1>
 ''')
 
         self.HTML_TITLE_PAGE_START = string.Template('''
@@ -82,7 +82,7 @@ ${output_css}
 
     def writeStoryImpl(self, out):
 
-        self._write(out,self.HTML_FILE_START.substitute(self.story.metadata))
+        self._write(out,self.HTML_FILE_START.substitute(self.story.getAllMetadata()))
 
         self.writeTitlePage(out,
                             self.HTML_TITLE_PAGE_START,
@@ -100,4 +100,4 @@ ${output_css}
                 self._write(out,self.HTML_CHAPTER_START.substitute({'chapter':title, 'index':"%04d"%(index+1)}))
                 self._write(out,html)
 
-        self._write(out,self.HTML_FILE_END.substitute(self.story.metadata))
+        self._write(out,self.HTML_FILE_END.substitute(self.story.getAllMetadata()))

fanficdownloader/writers/writer_mobi.py

@@ -43,7 +43,7 @@ class MobiWriter(BaseStoryWriter):
 <title>${title} by ${author}</title>
 </head>
 <body>
-<h3><a href="${storyUrl}">${title}</a> by <a href="${authorUrl}">${author}</a></h3>
+<h3><a href="${storyUrl}">${title}</a> by ${authorHTML}</h3>
 <div>
 ''')
 
@@ -69,7 +69,7 @@ ${value}<br />
 <title>${title} by ${author}</title>
 </head>
 <body>
-<h3><a href="${storyUrl}">${title}</a> by <a href="${authorUrl}">${author}</a></h3>
+<h3><a href="${storyUrl}">${title}</a> by ${authorHTML}</h3>
 <table class="full">
 ''')
 

fanficdownloader/writers/writer_txt.py

@@ -114,7 +114,7 @@ End file.
         
         wrapout = KludgeStringIO()
         
-        wrapout.write(self.TEXT_FILE_START.substitute(self.story.metadata))
+        wrapout.write(self.TEXT_FILE_START.substitute(self.story.getAllMetadata()))
 
         self.writeTitlePage(wrapout,
                             self.TEXT_TITLE_PAGE_START,
@@ -139,7 +139,7 @@ End file.
                 self._write(out,self.lineends(self.wraplines(removeAllEntities(self.TEXT_CHAPTER_START.substitute({'chapter':title, 'index':index+1})))))
                 self._write(out,self.lineends(html2text(html,wrap_width=self.wrap_width)))
 
-        self._write(out,self.lineends(self.wraplines(self.TEXT_FILE_END.substitute(self.story.metadata))))
+        self._write(out,self.lineends(self.wraplines(self.TEXT_FILE_END.substitute(self.story.getAllMetadata()))))
 
     def wraplines(self, text):
         

index.html

@@ -57,10 +57,8 @@
 	  <h3>New Sites</h3>
 	  <p>
 	    
-	    New sites samdean.archive.nu, www.yourfanfiction.com,
-	    www.destinysgateway.com, www.thealphagate.com,
-	    stargate-atlantis.org and www.ncisfiction.com added.
-	    That's 51 different supported sites now.  Thanks, Ida!
+	    New sites grangerenchanted.com, hlfiction.net and nha.magical-worlds.us.
+	    <br />Thanks, Ida!
 	    
 	  </p>
 	  <p>
@@ -71,7 +69,7 @@
 	    If you have any problems with this application, please
 	    report them in
 	    the <a href="http://groups.google.com/group/fanfic-downloader">FanFictionDownLoader Google Group</a>.  The
-	    <a href="http://4-4-15.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
+	    <a href="http://4-4-20.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
 	  </p>
 	  <div id='error'>
 	    {{ error_message }}
@@ -421,6 +419,93 @@
 	    Or use the URL of the story's chapter list, such as
 	    <br /><a href="http://www.ncisfiction.com/chapters.php?stid=1234">http://www.ncisfiction.com/chapters.php?stid=1234</a>
 	  </dd>
+	  <dt>www.fanfiktion.de</dt>
+	  <dd>
+	    Use the URL of the story's first chapter, such as
+	    <br /><a href="http://www.fanfiktion.de/s/46ccbef30000616306614050">http://www.fanfiktion.de/s/46ccbef30000616306614050</a>
+	  </dd>
+	  <dt>ponyfictionarchive.net</dt>
+	  <dd>
+	    Use the URL of the story's first chapter, such as
+	    <br /><a href="http://ponyfictionarchive.net/viewstory.php?sid=1234">http://ponyfictionarchive.net/viewstory.php?sid=1234</a>
+	    <br />Or:
+	    <br /><a href="http://explicit.ponyfictionarchive.net/viewstory.php?sid=1234">http://explicit.ponyfictionarchive.net/viewstory.php?sid=1234</a>
+	  </dd>
+	  <dt>sg1-heliopolis.com</dt>
+	  <dd>
+	    Use the URL of the story's first chapter, such as
+	    <br /><a href="http://sg1-heliopolis.com/archive/viewstory.php?sid=1234">http://sg1-heliopolis.com/archive/viewstory.php?sid=1234</a>
+	    <br />Or:
+	    <br /><a href="http://sg1-heliopolis.com/adult/viewstory.php?sid=1234">http://sg1-heliopolis.com/adult/viewstory.php?sid=1234</a>
+	    <br />Or:
+	    <br /><a href="http://sg1-heliopolis.com/atlantis/viewstory.php?sid=1234">http://sg1-heliopolis.com/atlantis/viewstory.php?sid=1234</a>
+	  </dd>
+	  <dt>ncisfic.com</dt>
+	  <dd>
+	    Use the URL of the story's first chapter, such as
+	    <br /><a href="http://ncisfic.com/viewstory.php?storyid=1234">http://ncisfic.com/viewstory.php?storyid=1234</a>
+	  </dd>
+	  <dt>national-library.net</dt>
+	  <dd>
+	    Use the URL of the story's first chapter, such as
+	    <br /><a href="http://national-library.net/viewstory.php?storyid=1234">http://national-library.net/viewstory.php?storyid=1234</a>
+	  </dd>
+	  <dt>dark-solace.org</dt>
+	  <dd>
+	    Use the URL of the story's first chapter, such as
+	    <br /><a href="http://dark-solace.org/elysian/viewstory.php?sid=1234">http://dark-solace.org/elysian/viewstory.php?sid=1234</a>
+	  </dd>
+	  <dt>pretendercentre.com</dt>
+	  <dd>
+	    Use the URL of the story's first chapter, such as
+	    <br /><a href="http://pretendercentre.com/missingpieces/viewstory.php?sid=1234">http://pretendercentre.com/missingpieces/viewstory.php?sid=1234</a>
+	  </dd>
+	  <dt>themasque.net</dt>
+	  <dd>
+	    Use the URL of the story's first chapter, such as
+	    <br /><a href="http://themasque.net/wiktt/efiction/viewstory.php?sid=1234">http://themasque.net/wiktt/efiction/viewstory.php?sid=1234</a>
+	  </dd>
+	  <dt>finestories.com</dt>
+	  <dd>
+	    Use the URL of the story's first chapter, such as
+	    <br /><a href="http://finestories.com/s/1234">http://finestories.com/s/1234</a>
+	  </dd>
+	  <dt>www.hpfanficarchive.com</dt>
+	  <dd>
+	    Use the URL of the story's first chapter, such as
+	    <br /><a href="http://www.hpfanficarchive.com/stories/viewstory.php?sid=1234">http://www.hpfanficarchive.com/stories/viewstory.php?sid=1234</a>
+	  </dd>
+	  <dt>svufiction.com</dt>
+	  <dd>
+	    Use the URL of the story's first chapter, such as
+	    <br /><a href="http://svufiction.com/viewstory.php?sid=1234">http://svufiction.com/viewstory.php?sid=1234</a>
+	  </dd>
+	  <dt>www.twilightarchives.com</dt>
+	  <dd>
+	    Use the URL of the story's first chapter, such as
+	    <br /><a href="http://www.twilightarchives.com/read/1234">http://www.twilightarchives.com/read/1234</a>
+	  </dd>
+	  <dt>www.wizardtales.net</dt>
+	  <dd>
+	    Use the URL of the story's first chapter, such as
+	    <br /><a href="http://www.wizardtales.net/viewstory.php?sid=1234">http://www.wizardtales.net/viewstory.php?sid=1234</a>
+	  </dd>
+	  <dt>grangerenchanted.com</dt>
+	  <dd>
+	    Use the URL of the story's chapter list, such as
+	    <br /><a href="http://grangerenchanted.com/enchant/viewstory.php?sid=1234">http://grangerenchanted.com/enchant/viewstory.php?sid=1234</a> or
+	    <br /><a href="http://malfoymanor.grangerenchanted.com/enchant/viewstory.php?sid=1234">http://grangerenchanted.com/enchant/viewstory.php?sid=1234</a>
+	  </dd>
+	  <dt>hlfiction.net</dt>
+	  <dd>
+	    Use the URL of the story's first chapter, such as
+	    <br /><a href="http://hlfiction.net/viewstory.php?sid=1234">http://hlfiction.net/viewstory.php?sid=1234</a>
+	  </dd>
+	  <dt>nha.magical-worlds.us</dt>
+	  <dd>
+	    Use the URL of the story's chapter list, such as
+	    <br /><a href="http://nha.magical-worlds.us/viewstory.php?sid=1234">http://nha.magical-worlds.us/viewstory.php?sid=1234</a>
+	  </dd>
 	</dl>
 	<p>
 	  A few additional things to know, which will make your life substantially easier:

makeplugin.py

@@ -24,7 +24,7 @@ from makezip import createZipFile
 
 if __name__=="__main__":
     filename="FanFictionDownLoader.zip"
-    exclude=['*.pyc','*~','*.xcf']
+    exclude=['*.pyc','*~','*.xcf','*[0-9].png']
     # from top dir. 'w' for overwrite
     createZipFile(filename,"w",
                   ['plugin-defaults.ini','plugin-example.ini','epubmerge.py','fanficdownloader'],

plugin-defaults.ini

@@ -71,7 +71,9 @@ extratags_label:Extra Tags
 version_label:FFDL Version
 
 ## items to include in the title page
-## Empty entries will *not* appear, even if in the list.
+## Empty metadata entries will *not* appear, even if in the list.
+## You can include extra text or HTML that will be included as-is in
+## the title page. Eg: titlepage_entries: ...,<br />,summary,<br />,...
 ## All current formats already include title and author.
 titlepage_entries: series,category,genre,language,characters,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,description
 
@@ -297,6 +299,19 @@ extratags: FanFiction,Testing,HTML
 
 [archive.skyehawke.com]
 
+[archiveofourown.org]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
 [ashwinder.sycophanthex.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -328,6 +343,14 @@ extratags: FanFiction,Testing,HTML
 ## cover image.  This lets you exclude them.
 cover_exclusion_regexp:/images/.*?ribbon.gif
 
+[dark-solace.org]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
 [dramione.org]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -378,18 +401,47 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 #username:YourName
 #password:yourpassword
 
+[finestories.com]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+[grangerenchanted.com]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
+[hlfiction.net]
+
 [lumos.sycophanthex.com]
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+[national-library.net]
+
+[ncisfic.com]
+
 [nfacommunity.com]
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+[nha.magical-worlds.us]
+
 [occlumency.sycophanthex.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -411,12 +463,56 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 ## personal.ini, not defaults.ini.
 #is_adult:true
 
+[ponyfictionarchive.net]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+[pretendercentre.com]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
 [samdean.archive.nu]
 
+[sg1-heliopolis.com]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
 [stargate-atlantis.org]
 
+[svufiction.com]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
 [thehexfiles.net]
 
+[themasque.net]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
 [thequidditchpitch.org]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -438,12 +534,6 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 
 [www.thealphagate.com]
 
-[www.archiveofourown.org]
-## Some sites do not require a login, but do require the user to
-## confirm they are adult for adult content.  In commandline version,
-## this should go in your personal.ini, not defaults.ini.
-#is_adult:true
-
 [www.checkmated.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -465,6 +555,14 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 ## Comment this out or change it to false to use them anyway.
 never_make_cover: true
 
+[www.fanfiktion.de]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
 [www.ficbook.net]
 
 [www.fictionalley.org]
@@ -508,6 +606,8 @@ extratags:
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+[www.hpfanficarchive.com]
+
 [www.ik-eternal.net]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -597,6 +697,8 @@ collect_series: false
 #username:YourName
 #password:yourpassword
 
+[www.twilightarchives.com]
+
 [www.twilighted.net]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -627,6 +729,19 @@ collect_series: false
 
 [www.whofic.com]
 
+[www.wizardtales.net]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
 [www.wraithbait.com]
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,