Fix extratags, document extra_subject_tags in .ini files, bump versions.

Fix dokuga.com--can't use setMetadata for temp variable now.
Add replace_metadata by metadata key feature, more meta for AO3, fix www handling.
2026-09-15 11:36:30 +08:00 · 2012-09-26 15:51:47 -05:00 · 2012-09-25 11:03:42 -05:00 · 2012-09-25 11:03:08 -05:00 · 2012-09-23 14:55:07 -05:00 · 2012-09-22 14:41:32 -05:00
111 changed files with 17306 additions and 952 deletions
@@ -0,0 +1,78 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01//EN">
+<html>
+  <head>
+    <link href="/css/index.css" rel="stylesheet" type="text/css">
+    <title>FanFictionDownLoader (fanfiction.net, fanficauthors, fictionalley, ficwad to epub and HTML)</title>
+    <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
+    <script type="text/javascript">
+
+      var _gaq = _gaq || [];
+      _gaq.push(['_setAccount', 'UA-12136939-1']);
+      _gaq.push(['_trackPageview']);
+
+      (function() {
+      var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true;
+      ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
+      var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s);
+      })();
+
+    </script>
+  </head>
+  <body>
+    <div id='main'>
+      <h1>
+	<a href="/" style="text-decoration: none; color: black;">FanFictionDownLoader</a>
+      </h1>
+      
+      <script type="text/javascript"><!--
+                                        google_ad_client = "ca-pub-0320924304307555";
+                                        /* Standard */
+                                        google_ad_slot = "8974025478";
+                                        google_ad_width = 468;
+                                        google_ad_height = 60;
+                                        //-->
+      </script>
+      <script type="text/javascript"
+              src="http://pagead2.googlesyndication.com/pagead/show_ads.js">
+      </script>
+      <!--			<div id='yourfile'> -->
+      {{yourfile}}
+      <!--			</div> -->
+      
+      <div id='helpbox'>
+	{% for fic in fics %}
+	<p>
+  	  <a href="{{ fic.url }}" title="Link to original story"><span class="recent"><i>{{ fic.title }}</i></span></a>
+	      by <a href="{{ fic.authorUrl }}">{{ fic.author }}</a>  <b>Download Count:</b> {{ fic.count }} <br />
+	  <b>Word Count:</b> {{ fic.numWords }} <b>Chapter Count:</b> {{ fic.numChapters }}<br />
+	  {% if fic.category %} <b>Categories:</b> {{ fic.category }} <br /> {% endif %}
+	  {% if fic.genre %} <b>Genres:</b> {{ fic.genre }} <br /> {% endif %}
+	  {% if fic.language %} <b>Language:</b> {{ fic.language }} <br /> {% endif %}
+	  {% if fic.series %} <b>Series:</b> {{ fic.series }} <br /> {% endif %}
+	  {% if fic.characters %} <b>Characters:</b> {{ fic.characters }} <br /> {% endif %}
+	  {% if fic.status %} <b>Status:</b> {{ fic.status }} <br /> {% endif %}
+	  {% if fic.datePublished %} <b>Published:</b> {{ fic.datePublished }} <br /> {% endif %}
+	  {% if fic.dateUpdated %} <b>Last Updated:</b> {{ fic.dateUpdated }} <br /> {% endif %}
+	  {% if fic.dateCreated %} <b>Last Downloaded:</b> {{ fic.dateCreated }} <br /> {% endif %}
+	  {% if fic.rating %} <b>Rating:</b> {{ fic.rating }} <br /> {% endif %}
+	  {% if fic.warnings %} <b>Warnings:</b> {{ fic.warnings }} <br /> {% endif %}
+	  {% if fic.description %} <b>Summary:</b> {{ fic.description }} <br /> {% endif %}
+	</p>
+	{% endfor %}
+      </div>
+      
+      <script type="text/javascript"><!--
+                                        google_ad_client = "ca-pub-0320924304307555";
+                                        /* Standard */
+                                        google_ad_slot = "8974025478";
+                                        google_ad_width = 468;
+                                        google_ad_height = 60;
+                                        //-->
+      </script>
+      <script type="text/javascript"
+              src="http://pagead2.googlesyndication.com/pagead/show_ads.js">
+      </script>				
+      
+    </div>
+  </body>
+</html>
@@ -1,6 +1,6 @@
 # ffd-retief-hrd fanfictiondownloader
 application: fanfictiondownloader
-version: 4-4-5
+version: 4-4-27
 runtime: python27
 api_version: 1
 threadsafe: true
@@ -35,8 +35,8 @@ handlers:
 - url: /.*
  script: main.app

-builtins:
- datastore_admin: on
+#builtins:
+#- datastore_admin: on

 libraries:
 - name: django
@@ -27,8 +27,8 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
    description         = 'UI plugin to download FanFiction stories from various sites.'
    supported_platforms = ['windows', 'osx', 'linux']
    author              = 'Jim Miller'
-    version             = (1, 5, 14)
-    minimum_calibre_version = (0, 8, 30)
+    version             = (1, 6, 11)
+    minimum_calibre_version = (0, 8, 57)

    #: This field defines the GUI plugin class that contains all the code
    #: that actually does something. Its format is module_path:class_name
@@ -1,10 +1,10 @@
 <hr />

-<p>Created by Jim Miller, borrowing heavily from Grant Drake's
+<p>Plugin created by Jim Miller, borrowing heavily from Grant Drake's
 '<a href="http://www.mobileread.com/forums/showthread.php?t=134856">Reading List</a>',
 '<a href="http://www.mobileread.com/forums/showthread.php?t=126727">Extract ISBN</a>' and
 '<a href="http://www.mobileread.com/forums/showthread.php?t=134000">Count Pages</a>'
-plugins.</p>
+plugins.  bbcodeutils code contributed by Pau Sanchez.</p>

 <p>
 Calibre officially distributes plugins from the mobileread.com forum site.
@@ -11,9 +11,11 @@ import os
 from PyQt4 import QtGui
 from PyQt4.Qt import (Qt, QIcon, QPixmap, QLabel, QDialog, QHBoxLayout,
                      QTableWidgetItem, QFont, QLineEdit, QComboBox,
-                      QVBoxLayout, QDialogButtonBox, QStyledItemDelegate, QDateTime)
+                      QVBoxLayout, QDialogButtonBox, QStyledItemDelegate, QDateTime,
+                      QTextEdit,
+                      QListWidget, QAbstractItemView)
 from calibre.constants import iswindows
-from calibre.gui2 import gprefs, error_dialog, UNDEFINED_QDATETIME
+from calibre.gui2 import gprefs, error_dialog, UNDEFINED_QDATETIME, info_dialog
 from calibre.gui2.actions import menu_action_unique_name
 from calibre.gui2.keyboard import ShortcutConfig
 from calibre.utils.config import config_dir
@@ -445,3 +447,97 @@ class DateDelegate(QStyledItemDelegate):
            model.setData(index, UNDEFINED_QDATETIME, Qt.EditRole)
        else:
            model.setData(index, QDateTime(val), Qt.EditRole)
+            
+class PrefsViewerDialog(SizePersistedDialog):
+
+    def __init__(self, gui, namespace):
+        SizePersistedDialog.__init__(self, gui, 'Prefs Viewer dialog')
+        self.setWindowTitle('Preferences for: '+namespace)
+
+        self.gui = gui
+        self.db = gui.current_db
+        self.namespace = namespace
+        self._init_controls()
+        self.resize_dialog()
+
+        self._populate_settings()
+
+        if self.keys_list.count():
+            self.keys_list.setCurrentRow(0)
+
+    def _init_controls(self):
+        layout = QVBoxLayout(self)
+        self.setLayout(layout)
+
+        ml = QHBoxLayout()
+        layout.addLayout(ml, 1)
+
+        self.keys_list = QListWidget(self)
+        self.keys_list.setSelectionMode(QAbstractItemView.SingleSelection)
+        self.keys_list.setFixedWidth(150)
+        self.keys_list.setAlternatingRowColors(True)
+        ml.addWidget(self.keys_list)
+        self.value_text = QTextEdit(self)
+        self.value_text.setTabStopWidth(24)
+        self.value_text.setReadOnly(True)
+        ml.addWidget(self.value_text, 1)
+
+        button_box = QDialogButtonBox(QDialogButtonBox.Ok)
+        button_box.accepted.connect(self.accept)
+        self.clear_button = button_box.addButton('Clear', QDialogButtonBox.ResetRole)
+        self.clear_button.setIcon(get_icon('trash.png'))
+        self.clear_button.setToolTip('Clear all settings for this plugin')
+        self.clear_button.clicked.connect(self._clear_settings)
+        layout.addWidget(button_box)
+
+    def _populate_settings(self):
+        self.keys_list.clear()
+        ns_prefix = self._get_ns_prefix()
+        keys = sorted([k[len(ns_prefix):] for k in self.db.prefs.iterkeys()
+                       if k.startswith(ns_prefix)])
+        for key in keys:
+            self.keys_list.addItem(key)
+        self.keys_list.setMinimumWidth(self.keys_list.sizeHintForColumn(0))
+        self.keys_list.currentRowChanged[int].connect(self._current_row_changed)
+
+    def _current_row_changed(self, new_row):
+        if new_row < 0:
+            self.value_text.clear()
+            return
+        key = unicode(self.keys_list.currentItem().text())
+        val = self.db.prefs.get_namespaced(self.namespace, key, '')
+        self.value_text.setPlainText(self.db.prefs.to_raw(val))
+        
+    def _get_ns_prefix(self):
+        return 'namespaced:%s:'% self.namespace
+        
+    def _clear_settings(self):
+        from calibre.gui2.dialogs.confirm_delete import confirm
+        message = '<p>Are you sure you want to clear your settings in this library for this plugin?</p>' \
+                  '<p>Any settings in other libraries or stored in a JSON file in your calibre plugins ' \
+                  'folder will not be touched.</p>' \
+                  '<p>You must restart calibre afterwards.</p>'
+        if not confirm(message, self.namespace+'_clear_settings', self):
+            return
+        ns_prefix = self._get_ns_prefix()
+        keys = [k for k in self.db.prefs.iterkeys() if k.startswith(ns_prefix)]
+        for k in keys:
+            del self.db.prefs[k]
+        self._populate_settings()
+        d = info_dialog(self, 'Settings deleted', 
+                        '<p>All settings for this plugin in this library have been cleared.</p>'
+                        '<p>Please restart calibre now.</p>',
+                        show_copy_button=False)
+        b = d.bb.addButton(_('Restart calibre now'), d.bb.AcceptRole)
+        b.setIcon(QIcon(I('lt.png')))
+        d.do_restart = False
+        def rf():
+            d.do_restart = True
+        b.clicked.connect(rf)
+        d.set_details('')
+        d.exec_()
+        b.clicked.disconnect()
+        self.close()
+        if d.do_restart:
+            self.gui.quit(restart=True)
+        
@@ -4,118 +4,141 @@ from __future__ import (unicode_literals, division, absolute_import,
                        print_function)

 __license__   = 'GPL v3'
-__copyright__ = '2011, Jim Miller'
+__copyright__ = '2012, Jim Miller'
 __docformat__ = 'restructuredtext en'

 import traceback, copy
+from collections import OrderedDict

-from PyQt4.Qt import (QDialog, QWidget, QVBoxLayout, QHBoxLayout, QLabel, QLineEdit, QFont,
-                      QTextEdit, QComboBox, QCheckBox, QPushButton, QTabWidget, QVariant)
+from PyQt4.Qt import (QDialog, QWidget, QVBoxLayout, QHBoxLayout, QLabel, QLineEdit, QFont, QWidget,
+                      QTextEdit, QComboBox, QCheckBox, QPushButton, QTabWidget, QVariant, QScrollArea)

 from calibre.gui2 import dynamic, info_dialog
 from calibre.utils.config import JSONConfig
 from calibre.gui2.ui import get_gui

 from calibre_plugins.fanfictiondownloader_plugin.dialogs \
-    import (SKIP, ADDNEW, UPDATE, UPDATEALWAYS, OVERWRITE, OVERWRITEALWAYS,
-             CALIBREONLY,collision_order)
+    import (UPDATE, UPDATEALWAYS, OVERWRITE, collision_order)
+
+from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.adapters import getConfigSections

 from calibre_plugins.fanfictiondownloader_plugin.common_utils \
-    import ( get_library_uuid, KeyboardConfigDialog )
+    import ( get_library_uuid, KeyboardConfigDialog, PrefsViewerDialog )

 from calibre.gui2.complete import MultiCompleteLineEdit

-# This is where all preferences for this plugin will be stored
+PREFS_NAMESPACE = 'FanFictionDownLoaderPlugin'
+PREFS_KEY_SETTINGS = 'settings'
+
+# Set defaults used by all.  Library specific settings continue to
+# take from here.
+default_prefs = {}
+default_prefs['personal.ini'] = get_resources('plugin-example.ini')
+
+default_prefs['updatemeta'] = True
+default_prefs['updatecover'] = False
+default_prefs['updateepubcover'] = False
+default_prefs['keeptags'] = False
+default_prefs['urlsfromclip'] = True
+default_prefs['updatedefault'] = True
+default_prefs['fileform'] = 'epub'
+default_prefs['collision'] = OVERWRITE
+default_prefs['deleteotherforms'] = False
+default_prefs['adddialogstaysontop'] = False
+default_prefs['includeimages'] = False
+default_prefs['lookforurlinhtml'] = False
+default_prefs['injectseries'] = False
+
+default_prefs['send_lists'] = ''
+default_prefs['read_lists'] = ''
+default_prefs['addtolists'] = False
+default_prefs['addtoreadlists'] = False
+default_prefs['addtolistsonread'] = False
+
+default_prefs['gcnewonly'] = False
+default_prefs['gc_site_settings'] = {}
+default_prefs['allow_gc_from_ini'] = True
+
+default_prefs['countpagesstats'] = []
+
+default_prefs['errorcol'] = ''
+default_prefs['custom_cols'] = {}
+default_prefs['custom_cols_newonly'] = {}
+default_prefs['allow_custcol_from_ini'] = True
+
+default_prefs['std_cols_newonly'] = {}
+
+def set_library_config(library_config):
+    get_gui().current_db.prefs.set_namespaced(PREFS_NAMESPACE,
+                                              PREFS_KEY_SETTINGS,
+                                              library_config)
+    
+def get_library_config():
+    db = get_gui().current_db
+    library_id = get_library_uuid(db)
+    library_config = None
+    # Check whether this is a configuration needing to be migrated
+    # from json into database.  If so: get it, set it, rename it in json.
+    if library_id in old_prefs:
+        #print("get prefs from old_prefs")
+        library_config = old_prefs[library_id]
+        set_library_config(library_config)
+        old_prefs["migrated to library db %s"%library_id] = old_prefs[library_id]
+        del old_prefs[library_id]
+
+    if library_config is None:
+        #print("get prefs from db")
+        library_config = db.prefs.get_namespaced(PREFS_NAMESPACE, PREFS_KEY_SETTINGS,
+                                                 copy.deepcopy(default_prefs))
+    return library_config
+
+# This is where all preferences for this plugin *were* stored
 # Remember that this name (i.e. plugins/fanfictiondownloader_plugin) is also
 # in a global namespace, so make it as unique as possible.
 # You should always prefix your config file name with plugins/,
 # so as to ensure you dont accidentally clobber a calibre config file
-all_prefs = JSONConfig('plugins/fanfictiondownloader_plugin')
-
-# Set defaults used by all.  Library specific settings continue to
-# take from here.
-all_prefs.defaults['personal.ini'] = get_resources('plugin-example.ini')
-
-all_prefs.defaults['updatemeta'] = True
-all_prefs.defaults['updatecover'] = False
-all_prefs.defaults['keeptags'] = False
-all_prefs.defaults['urlsfromclip'] = True
-all_prefs.defaults['updatedefault'] = True
-all_prefs.defaults['fileform'] = 'epub'
-all_prefs.defaults['collision'] = OVERWRITE
-all_prefs.defaults['deleteotherforms'] = False
-all_prefs.defaults['adddialogstaysontop'] = False
-all_prefs.defaults['includeimages'] = False
-
-all_prefs.defaults['send_lists'] = ''
-all_prefs.defaults['read_lists'] = ''
-all_prefs.defaults['addtolists'] = False
-all_prefs.defaults['addtoreadlists'] = False
-all_prefs.defaults['addtolistsonread'] = False
-
-all_prefs.defaults['custom_cols'] = {}
-
-# The list of settings to copy from all_prefs or the previous library
-# when config is called for the first time on a library.
-copylist = ['personal.ini',
-            'updatemeta',
-            'updatecover',
-            'keeptags',
-            'urlsfromclip',
-            'updatedefault',
-            'fileform',
-            'collision',
-            'deleteotherforms',
-            'adddialogstaysontop',
-            'includeimages']
+old_prefs = JSONConfig('plugins/fanfictiondownloader_plugin')

 # fake out so I don't have to change the prefs calls anywhere.  The
 # Java programmer in me is offended by op-overloading, but it's very
 # tidy.
 class PrefsFacade():
-    def __init__(self,all_prefs):
-        self.all_prefs = all_prefs
-        self.lastlibid = None
-
-    def _get_copylist_prefs(self,frompref):
-        return filter( lambda x : x[0] in copylist, frompref.items() )
+    def __init__(self,default_prefs):
+        self.default_prefs = default_prefs
+        self.libraryid = None
+        self.current_prefs = None
        
    def _get_prefs(self):
        libraryid = get_library_uuid(get_gui().current_db)
-        if libraryid not in self.all_prefs:
-            if self.lastlibid == None:
-                self.all_prefs[libraryid] = dict(self._get_copylist_prefs(self.all_prefs))
-            else:
-                self.all_prefs[libraryid] = dict(self._get_copylist_prefs(self.all_prefs[self.lastlibid]))
-            self.lastlibid = libraryid
-            
-        return self.all_prefs[libraryid]
-
-    def _save_prefs(self,prefs):
-        libraryid = get_library_uuid(get_gui().current_db)
-        self.all_prefs[libraryid] = prefs
+        if self.current_prefs == None or self.libraryid != libraryid:
+            #print("self.current_prefs == None(%s) or self.libraryid != libraryid(%s)"%(self.current_prefs == None,self.libraryid != libraryid))
+            self.libraryid = libraryid
+            self.current_prefs = get_library_config()
+        return self.current_prefs
        
    def __getitem__(self,k):            
        prefs = self._get_prefs()
        if k not in prefs:
-            # pulls from all_prefs.defaults automatically if not set
-            # in all_prefs
-            return self.all_prefs[k]
+            # pulls from default_prefs.defaults automatically if not set
+            # in default_prefs
+            return self.default_prefs[k]
        return prefs[k]

    def __setitem__(self,k,v):
        prefs = self._get_prefs()
        prefs[k]=v
-        self._save_prefs(prefs)
+        # self._save_prefs(prefs)

-    # to be avoided--can cause unexpected results as possibly ancient
-    # all_pref settings may be pulled.
    def __delitem__(self,k):
        prefs = self._get_prefs()
-        del prefs[k]
-        self._save_prefs(prefs)
+        if k in prefs:
+            del prefs[k]

-prefs = PrefsFacade(all_prefs)
+    def save_to_db(self):
+        set_library_config(self._get_prefs())
+        
+
+prefs = PrefsFacade(default_prefs)
    
 class ConfigWidget(QWidget):

@@ -139,13 +162,26 @@ class ConfigWidget(QWidget):
        self.personalini_tab = PersonalIniTab(self, plugin_action)
        tab_widget.addTab(self.personalini_tab, 'personal.ini')
        
-        self.list_tab = ListTab(self, plugin_action)
-        tab_widget.addTab(self.list_tab, 'Reading Lists')
+        self.readinglist_tab = ReadingListTab(self, plugin_action)
+        tab_widget.addTab(self.readinglist_tab, 'Reading Lists')
        if 'Reading List' not in plugin_action.gui.iactions:
-            self.list_tab.setEnabled(False)
+            self.readinglist_tab.setEnabled(False)

-        self.columns_tab = ColumnsTab(self, plugin_action)
-        tab_widget.addTab(self.columns_tab, 'Custom Columns')
+        self.generatecover_tab = GenerateCoverTab(self, plugin_action)
+        tab_widget.addTab(self.generatecover_tab, 'Generate Cover')
+        if 'Generate Cover' not in plugin_action.gui.iactions:
+            self.generatecover_tab.setEnabled(False)
+
+        self.countpages_tab = CountPagesTab(self, plugin_action)
+        tab_widget.addTab(self.countpages_tab, 'Count Pages')
+        if 'Count Pages' not in plugin_action.gui.iactions:
+            self.countpages_tab.setEnabled(False)
+
+        self.std_columns_tab = StandardColumnsTab(self, plugin_action)
+        tab_widget.addTab(self.std_columns_tab, 'Standard Columns')
+
+        self.cust_columns_tab = CustomColumnsTab(self, plugin_action)
+        tab_widget.addTab(self.cust_columns_tab, 'Custom Columns')

        self.other_tab = OtherTab(self, plugin_action)
        tab_widget.addTab(self.other_tab, 'Other')
@@ -158,22 +194,25 @@ class ConfigWidget(QWidget):
        prefs['collision'] = unicode(self.basic_tab.collision.currentText())
        prefs['updatemeta'] = self.basic_tab.updatemeta.isChecked()
        prefs['updatecover'] = self.basic_tab.updatecover.isChecked()
+        prefs['updateepubcover'] = self.basic_tab.updateepubcover.isChecked()
        prefs['keeptags'] = self.basic_tab.keeptags.isChecked()
        prefs['urlsfromclip'] = self.basic_tab.urlsfromclip.isChecked()
        prefs['updatedefault'] = self.basic_tab.updatedefault.isChecked()
        prefs['deleteotherforms'] = self.basic_tab.deleteotherforms.isChecked()
        prefs['adddialogstaysontop'] = self.basic_tab.adddialogstaysontop.isChecked()
        prefs['includeimages'] = self.basic_tab.includeimages.isChecked()
+        prefs['lookforurlinhtml'] = self.basic_tab.lookforurlinhtml.isChecked()
+        prefs['injectseries'] = self.basic_tab.injectseries.isChecked()

-        if self.list_tab:
+        if self.readinglist_tab:
            # lists
-            prefs['send_lists'] = ', '.join(map( lambda x : x.strip(), filter( lambda x : x.strip() != '', unicode(self.list_tab.send_lists_box.text()).split(','))))
-            prefs['read_lists'] = ', '.join(map( lambda x : x.strip(), filter( lambda x : x.strip() != '', unicode(self.list_tab.read_lists_box.text()).split(','))))
+            prefs['send_lists'] = ', '.join(map( lambda x : x.strip(), filter( lambda x : x.strip() != '', unicode(self.readinglist_tab.send_lists_box.text()).split(','))))
+            prefs['read_lists'] = ', '.join(map( lambda x : x.strip(), filter( lambda x : x.strip() != '', unicode(self.readinglist_tab.read_lists_box.text()).split(','))))
            # print("send_lists: %s"%prefs['send_lists'])
            # print("read_lists: %s"%prefs['read_lists'])
-            prefs['addtolists'] = self.list_tab.addtolists.isChecked()
-            prefs['addtoreadlists'] = self.list_tab.addtoreadlists.isChecked()
-            prefs['addtolistsonread'] = self.list_tab.addtolistsonread.isChecked()
+            prefs['addtolists'] = self.readinglist_tab.addtolists.isChecked()
+            prefs['addtoreadlists'] = self.readinglist_tab.addtoreadlists.isChecked()
+            prefs['addtolistsonread'] = self.readinglist_tab.addtolistsonread.isChecked()

        # personal.ini
        ini = unicode(self.personalini_tab.ini.toPlainText())
@@ -183,15 +222,61 @@ class ConfigWidget(QWidget):
            # if they've removed everything, reset to default.
            prefs['personal.ini'] = get_resources('plugin-example.ini')

+        # Generate Covers tab
+        prefs['gcnewonly'] = self.generatecover_tab.gcnewonly.isChecked()
+        gc_site_settings = {}
+        for (site,combo) in self.generatecover_tab.gc_dropdowns.iteritems():
+            val = unicode(combo.itemData(combo.currentIndex()).toString())
+            if val != 'none':
+                gc_site_settings[site] = val
+                #print("gc_site_settings[%s]:%s"%(site,gc_site_settings[site]))
+        prefs['gc_site_settings'] = gc_site_settings
+        prefs['allow_gc_from_ini'] = self.generatecover_tab.allow_gc_from_ini.isChecked()
+
+        # Count Pages tab
+        countpagesstats = []
+        
+        if self.countpages_tab.pagecount.isChecked():
+            countpagesstats.append('PageCount')
+        if self.countpages_tab.wordcount.isChecked():
+            countpagesstats.append('WordCount')
+        if self.countpages_tab.fleschreading.isChecked():
+            countpagesstats.append('FleschReading')
+        if self.countpages_tab.fleschgrade.isChecked():
+            countpagesstats.append('FleschGrade')
+        if self.countpages_tab.gunningfog.isChecked():
+            countpagesstats.append('GunningFog')
+            
+        prefs['countpagesstats'] = countpagesstats
+        
+        # Standard Columns tab
+        colsnewonly = {}
+        for (col,checkbox) in self.std_columns_tab.stdcol_newonlycheck.iteritems():
+            colsnewonly[col] = checkbox.isChecked()
+        prefs['std_cols_newonly'] = colsnewonly
+
        # Custom Columns tab
+        # error column
+        prefs['errorcol'] = unicode(self.cust_columns_tab.errorcol.itemData(self.cust_columns_tab.errorcol.currentIndex()).toString())
+
+        # cust cols tab
        colsmap = {}
-        for (col,combo) in self.columns_tab.custcol_dropdowns.iteritems():
+        for (col,combo) in self.cust_columns_tab.custcol_dropdowns.iteritems():
            val = unicode(combo.itemData(combo.currentIndex()).toString())
            if val != 'none':
                colsmap[col] = val
                #print("colsmap[%s]:%s"%(col,colsmap[col]))
        prefs['custom_cols'] = colsmap
+
+        colsnewonly = {}
+        for (col,checkbox) in self.cust_columns_tab.custcol_newonlycheck.iteritems():
+            colsnewonly[col] = checkbox.isChecked()
+        prefs['custom_cols_newonly'] = colsnewonly
        
+        prefs['allow_custcol_from_ini'] = self.cust_columns_tab.allow_custcol_from_ini.isChecked()
+        
+        prefs.save_to_db()
+
    def edit_shortcuts(self):
        self.save_settings()
        # Force the menus to be rebuilt immediately, so we have all our actions registered
@@ -214,9 +299,11 @@ class BasicTab(QWidget):
        label.setWordWrap(True)
        self.l.addWidget(label)
        self.l.addSpacing(5)
-        
+
+        tooltip = "On each download, FFDL offers an option to select the output format. <br />This sets what that option will default to."
        horz = QHBoxLayout()
        label = QLabel('Default Output &Format:')
+        label.setToolTip(tooltip)
        horz.addWidget(label)
        self.fileform = QComboBox(self)
        self.fileform.addItem('epub')
@@ -224,15 +311,16 @@ class BasicTab(QWidget):
        self.fileform.addItem('html')
        self.fileform.addItem('txt')
        self.fileform.setCurrentIndex(self.fileform.findText(prefs['fileform']))
-        self.fileform.setToolTip('Choose output format to create.  May set default from plugin configuration.')
+        self.fileform.setToolTip(tooltip)
        self.fileform.activated.connect(self.set_collisions)
        label.setBuddy(self.fileform)
        horz.addWidget(self.fileform)
        self.l.addLayout(horz)

+        tooltip = "On each download, FFDL offers an option of what happens if that story already exists. <br />This sets what that option will default to."
        horz = QHBoxLayout()
        label = QLabel('Default If Story Already Exists?')
-        label.setToolTip("What to do if there's already an existing story with the same title and author.")
+        label.setToolTip(tooltip)
        horz.addWidget(label)
        self.collision = QComboBox(self)
        # add collision options
@@ -240,26 +328,40 @@ class BasicTab(QWidget):
        i = self.collision.findText(prefs['collision'])
        if i > -1:
            self.collision.setCurrentIndex(i)
-        # self.collision.setToolTip('Overwrite will replace the existing story.  Add New will create a new story with the same title and author.')
+        self.collision.setToolTip(tooltip)
        label.setBuddy(self.collision)
        horz.addWidget(self.collision)
        self.l.addLayout(horz)

        self.updatemeta = QCheckBox('Default Update Calibre &Metadata?',self)
-        self.updatemeta.setToolTip('Update title, author, URL, tags, custom columns, etc for story in Calibre from web site.')
+        self.updatemeta.setToolTip("On each download, FFDL offers an option to update Calibre's metadata (title, author, URL, tags, custom columns, etc) from the web site. <br />This sets whether that will default to on or off. <br />Columns set to 'New Only' in the column tabs will only be set for new books.")
        self.updatemeta.setChecked(prefs['updatemeta'])
        self.l.addWidget(self.updatemeta)

-        self.updatecover = QCheckBox('Update Cover when Updating Metadata?',self)
-        self.updatecover.setToolTip("Update cover image from EPUB when metadata is updated.  (EPUB only.)\nDoesn't go looking for new images on 'Update Calibre Metadata Only'.")
+        self.updateepubcover = QCheckBox('Default Update EPUB Cover when Updating EPUB?',self)
+        self.updateepubcover.setToolTip("On each download, FFDL offers an option to update the book cover image <i>inside</i> the EPUB from the web site when the EPUB is updated.<br />This sets whether that will default to on or off.")
+        self.updateepubcover.setChecked(prefs['updateepubcover'])
+        self.l.addWidget(self.updateepubcover)
+
+        self.l.addSpacing(10)        
+
+        self.deleteotherforms = QCheckBox('Delete other existing formats?',self)
+        self.deleteotherforms.setToolTip('Check this to automatically delete all other ebook formats when updating an existing book.\nHandy if you have both a Nook(epub) and Kindle(mobi), for example.')
+        self.deleteotherforms.setChecked(prefs['deleteotherforms'])
+        self.l.addWidget(self.deleteotherforms)
+        
+        self.updatecover = QCheckBox('Update Calibre Cover when Updating Metadata?',self)
+        self.updatecover.setToolTip("Update calibre book cover image from EPUB when metadata is updated.  (EPUB only.)\nDoesn't go looking for new images on 'Update Calibre Metadata Only'.")
        self.updatecover.setChecked(prefs['updatecover'])
        self.l.addWidget(self.updatecover)

        self.keeptags = QCheckBox('Keep Existing Tags when Updating Metadata?',self)
-        self.keeptags.setToolTip('Existing tags will be kept and any new tags added.\nCompleted and In-Progress tags will be still be updated, if known.\nLast Updated tags will be updated if lastupdate in include_subject_tags.')
+        self.keeptags.setToolTip("Existing tags will be kept and any new tags added.\nCompleted and In-Progress tags will be still be updated, if known.\nLast Updated tags will be updated if lastupdate in include_subject_tags.\n(If Tags is set to 'New Only' in the Standard Columns tab, this has no effect.)")
        self.keeptags.setChecked(prefs['keeptags'])
        self.l.addWidget(self.keeptags)

+        self.l.addSpacing(10)        
+
        self.urlsfromclip = QCheckBox('Take URLs from Clipboard?',self)
        self.urlsfromclip.setToolTip('Prefill URLs from valid URLs in Clipboard when Adding New.')
        self.urlsfromclip.setChecked(prefs['urlsfromclip'])
@@ -271,22 +373,29 @@ class BasicTab(QWidget):
        self.updatedefault.setChecked(prefs['updatedefault'])
        self.l.addWidget(self.updatedefault)

-        self.deleteotherforms = QCheckBox('Delete other existing formats?',self)
-        self.deleteotherforms.setToolTip('Check this to automatically delete all other ebook formats when updating an existing book.\nHandy if you have both a Nook(epub) and Kindle(mobi), for example.')
-        self.deleteotherforms.setChecked(prefs['deleteotherforms'])
-        self.l.addWidget(self.deleteotherforms)
-        
        self.adddialogstaysontop = QCheckBox("Keep 'Add New from URL(s)' dialog on top?",self)
        self.adddialogstaysontop.setToolTip("Instructs the OS and Window Manager to keep the 'Add New from URL(s)'\ndialog on top of all other windows.  Useful for dragging URLs onto it.")
        self.adddialogstaysontop.setChecked(prefs['adddialogstaysontop'])
        self.l.addWidget(self.adddialogstaysontop)

+        self.l.addSpacing(10)        
+
        # this is a cheat to make it easier for users to realize there's a new include_images features.
        self.includeimages = QCheckBox("Include images in EPUBs?",self)
        self.includeimages.setToolTip("Download and include images in EPUB stories.  This is equivalent to adding:\n\n[epub]\ninclude_images:true\nkeep_summary_html:true\nmake_firstimage_cover:true\n\n ...to the top of personal.ini.  Your settings in personal.ini will override this.")
        self.includeimages.setChecked(prefs['includeimages'])
        self.l.addWidget(self.includeimages)
-        
+
+        self.lookforurlinhtml = QCheckBox("Search EPUB text for Story URL?",self)
+        self.lookforurlinhtml.setToolTip("Look for first valid story URL inside EPUB text if not found in metadata.\nSomewhat risky, could find wrong URL depending on EPUB content.\nAlso finds and corrects bad ffnet URLs from ficsaver.com files.")
+        self.lookforurlinhtml.setChecked(prefs['lookforurlinhtml'])
+        self.l.addWidget(self.lookforurlinhtml)
+
+        self.injectseries = QCheckBox("Inject calibre Series when none found?",self)
+        self.injectseries.setToolTip("If no series is found, inject the calibre series (if there is one) so it appears on the FFDL title page(not cover).")
+        self.injectseries.setChecked(prefs['injectseries'])
+        self.l.addWidget(self.injectseries)
+
        self.l.insertStretch(-1)
        
    def set_collisions(self):
@@ -331,7 +440,7 @@ class PersonalIniTab(QWidget):
        self.ini.setText(prefs['personal.ini'])
        self.l.addWidget(self.ini)

-        self.defaults = QPushButton('View Defaults', self)
+        self.defaults = QPushButton('View Defaults (plugin-defaults.ini)', self)
        self.defaults.setToolTip("View all of the plugin's configurable settings\nand their default settings.")
        self.defaults.clicked.connect(self.show_defaults)
        self.l.addWidget(self.defaults)
@@ -350,7 +459,7 @@ class ShowDefaultsIniDialog(QDialog):
        self.resize(600, 500)
        self.l = QVBoxLayout()
        self.setLayout(self.l)
-        self.label = QLabel("Plugin Defaults (Read-Only)")
+        self.label = QLabel("Plugin Defaults (plugin-defaults.ini) (Read-Only)")
        self.label.setToolTip("These are all of the plugin's configurable options\nand their default settings.")
        self.setWindowTitle(_('Plugin Defaults'))
        self.setWindowIcon(icon)
@@ -372,7 +481,7 @@ class ShowDefaultsIniDialog(QDialog):
        self.ok_button.clicked.connect(self.hide)
        self.l.addWidget(self.ok_button)
        
-class ListTab(QWidget):
+class ReadingListTab(QWidget):

    def __init__(self, parent_dialog, plugin_action):
        self.parent_dialog = parent_dialog
@@ -432,6 +541,123 @@ class ListTab(QWidget):
            
        self.l.insertStretch(-1)
        
+class GenerateCoverTab(QWidget):
+
+    def __init__(self, parent_dialog, plugin_action):
+        self.parent_dialog = parent_dialog
+        self.plugin_action = plugin_action
+        QWidget.__init__(self)
+        
+        self.l = QVBoxLayout()
+        self.setLayout(self.l)
+
+        try:
+            gc_plugin = plugin_action.gui.iactions['Generate Cover']
+            gc_settings = gc_plugin.get_saved_setting_names()
+        except KeyError:
+            gc_settings= []
+            
+        label = QLabel('The Generate Cover plugin can create cover images for books using various metadata and configurations.  If you have GC installed, FFDL can run GC on new downloads and metadata updates.  Pick a GC setting by site or Default.')
+        label.setWordWrap(True)
+        self.l.addWidget(label)
+        self.l.addSpacing(5)
+
+        scrollable = QScrollArea()
+        scrollcontent = QWidget()
+        scrollable.setWidget(scrollcontent)
+        scrollable.setWidgetResizable(True)
+        self.l.addWidget(scrollable)
+
+        self.sl = QVBoxLayout()
+        scrollcontent.setLayout(self.sl)
+        
+        self.gc_dropdowns = {}
+
+        sitelist = getConfigSections()
+        sitelist.sort()
+        sitelist.insert(0,u"Default")
+        for site in sitelist:
+            horz = QHBoxLayout()
+            label = QLabel(site)
+            if site == u"Default":
+                s = "On Metadata update, run Generate Cover with this setting, if not selected for specific site."
+            else:
+                s = "On Metadata update, run Generate Cover with this setting for %s stories."%site
+
+            label.setToolTip(s)
+            horz.addWidget(label)
+            dropdown = QComboBox(self)
+            dropdown.setToolTip(s)
+            dropdown.addItem('',QVariant('none'))
+            for setting in gc_settings:
+                dropdown.addItem(setting,QVariant(setting))
+            self.gc_dropdowns[site] = dropdown
+            if site in prefs['gc_site_settings']:
+                dropdown.setCurrentIndex(dropdown.findData(QVariant(prefs['gc_site_settings'][site])))
+
+            horz.addWidget(dropdown)
+            self.sl.addLayout(horz)
+        
+        self.sl.insertStretch(-1)
+        
+        self.gcnewonly = QCheckBox("Run Generate Cover Only on New Books",self)
+        self.gcnewonly.setToolTip("Default is to run GC any time the calibre metadata is updated.")
+        self.gcnewonly.setChecked(prefs['gcnewonly'])
+        self.l.addWidget(self.gcnewonly)
+
+        self.allow_gc_from_ini = QCheckBox('Allow generate_cover_settings from personal.ini to override',self)
+        self.allow_gc_from_ini.setToolTip("The personal.ini parameter generate_cover_settings allows you to choose a GC setting based on metadata rather than site, but it's much more complex.<br \>generate_cover_settings is ignored when this is off.")
+        self.allow_gc_from_ini.setChecked(prefs['allow_gc_from_ini'])
+        self.l.addWidget(self.allow_gc_from_ini)
+            
+class CountPagesTab(QWidget):
+
+    def __init__(self, parent_dialog, plugin_action):
+        self.parent_dialog = parent_dialog
+        self.plugin_action = plugin_action
+        QWidget.__init__(self)
+        
+        self.l = QVBoxLayout()
+        self.setLayout(self.l)
+
+        label = QLabel('These settings provide integration with the Count Pages Plugin.  Count Pages can automatically update custom columns with page, word and reading level statistics.  You have to create and configure the columns in Count Pages first.')
+        label.setWordWrap(True)
+        self.l.addWidget(label)
+        self.l.addSpacing(5)
+
+        label = QLabel('If any of the settings below are checked, when stories are added or updated, the Count Pages Plugin will be called to update the checked statistics.')
+        label.setWordWrap(True)
+        self.l.addWidget(label)
+        self.l.addSpacing(5)
+        
+        # 'PageCount', 'WordCount', 'FleschReading', 'FleschGrade', 'GunningFog'
+        self.pagecount = QCheckBox('Page Count',self)
+        self.pagecount.setToolTip('Which column and algorithm to use are configured in Count Pages.')
+        self.pagecount.setChecked('PageCount' in prefs['countpagesstats'])
+        self.l.addWidget(self.pagecount)
+            
+        self.wordcount = QCheckBox('Word Count',self)
+        self.wordcount.setToolTip('Which column and algorithm to use are configured in Count Words.\nWill overwrite word count from FFDL metadata if set to update the same custom column.')
+        self.wordcount.setChecked('WordCount' in prefs['countpagesstats'])
+        self.l.addWidget(self.wordcount)
+
+        self.fleschreading = QCheckBox('Flesch Reading Ease',self)
+        self.fleschreading.setToolTip('Which column and algorithm to use are configured in Count Pages.')
+        self.fleschreading.setChecked('FleschReading' in prefs['countpagesstats'])
+        self.l.addWidget(self.fleschreading)
+        
+        self.fleschgrade = QCheckBox('Flesch-Kincaid Grade Level',self)
+        self.fleschgrade.setToolTip('Which column and algorithm to use are configured in Count Pages.')
+        self.fleschgrade.setChecked('FleschGrade' in prefs['countpagesstats'])
+        self.l.addWidget(self.fleschgrade)
+        
+        self.gunningfog = QCheckBox('Gunning Fog Index',self)
+        self.gunningfog.setToolTip('Which column and algorithm to use are configured in Count Pages.')
+        self.gunningfog.setChecked('GunningFog' in prefs['countpagesstats'])
+        self.l.addWidget(self.gunningfog)
+        
+        self.l.insertStretch(-1)
+        
 class OtherTab(QWidget):

    def __init__(self, parent_dialog, plugin_action):
@@ -459,6 +685,12 @@ class OtherTab(QWidget):
        reset_confirmation_button.clicked.connect(self.reset_dialogs)
        self.l.addWidget(reset_confirmation_button)
        
+        view_prefs_button = QPushButton('&View library preferences...', self)
+        view_prefs_button.setToolTip(_(
+                    'View data stored in the library database for this plugin'))
+        view_prefs_button.clicked.connect(self.view_prefs)
+        self.l.addWidget(view_prefs_button)
+        
        self.l.insertStretch(-1)
        
    def reset_dialogs(self):
@@ -470,6 +702,10 @@ class OtherTab(QWidget):
                    _('Confirmation dialogs have all been reset'),
                    show=True,
                    show_copy_button=False)
+        
+    def view_prefs(self):
+        d = PrefsViewerDialog(self.plugin_action.gui, PREFS_NAMESPACE)
+        d.exec_()

 permitted_values = {
    'int' : ['numWords','numChapters'],
@@ -482,6 +718,7 @@ permitted_values = {
                     'language',
                     'series',
                     'characters',
+                     'ships',
                     'status',
                     'datePublished',
                     'dateUpdated',
@@ -499,10 +736,10 @@ permitted_values = {
                     'description',
                     'author',
                     'authorUrl',
-                     'formatname'
+                     'formatname',
+                     'version'
                     #,'formatext'   # not useful information.
                     #,'siteabbrev'
-                     #,'version'
                     ]
    }
 # no point copying the whole list.
@@ -518,6 +755,7 @@ titleLabels = {
    'status-I':'Status:In-Progress',
    'series':'Series',
    'characters':'Characters',
+    'ships':'Relationships',
    'datePublished':'Published',
    'dateUpdated':'Updated',
    'dateCreated':'Packaged',
@@ -540,13 +778,15 @@ titleLabels = {
    'version':'FFDL Version'
    }

-class ColumnsTab(QWidget):
+class CustomColumnsTab(QWidget):

    def __init__(self, parent_dialog, plugin_action):
        self.parent_dialog = parent_dialog
        self.plugin_action = plugin_action
        QWidget.__init__(self)
        
+        custom_columns = self.plugin_action.gui.library_view.model().custom_columns
+
        self.l = QVBoxLayout()
        self.setLayout(self.l)

@@ -556,9 +796,17 @@ class ColumnsTab(QWidget):
        self.l.addSpacing(5)
        
        self.custcol_dropdowns = {}
+        self.custcol_newonlycheck = {}

-        custom_columns = self.plugin_action.gui.library_view.model().custom_columns
+        scrollable = QScrollArea()
+        scrollcontent = QWidget()
+        scrollable.setWidget(scrollcontent)
+        scrollable.setWidgetResizable(True)
+        self.l.addWidget(scrollable)

+        self.sl = QVBoxLayout()
+        scrollcontent.setLayout(self.sl)
+        
        for key, column in custom_columns.iteritems():

            if column['datatype'] in permitted_values:
@@ -566,8 +814,8 @@ class ColumnsTab(QWidget):
                # for (k,v) in column.iteritems():
                #     print("column['%s'] => %s"%(k,v))
                horz = QHBoxLayout()
-                label = QLabel('%s(%s)'%(column['name'],key))
-                label.setToolTip("Update this %s column with..."%column['datatype'])
+                label = QLabel(column['name'])
+                label.setToolTip("Update this %s column(%s) with..."%(key,column['datatype']))
                horz.addWidget(label)
                dropdown = QComboBox(self)
                dropdown.addItem('',QVariant('none'))
@@ -580,10 +828,91 @@ class ColumnsTab(QWidget):
                    dropdown.setToolTip("Metadata values valid for this type of column.\nValues that aren't valid for this enumeration column will be ignored.")
                else:
                    dropdown.setToolTip("Metadata values valid for this type of column.")
-
                horz.addWidget(dropdown)
-                self.l.addLayout(horz)
+
+                newonlycheck = QCheckBox("New Only",self)
+                newonlycheck.setToolTip("Write to %s(%s) only for new\nbooks, not updates to existing books."%(column['name'],key))
+                self.custcol_newonlycheck[key] = newonlycheck
+                if key in prefs['custom_cols_newonly']:
+                    newonlycheck.setChecked(prefs['custom_cols_newonly'][key])
+                horz.addWidget(newonlycheck)
+                
+                self.sl.addLayout(horz)
        
-        self.l.insertStretch(-1)
+        self.sl.insertStretch(-1)
+
+        self.l.addSpacing(5)
+        self.allow_custcol_from_ini = QCheckBox('Allow custom_columns_settings from personal.ini to override',self)
+        self.allow_custcol_from_ini.setToolTip("The personal.ini parameter custom_columns_settings allows you to set custom columns to site specific values that aren't common to all sites.<br \>custom_columns_settings is ignored when this is off.")
+        self.allow_custcol_from_ini.setChecked(prefs['allow_custcol_from_ini'])
+        self.l.addWidget(self.allow_custcol_from_ini)
+        
+        self.l.addSpacing(5)
+        label = QLabel("Special column:")
+        label.setWordWrap(True)
+        self.l.addWidget(label)
+        
+        horz = QHBoxLayout()
+        label = QLabel("Update/Overwrite Error Column:")
+        tooltip="When an update or overwrite of an existing story fails, record the reason in this column.\n(Text and Long Text columns only.)"
+        label.setToolTip(tooltip)
+        horz.addWidget(label)
+        self.errorcol = QComboBox(self)
+        self.errorcol.setToolTip(tooltip)
+        self.errorcol.addItem('',QVariant('none'))
+        for key, column in custom_columns.iteritems():
+            if column['datatype'] in ('text','comments'):
+                self.errorcol.addItem(column['name'],QVariant(key))
+        self.errorcol.setCurrentIndex(self.errorcol.findData(QVariant(prefs['errorcol'])))
+        horz.addWidget(self.errorcol)
+        self.l.addLayout(horz)

        #print("prefs['custom_cols'] %s"%prefs['custom_cols'])
+
+
+class StandardColumnsTab(QWidget):
+
+    def __init__(self, parent_dialog, plugin_action):
+        self.parent_dialog = parent_dialog
+        self.plugin_action = plugin_action
+        QWidget.__init__(self)
+
+        columns=OrderedDict()
+        
+        columns["title"]="Title"
+        columns["authors"]="Author(s)"
+        columns["publisher"]="Publisher"
+        columns["tags"]="Tags"
+        columns["languages"]="Languages"
+        columns["pubdate"]="Published Date"
+        columns["timestamp"]="Date"
+        columns["comments"]="Comments"
+        columns["series"]="Series"
+        columns["identifiers"]="Ids(url id only)"
+
+        self.l = QVBoxLayout()
+        self.setLayout(self.l)
+
+        label = QLabel("The standard calibre metadata columns are listed below.  You may choose whether FFDL will fill each column automatically on updates or only for new books.")
+        label.setWordWrap(True)
+        self.l.addWidget(label)
+        self.l.addSpacing(5)
+        
+        self.stdcol_newonlycheck = {}
+
+        for key, column in columns.iteritems():
+            horz = QHBoxLayout()
+            label = QLabel(column)
+            #label.setToolTip("Update this %s column(%s) with..."%(key,column['datatype']))
+            horz.addWidget(label)
+
+            newonlycheck = QCheckBox("New Only",self)
+            newonlycheck.setToolTip("Write to %s only for new\nbooks, not updates to existing books."%column)
+            self.stdcol_newonlycheck[key] = newonlycheck
+            if key in prefs['std_cols_newonly']:
+                newonlycheck.setChecked(prefs['std_cols_newonly'][key])
+            horz.addWidget(newonlycheck)
+            
+            self.l.addLayout(horz)
+        
+        self.l.insertStretch(-1)
@@ -83,7 +83,7 @@ class AddNewDialog(SizePersistedDialog):

        self.l.addWidget(QLabel('Story URL(s), one per line:'))
        self.url = DroppableQTextEdit(self)
-        self.url.setToolTip('URLs for stories, one per line.\nWill take URLs from clipboard, but only valid URLs.')
+        self.url.setToolTip('URLs for stories, one per line.\nWill take URLs from clipboard, but only valid URLs.\nAdd [1,5] after the URL to limit the download to chapters 1-5.')
        self.url.setLineWrapMode(QTextEdit.NoWrap)
        self.url.setText(url_list_text)
        self.l.addWidget(self.url)
@@ -106,27 +106,30 @@ class AddNewDialog(SizePersistedDialog):

        horz = QHBoxLayout()
        label = QLabel('If Story Already Exists?')
-        label.setToolTip("What to do if there's already an existing story with the same title and author.")
        horz.addWidget(label)
        self.collision = QComboBox(self)
+        self.collision.setToolTip("What to do if there's already an existing story with the same URL or title and author.")
        # add collision options
        self.set_collisions()
        i = self.collision.findText(prefs['collision'])
        if i > -1:
            self.collision.setCurrentIndex(i)
-        # self.collision.setToolTip(OVERWRITE+' will replace the existing story.\n'+
-        #                           UPDATE+' will download new chapters only and add to existing EPUB.\n'+
-        #                           ADDNEW+' will create a new story with the same title and author.\n'+
-        #                           SKIP+' will not download existing stories.\n'+
-        #                           CALIBREONLY+' will not download stories, but will update Calibre metadata.')
        label.setBuddy(self.collision)
        horz.addWidget(self.collision)
        self.l.addLayout(horz)

+        horz = QHBoxLayout()
        self.updatemeta = QCheckBox('Update Calibre &Metadata?',self)
-        self.updatemeta.setToolTip('Update metadata for story in Calibre from web site?')
+        self.updatemeta.setToolTip("Update metadata for existing stories in Calibre from web site?\n(Columns set to 'New Only' in the column tabs will only be set for new books.)")
        self.updatemeta.setChecked(prefs['updatemeta'])
-        self.l.addWidget(self.updatemeta)
+        horz.addWidget(self.updatemeta)
+
+        self.updateepubcover = QCheckBox('Update EPUB Cover?',self)
+        self.updateepubcover.setToolTip('Update book cover image from site or defaults (if found) <i>inside</i> the EPUB when EPUB is updated.')
+        self.updateepubcover.setChecked(prefs['updateepubcover'])
+        horz.addWidget(self.updateepubcover)
+        
+        self.l.addLayout(horz)

        button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
        button_box.accepted.connect(self.accept)
@@ -155,6 +158,7 @@ class AddNewDialog(SizePersistedDialog):
            'fileform': unicode(self.fileform.currentText()),
            'collision': unicode(self.collision.currentText()),
            'updatemeta': self.updatemeta.isChecked(),
+            'updateepubcover': self.updateepubcover.isChecked(),
            }

    def get_urlstext(self):
@@ -168,6 +172,47 @@ class FakeLineEdit():
    def text(self):
        pass
    
+class CollectURLDialog(SizePersistedDialog):
+    '''
+    Collect single url for get urls.
+    '''
+    def __init__(self, gui, title, url_text): 
+        SizePersistedDialog.__init__(self, gui, 'FanFictionDownLoader plugin:get story urls')
+        self.gui = gui
+        self.status=False
+
+        self.setMinimumWidth(300)
+        
+        self.l = QGridLayout()
+        self.setLayout(self.l)
+
+        self.setWindowTitle(title)
+        self.l.addWidget(QLabel(title),0,0,1,2)
+        
+        self.l.addWidget(QLabel("URL:"),1,0)
+        self.url = QLineEdit(self)
+        self.url.setText(url_text)
+        self.l.addWidget(self.url,1,1)
+   
+        self.ok_button = QPushButton('OK', self)
+        self.ok_button.clicked.connect(self.ok)
+        self.l.addWidget(self.ok_button,2,0)
+
+        self.cancel_button = QPushButton('Cancel', self)
+        self.cancel_button.clicked.connect(self.cancel)
+        self.l.addWidget(self.cancel_button,2,1)
+
+        # restore saved size.
+        self.resize_dialog()
+
+    def ok(self):
+        self.status=True
+        self.accept()
+
+    def cancel(self):
+        self.status=False
+        self.reject()
+
 class UserPassDialog(QDialog):
    '''
    Need to collect User/Pass for some sites.
@@ -325,7 +370,7 @@ class AuthorTableWidgetItem(ReadOnlyTableWidgetItem):

    #Qt uses a simple < check for sorting items, override this to use the sortKey
    def __lt__(self, other):
-        return self.sort_key < other.sort_key
+        return self.sort_key.lower() < other.sort_key.lower()

 class UpdateExistingDialog(SizePersistedDialog):
    def __init__(self, gui, header, prefs, icon, books,
@@ -385,9 +430,9 @@ class UpdateExistingDialog(SizePersistedDialog):
        options_layout.addWidget(self.fileform)
        
        label = QLabel('Update Mode:')
-        label.setToolTip("What sort of update to perform.  May set default from plugin configuration.")
        options_layout.addWidget(label)
        self.collision = QComboBox(self)
+        self.collision.setToolTip("What sort of update to perform.  May set default from plugin configuration.")
        # add collision options
        self.set_collisions()
        i = self.collision.findText(prefs['collision'])
@@ -398,10 +443,15 @@ class UpdateExistingDialog(SizePersistedDialog):
        options_layout.addWidget(self.collision)

        self.updatemeta = QCheckBox('Update Calibre &Metadata?',self)
-        self.updatemeta.setToolTip('Update metadata for story in Calibre from web site?  May set default from plugin configuration.')
+        self.updatemeta.setToolTip("Update metadata for existing stories in Calibre from web site?\n(Columns set to 'New Only' in the column tabs will only be set for new books.)")
        self.updatemeta.setChecked(prefs['updatemeta'])
        options_layout.addWidget(self.updatemeta)
                
+        self.updateepubcover = QCheckBox('Update EPUB Cover?',self)
+        self.updateepubcover.setToolTip('Update book cover image from site or defaults (if found) <i>inside</i> the EPUB when EPUB is updated.')
+        self.updateepubcover.setChecked(prefs['updateepubcover'])
+        options_layout.addWidget(self.updateepubcover)
+        
        button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
        button_box.accepted.connect(self.accept)
        button_box.rejected.connect(self.reject)
@@ -435,6 +485,7 @@ class UpdateExistingDialog(SizePersistedDialog):
            'fileform': unicode(self.fileform.currentText()),
            'collision': unicode(self.collision.currentText()),
            'updatemeta': self.updatemeta.isChecked(),
+            'updateepubcover': self.updateepubcover.isChecked(),
            }

 def display_story_list(gui, header, prefs, icon, books,
@@ -571,7 +622,7 @@ class StoryListTableWidget(QTableWidget):
        title_cell.setData(Qt.UserRole, QVariant(row))
        self.setItem(row, 1, title_cell)
        
-        self.setItem(row, 2, AuthorTableWidgetItem(book['author'], book['author_sort']))
+        self.setItem(row, 2, AuthorTableWidgetItem(", ".join(book['author']), ", ".join(book['author_sort'])))
        
        url_cell = ReadOnlyTableWidgetItem(book['url'])
        #url_cell.setData(Qt.UserRole, QVariant(book['url']))
@@ -7,17 +7,18 @@ __license__   = 'GPL v3'
 __copyright__ = '2012, Jim Miller'
 __docformat__ = 'restructuredtext en'

-import time, os, copy, threading
-from ConfigParser import SafeConfigParser
+import time, os, copy, threading, re, platform
 from StringIO import StringIO
 from functools import partial
 from datetime import datetime
+from string import Template

 from PyQt4.Qt import (QApplication, QMenu, QToolButton)

 from PyQt4.Qt import QPixmap, Qt
 from PyQt4.QtCore import QBuffer

+from calibre.constants import numeric_version as calibre_version

 from calibre.ptempfile import PersistentTemporaryFile, PersistentTemporaryDirectory, remove_dir
 from calibre.ebooks.metadata import MetaInformation, authors_to_string
@@ -26,6 +27,8 @@ from calibre.gui2 import error_dialog, warning_dialog, question_dialog, info_dia
 from calibre.gui2.dialogs.message_box import ViewLog
 from calibre.gui2.dialogs.confirm_delete import confirm
 from calibre.utils.date import local_tz
+from calibre.library.comments import sanitize_comments_html
+from calibre.constants import config_dir as calibre_config_dir

 # The class that all interface action plugins must inherit from
 from calibre.gui2.actions import InterfaceAction
@@ -33,14 +36,15 @@ from calibre.gui2.actions import InterfaceAction
 from calibre_plugins.fanfictiondownloader_plugin.common_utils import (set_plugin_icon_resources, get_icon,
                                         create_menu_action_unique, get_library_uuid)

-from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapters, writers, exceptions
-from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.htmlcleanup import stripHTML
-from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.epubutils import get_dcsource, get_dcsource_chaptercount
+from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapters, exceptions
+from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.configurable import Configuration
+from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.epubutils import get_dcsource, get_dcsource_chaptercount, get_story_url_from_html
+from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.geturls import get_urls_from_page

 from calibre_plugins.fanfictiondownloader_plugin.config import (prefs, permitted_values)
 from calibre_plugins.fanfictiondownloader_plugin.dialogs import (
    AddNewDialog, UpdateExistingDialog, display_story_list, DisplayStoryListDialog,
-    LoopProgressDialog, UserPassDialog, AboutDialog,
+    LoopProgressDialog, UserPassDialog, AboutDialog, CollectURLDialog, 
    OVERWRITE, OVERWRITEALWAYS, UPDATE, UPDATEALWAYS, ADDNEW, SKIP, CALIBREONLY,
    NotGoingToDownload )

@@ -113,15 +117,28 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
        # items to prevent GC removing it.
        self.menu_actions = []
        self.qaction.setMenu(self.menu)
-        self.menu.aboutToShow.connect(self.about_to_show_menu)
-
        self.menus_lock = threading.RLock()
+        self.menu.aboutToShow.connect(self.about_to_show_menu)

    def initialization_complete(self):
        # otherwise configured hot keys won't work until the menu's
        # been displayed once.
        self.rebuild_menus()

+    ## Kludgey, yes, but with the real configuration inside the
+    ## library now, how else would a user be able to change this
+    ## setting if it's crashing calibre?
+    def check_macmenuhack(self):
+        try:
+            return self.macmenuhack
+        except:
+            file_path = os.path.join(calibre_config_dir,
+                                     *("plugins/fanfictiondownloader_macmenuhack.txt".split('/')))
+            file_path = os.path.abspath(file_path)
+            print("macmenuhack file_path:%s"%file_path)
+            self.macmenuhack = os.access(file_path, os.F_OK)
+            return self.macmenuhack
+
    def about_to_show_menu(self):
        self.rebuild_menus()

@@ -131,10 +148,6 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
        
    def rebuild_menus(self):
        with self.menus_lock:
-            # Show the config dialog
-            # The config dialog can also be shown from within
-            # Preferences->Plugins, which is why the do_user_config
-            # method is defined on the base plugin class
            do_user_config = self.interface_action_base_plugin.do_user_config
            self.menu.clear()
            self.actions_unique_map = {}
@@ -145,8 +158,6 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                                                       triggered=self.add_dialog )

            self.update_action = self.create_menu_item_ex(self.menu, '&Update Existing FanFiction Book(s)', image='plusplus.png',
-                                                          unique_name='Update Existing FanFiction Book(s)',
-                                                          shortcut_name='Update Existing FanFiction Book(s)',
                                                          triggered=self.update_existing)

            if 'Reading List' in self.gui.iactions and (prefs['addtolists'] or prefs['addtoreadlists']) :
@@ -166,43 +177,34 @@ class FanFictionDownLoaderPlugin(InterfaceAction):

                if addmenutxt:
                    self.add_send_action = self.create_menu_item_ex(self.menu, addmenutxt, image='plusplus.png',
-                                                                    unique_name=addmenutxt,
-                                                                    shortcut_name=addmenutxt,
                                                                    triggered=partial(self.update_lists,add=True))

                if rmmenutxt:
                    self.add_remove_action = self.create_menu_item_ex(self.menu, rmmenutxt, image='minusminus.png',
-                                                                      unique_name=rmmenutxt,
-                                                                      shortcut_name=rmmenutxt,
                                                                      triggered=partial(self.update_lists,add=False))
                
-                # try:
-                #     self.add_send_action.setEnabled( len(self.gui.library_view.get_selected_ids()) > 0 )
-                # except:
-                #     pass
-                # try:
-                #     self.add_remove_action.setEnabled( len(self.gui.library_view.get_selected_ids()) > 0 )
-                # except:
-                #     pass
-
            self.menu.addSeparator()
            self.get_list_action = self.create_menu_item_ex(self.menu, 'Get URLs from Selected Books', image='bookmarks.png',
-                                                            unique_name='Get URLs from Selected Books',
-                                                            shortcut_name='Get URLs from Selected Books',
                                                            triggered=self.get_list_urls)

-            self.menu.addSeparator()
-            self.config_action = create_menu_action_unique(self, self.menu, '&Configure Plugin', shortcut=False,
-                                                           image= 'config.png',
-                                                           unique_name='Configure FanFictionDownLoader',
-                                                           shortcut_name='Configure FanFictionDownLoader',
-                                                           triggered=partial(do_user_config,parent=self.gui))
+            self.get_list_url_action = self.create_menu_item_ex(self.menu, 'Get Story URLs from Web Page', image='view.png',
+                                                                triggered=self.get_urls_from_page)
+
+            # print("platform.system():%s"%platform.system())
+            # print("platform.mac_ver()[0]:%s"%platform.mac_ver()[0])
+            if not self.check_macmenuhack(): # not platform.mac_ver()[0]: # Some macs crash on these menu items for unknown reasons.
+                self.menu.addSeparator()
+                self.config_action = self.create_menu_item_ex(self.menu, '&Configure Plugin',
+                                                              image= 'config.png',
+                                                              unique_name='Configure FanFictionDownLoader',
+                                                              shortcut_name='Configure FanFictionDownLoader',
+                                                              triggered=partial(do_user_config,parent=self.gui))
            
-            self.config_action = create_menu_action_unique(self, self.menu, '&About Plugin', shortcut=False,
-                                                           image= 'images/icon.png',
-                                                           unique_name='About FanFictionDownLoader',
-                                                           shortcut_name='About FanFictionDownLoader',
-                                                           triggered=self.about)
+                self.about_action = self.create_menu_item_ex(self.menu, 'About Plugin',
+                                                             image= 'images/icon.png',
+                                                             unique_name='About FanFictionDownLoader', 
+                                                             shortcut_name='About FanFictionDownLoader', 
+                                                             triggered=self.about)
            
            # Before we finalize, make sure we delete any actions for menus that are no longer displayed
            for menu_id, unique_name in self.old_actions_unique_map.iteritems():
@@ -228,10 +230,12 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
    def create_menu_item_ex(self, parent_menu, menu_text, image=None, tooltip=None,
                           shortcut=None, triggered=None, is_checked=None, shortcut_name=None,
                           unique_name=None):
+        #print("create_menu_item_ex before %s"%menu_text)
        ac = create_menu_action_unique(self, parent_menu, menu_text, image, tooltip,
                                       shortcut, triggered, is_checked, shortcut_name, unique_name)
        self.actions_unique_map[ac.calibre_shortcut_unique_name] = ac.calibre_shortcut_unique_name
        self.menu_actions.append(ac)
+        #print("create_menu_item_ex after %s"%menu_text)
        return ac

    def plugin_button(self):
@@ -245,6 +249,40 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                (prefs['addtolists'] or prefs['addtoreadlists']) :
            self._update_reading_lists(self.gui.library_view.get_selected_ids(),add)

+    def get_urls_from_page(self):
+
+        if prefs['urlsfromclip']:
+            try:
+                urltxt = self.get_urls_clip(storyurls=False)[0]
+            except:
+                urltxt = ""
+        
+        d = CollectURLDialog(self.gui,"Get Story URLs from Web Page",urltxt)
+        d.exec_()
+        if not d.status:
+            return
+        url = u"%s"%d.url.text()
+        print("get_urls_from_page URL:%s"%url)
+
+        if 'archiveofourown.org' in url:
+            configuration = Configuration(adapters.getConfigSectionFor(url),"EPUB")
+            configuration.readfp(StringIO(get_resources("plugin-defaults.ini")))
+            configuration.readfp(StringIO(prefs['personal.ini']))
+        else:
+            configuration = None
+        url_list = get_urls_from_page(url,configuration)
+
+        if url_list:
+            d = ViewLog(_("List of URLs"),"\n".join(url_list),parent=self.gui)
+            d.setWindowIcon(get_icon('bookmarks.png'))
+            d.exec_()
+        else:
+            info_dialog(self.gui, _('List of URLs'),
+                        _('No Valid URLs found on given page.'),
+                        show=True,
+                        show_copy_button=False)
+        
+            
    def get_list_urls(self):
        if len(self.gui.library_view.get_selected_ids()) > 0:
            book_list = map( partial(self._convert_id_to_book, good=False), self.gui.library_view.get_selected_ids() )
@@ -349,13 +387,14 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
            options['version'] = self.version
            print(self.version)
            self.start_downloads( options, update_books )
-        
-    def get_urls_clip(self):
+            
+    def get_urls_clip(self,storyurls=True):
        url_list = []
        if prefs['urlsfromclip']:
            for url in unicode(QApplication.instance().clipboard().text()).split():
-                if( self._is_good_downloader_url(url) ):
+                if not storyurls or self._is_good_downloader_url(url):
                    url_list.append(url)
+                    
        return url_list

    def apply_settings(self):
@@ -385,7 +424,8 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
    def get_metadata_for_book(self,book,
                              options={'fileform':'epub',
                                       'collision':ADDNEW,
-                                       'updatemeta':True}):
+                                       'updatemeta':True,
+                                       'updateepubcover':True}):
        '''
        Update passed in book dict with metadata from website and
        necessary data.  To be called from LoopProgressDialog
@@ -401,6 +441,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
        fileform  = options['fileform']
        collision = options['collision']
        updatemeta= options['updatemeta']
+        updateepubcover= options['updateepubcover']

        if not book['good']:
            # book has already been flagged bad for whatever reason.
@@ -410,13 +451,6 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
        print("url:%s"%url)
        skip_date_update = False
        
-        ## was self.ffdlconfig, but we need to be able to change it
-        ## when doing epub update.
-        ffdlconfig = SafeConfigParser()
-        ffdlconfig.readfp(StringIO(get_resources("plugin-defaults.ini")))
-        ffdlconfig.readfp(StringIO(prefs['personal.ini']))
-        adapter = adapters.getAdapter(ffdlconfig,url,fileform)
-
        options['personal.ini'] = prefs['personal.ini']
        if prefs['includeimages']:
            # this is a cheat to make it easier for users.
@@ -426,6 +460,11 @@ keep_summary_html:true
 make_firstimage_cover:true
 ''' + options['personal.ini']

+        configuration = Configuration(adapters.getConfigSectionFor(url),fileform)
+        configuration.readfp(StringIO(get_resources("plugin-defaults.ini")))
+        configuration.readfp(StringIO(options['personal.ini']))
+        adapter = adapters.getAdapter(configuration,url)
+
        ## three tries, that's enough if both user/pass & is_adult needed,
        ## or a couple tries of one or the other
        for x in range(0,2):
@@ -447,15 +486,18 @@ make_firstimage_cover:true

        # let other exceptions percolate up.
        story = adapter.getStoryMetadataOnly()
-        writer = writers.getWriter(options['fileform'],adapter.config,adapter)
+
+        # set PI version instead of default.
+        if 'version' in options:
+            story.setMetadata('version',options['version'])

        book['all_metadata'] = story.getAllMetadata(removeallentities=True)
        book['title'] = story.getMetadata("title", removeallentities=True)
-        book['author_sort'] = book['author'] = story.getMetadata("author", removeallentities=True)
+        book['author_sort'] = book['author'] = story.getList("author", removeallentities=True)
        book['publisher'] = story.getMetadata("site")
-        book['tags'] = writer.getTags() # getTags could be moved up into adapter now.  Adapter didn't used to know the fileform
-        book['comments'] = stripHTML(story.getMetadata("description")) #, removeallentities=True) comments handles entities better.
-        book['series'] = story.getMetadata("series")
+        book['tags'] = story.getSubjectTags(removeallentities=True)
+        book['comments'] = sanitize_comments_html(story.getMetadata("description"))        
+        book['series'] = story.getMetadata("series", removeallentities=True)
        
        # adapter.opener is the element with a threadlock.  But del
        # adapter.opener doesn't work--subproc fails when it tries
@@ -467,6 +509,7 @@ make_firstimage_cover:true
        book['password'] = adapter.password

        book['icon'] = 'plus.png'
+        book['status'] = 'Add'
        if story.getMetadataRaw('datePublished'):
            # should only happen when an adapter is broken, but better to
            # fail gracefully.
@@ -475,6 +518,7 @@ make_firstimage_cover:true
        
        if collision in (CALIBREONLY):
            book['icon'] = 'metadata.png'
+            book['status'] = 'Meta'
        
        # Dialogs should prevent this case now.
        if collision in (UPDATE,UPDATEALWAYS) and fileform != 'epub':
@@ -491,36 +535,46 @@ make_firstimage_cover:true
        # only care about collisions when not ADDNEW
        elif collision != ADDNEW:
            # 'new' book from URL.  collision handling applies.
-            print("from URL")
+            print("from URL(%s)"%url)

-            # find dups
-            mi = MetaInformation(story.getMetadata("title", removeallentities=True),
-                                 (story.getMetadata("author", removeallentities=True),)) # author is a list.
-            identicalbooks = db.find_identical_books(mi)
-            ## removed for being overkill.
-            # for ib in identicalbooks:
-            #     # only *really* identical if URL matches, too.
-            #     # XXX make an option?
-            #     if self._get_story_url(db,ib) == url:
-            #         identicalbooks.append(ib)
-            #print("identicalbooks:%s"%identicalbooks)
+            # try to find by identifier url first.
+            searchstr = 'identifiers:"=url:%s"'%url.replace(":","|")
+            identicalbooks = db.search_getting_ids(searchstr, None)
+            if len(identicalbooks) < 1:
+                # find dups
+                authlist = story.getList("author", removeallentities=True)
+                if len(authlist) > 100 and calibre_version < (0, 8, 61):
+                    ## should be fixed from 0.8.61 on.  In the
+                    ## meantime, if it matches the title *and* first
+                    ## 100 authors, I'm prepared to assume it's a
+                    ## match.
+                    print("reduce author list to 100 only when calibre < 0.8.61")
+                    authlist = authlist[:100]
+                mi = MetaInformation(story.getMetadata("title", removeallentities=True),
+                                     authlist)
+                identicalbooks = db.find_identical_books(mi)
+                if len(identicalbooks) > 0:
+                    print("existing found by title/author(s)")

+            else:
+                print("existing found by identifier URL")
+                
            if collision == SKIP and identicalbooks:
                raise NotGoingToDownload("Skipping duplicate story.","list_remove.png")

            if len(identicalbooks) > 1:
-                raise NotGoingToDownload("More than one identical book--can't tell which to update/overwrite.","minusminus.png")
+                raise NotGoingToDownload("More than one identical book by Identifer URL or title/author(s)--can't tell which book to update/overwrite.","minusminus.png")

            ## changed: add new book when CALIBREONLY if none found.
            if collision == CALIBREONLY and not identicalbooks:
                collision = ADDNEW
                options['collision'] = ADDNEW
-            #     raise NotGoingToDownload("Not updating Calibre Metadata, no existing book to update.","search_delete_saved.png")

            if len(identicalbooks)>0:
                book_id = identicalbooks.pop()
                book['calibre_id'] = book_id
                book['icon'] = 'edit-redo.png'
+                book['status'] = 'Update'

        if book_id != None and collision != ADDNEW:
            if collision in (CALIBREONLY):
@@ -571,7 +625,13 @@ make_firstimage_cover:true
            # this behavior matches how epubs come out when imported
            # dateCreated == packaged--epub/etc created.
            book['timestamp'] = story.getMetadataRaw('dateCreated').replace(tzinfo=local_tz)
-        
+
+        if book_id != None and prefs['injectseries']:
+            mi = db.get_metadata(book_id,index_is_id=True)
+            if not book['series'] and mi.series != None:
+                book['calibre_series'] = (mi.series,mi.series_index)
+                print("calibre_series:%s [%s]"%book['calibre_series'])
+            
        if book['good']: # there shouldn't be any !'good' books at this point.
            # if still 'good', make a temp file to write the output to.
            tmp = PersistentTemporaryFile(prefix='new-%s-'%book['calibre_id'],
@@ -586,7 +646,8 @@ make_firstimage_cover:true
    def start_download_list(self,book_list,
                            options={'fileform':'epub',
                                     'collision':ADDNEW,
-                                     'updatemeta':True}):
+                                     'updatemeta':True,
+                                     'updateepubcover':True}):
        '''
        Called by LoopProgressDialog to start story downloads BG processing.
        adapter_list is a list of tuples of (url,adapter)
@@ -617,6 +678,22 @@ make_firstimage_cover:true
                                       label_text='None of the URLs/stories given can be/need to be downloaded.'
                                       )
            d.exec_()
+
+            
+            custom_columns = self.gui.library_view.model().custom_columns
+            if prefs['errorcol'] != '' and prefs['errorcol'] in custom_columns:
+                label = custom_columns[prefs['errorcol']]['label']
+                ## if error column and all bad.
+                self.previous = self.gui.library_view.currentIndex()
+                LoopProgressDialog(self.gui,
+                                   book_list,
+                                   partial(self._update_bad_book, label=label, options=options, db=self.gui.current_db),
+                                   partial(self._update_books_completed, options=options, showlist=False),
+                                   init_label="Updating calibre for BAD FanFiction stories...",
+                                   win_title="Update calibre for BAD FanFiction stories",
+                                   status_prefix="Updated")
+            
+            
            return
            
        func = 'arbitrary_n'
@@ -634,7 +711,8 @@ make_firstimage_cover:true
    def _update_book(self,book,db=None,
                     options={'fileform':'epub',
                              'collision':ADDNEW,
-                              'updatemeta':True}):
+                              'updatemeta':True,
+                              'updateepubcover':True}):
        print("add/update %s %s"%(book['title'],book['url']))
        mi = self._make_mi_from_book(book)
                
@@ -642,10 +720,19 @@ make_firstimage_cover:true
            self._add_or_update_book(book,options,prefs,mi)

        if options['collision'] == CALIBREONLY or \
-                (options['updatemeta'] and book['good']):
+                ( (options['updatemeta'] or book['added']) and book['good'] ):
            self._update_metadata(db, book['calibre_id'], book, mi, options)

-    def _update_books_completed(self, book_list, options={}):
+    def _update_bad_book(self,book,db=None,label='errorcol',
+                         options={'fileform':'epub',
+                                  'collision':ADDNEW,
+                                  'updatemeta':True,
+                                  'updateepubcover':True},):
+        if book['calibre_id']:
+            print("add/update bad %s %s %s"%(book['title'],book['url'],book['comment']))
+            db.set_custom(book['calibre_id'], book['comment'], label=label, commit=True)
+
+    def _update_books_completed(self, book_list, options={}, showlist=True):
        
        add_list = filter(lambda x : x['good'] and x['added'], book_list)
        add_ids = [ x['calibre_id'] for x in add_list ]
@@ -657,7 +744,7 @@ make_firstimage_cover:true
            self.gui.library_view.model().books_added(len(add_list))
            self.gui.library_view.model().refresh_ids(add_ids)

-        if update_ids:
+        if len(update_list):
            self.gui.library_view.model().refresh_ids(update_ids)

        current = self.gui.library_view.currentIndex()
@@ -669,7 +756,7 @@ make_firstimage_cover:true
        
        self.gui.status_bar.show_message(_('Finished Adding/Updating %d books.'%(len(update_list) + len(add_list))), 3000)
            
-        if len(update_list) + len(add_list) != len(book_list):
+        if showlist and (len(update_list) + len(add_list) != len(book_list)):
            d = DisplayStoryListDialog(self.gui,
                                       'Updates completed, final status',
                                       prefs,
@@ -681,7 +768,13 @@ make_firstimage_cover:true
    
        print("all done, remove temp dir.")
        remove_dir(options['tdir'])
-        
+
+        all_ids = add_ids
+        all_ids.extend(update_ids)
+        if 'Count Pages' in self.gui.iactions and len(prefs['countpagesstats']) and len(all_ids):
+            cp_plugin = self.gui.iactions['Count Pages']
+            cp_plugin.count_statistics(all_ids,prefs['countpagesstats'])
+
    def download_list_completed(self, job, options={}):
        if job.failed:
            self.gui.job_exception(job, dialog_title='Failed to Download Stories')
@@ -690,29 +783,73 @@ make_firstimage_cover:true
        self.previous = self.gui.library_view.currentIndex()
        db = self.gui.current_db

-        if display_story_list(self.gui,
-                              'Downloads finished, confirm to update Calibre',
-                              prefs,
-                              self.qaction.icon(),
-                              job.result,
-                              label_text='Stories will not be added or updated in Calibre without confirmation.',
-                              offer_skip=True):
-            
-            book_list = job.result
-            good_list = filter(lambda x : x['good'], book_list)
-            total_good = len(good_list)
+        book_list = job.result
+        good_list = filter(lambda x : x['good'], book_list)
+        bad_list = filter(lambda x : x['calibre_id'] and not x['good'], book_list)
+        payload = (good_list, bad_list, options)
+        
+        msg = '''
+<p>FFDL found <b>%s</b> good and <b>%s</b> bad updates.</p>
+<p>See log for details.</p>
+<p>Proceed with updating your library?</p>
+'''%(len(good_list),len(bad_list))

-            self.gui.status_bar.show_message(_('Adding/Updating %s books.'%total_good))
+        htmllog='<html><body><table border="1"><tr><th>Status</th><th>Title</th><th>Author</th><th>Comment</th><th>URL</th></tr>'
+        for book in good_list:
+            if 'status' in book:
+                status = book['status']
+            else:
+                status = 'Good'
+            htmllog = htmllog + '<tr><td>' + '</td><td>'.join([status,book['title'],", ".join(book['author']),book['comment'],book['url']]) + '</td></tr>'
+        
+        for book in bad_list:
+            if 'status' in book:
+                status = book['status']
+            else:
+                status = 'Bad'
+            htmllog = htmllog + '<tr><td>' + '</td><td>'.join([status,book['title'],", ".join(book['author']),book['comment'],book['url']]) + '</td></tr>'
+        
+        htmllog = htmllog + '</table></body></html>'
+        
+        self.gui.proceed_question(self._do_download_list_update,
+                payload, htmllog,
+                'FFDL log', 'FFDL download complete', msg,
+                show_copy_button=False)
+        
+        
+    def _do_download_list_update(self, payload):
+        
+        (good_list,bad_list,options) = payload
+        total_good = len(good_list)

-            if total_good > 0:
+        self.gui.status_bar.show_message(_('Adding/Updating %s books.'%total_good))
+
+        if total_good > 0:
+            LoopProgressDialog(self.gui,
+                               good_list,
+                               partial(self._update_book, options=options, db=self.gui.current_db),
+                               partial(self._update_books_completed, options=options),
+                               init_label="Updating calibre for FanFiction stories...",
+                               win_title="Update calibre for FanFiction stories",
+                               status_prefix="Updated")
+
+        total_bad = len(bad_list)
+
+        if total_bad > 0:
+            custom_columns = self.gui.library_view.model().custom_columns
+            if prefs['errorcol'] != '' and prefs['errorcol'] in custom_columns:
+                self.gui.status_bar.show_message(_('Adding/Updating %s BAD books.'%total_bad))
+                label = custom_columns[prefs['errorcol']]['label']
+                ## if error column and all bad.
                LoopProgressDialog(self.gui,
-                                   good_list,
-                                   partial(self._update_book, options=options, db=self.gui.current_db),
-                                   partial(self._update_books_completed, options=options),
-                                   init_label="Updating calibre for stories...",
-                                   win_title="Update calibre for stories",
+                                   bad_list,
+                                   partial(self._update_bad_book, label=label, options=options, db=self.gui.current_db),
+                                   partial(self._update_books_completed, options=options, showlist=False),
+                                   init_label="Updating calibre for BAD FanFiction stories...",
+                                   win_title="Update calibre for BAD FanFiction stories",
                                   status_prefix="Updated")

+                
    def _add_or_update_book(self,book,options,prefs,mi=None):
        db = self.gui.current_db
        
@@ -734,6 +871,7 @@ make_firstimage_cover:true
            book['comment'] = "Adding format to book failed for some reason..."
            book['good']=False
            book['icon']='dialog_error.png'
+            book['status'] = 'Error'

        if prefs['deleteotherforms']:
            fmts = db.formats(book['calibre_id'], index_is_id=True).split(',')
@@ -748,6 +886,7 @@ make_firstimage_cover:true
        return book_id

    def _update_metadata(self, db, book_id, book, mi, options):
+        oldmi = db.get_metadata(book_id,index_is_id=True)
        if prefs['keeptags']:
            old_tags = db.get_tags(book_id)
            # remove old Completed/In-Progress only if there's a new one.
@@ -763,7 +902,6 @@ make_firstimage_cover:true
            mi.languages=[book['all_metadata']['langcode']]
        else:
            # Set language english, but only if not already set.
-            oldmi = db.get_metadata(book_id,index_is_id=True)
            if not oldmi.languages:
                mi.languages=['eng']

@@ -772,14 +910,29 @@ make_firstimage_cover:true
            epubmi = get_metadata(existingepub,'EPUB')
            if epubmi.cover_data[1] is not None:
                db.set_cover(book_id, epubmi.cover_data[1])
-            #mi.cover = epubmi.cover_data[1]

-        # set author link if found.  All current adapters have authorUrl.
+        # set author link if found.  All current adapters have authorUrl, except anonymous on AO3.
        if 'authorUrl' in book['all_metadata']:
-            autid=db.get_author_id(book['author'])
-            db.set_link_field_for_author(autid, unicode(book['all_metadata']['authorUrl']),
-                                         commit=False, notify=False)
-            
+            authurls = book['all_metadata']['authorUrl'].split(", ")
+            for i, auth in enumerate(book['author']):
+                autid=db.get_author_id(auth)
+                db.set_link_field_for_author(autid, unicode(authurls[i]),
+                                             commit=False, notify=False)
+
+        # implement 'newonly' flags here by setting to the current
+        # value again.
+        if not book['added']:
+            for (col,newonly) in prefs['std_cols_newonly'].iteritems():
+                if newonly:
+                    if col == "identifiers":
+                        mi.set_identifiers(oldmi.get_identifiers())
+                    else:
+                        try:
+                            mi.__setattr__(col,oldmi.__getattribute__(col))
+                        except AttributeError:
+                            print("AttributeError? %s"%col)
+                            pass
+
        db.set_metadata(book_id,mi)

        # do configured column updates here.
@@ -793,8 +946,12 @@ make_firstimage_cover:true
                print("%s not an existing column, skipping."%col)
                continue
            coldef = custom_columns[col]
-            if not meta.startswith('status-') and meta not in book['all_metadata']:
-                print("No value for %s, skipping."%meta)
+            if col in prefs['custom_cols_newonly'] and prefs['custom_cols_newonly'][col] and not book['added']:
+                print("Skipping custom column(%s) update, set to New Books Only"%coldef['name'])
+                continue
+            if not meta.startswith('status-') and meta not in book['all_metadata'] or \
+                    meta.startswith('status-') and 'status' not in book['all_metadata']:
+                print("No value for %s, skipping custom column(%s) update."%(meta,coldef['name']))
                continue
            if meta not in permitted_values[coldef['datatype']]:
                print("%s not a valid column type for %s, skipping."%(col,meta))
@@ -811,9 +968,107 @@ make_firstimage_cover:true
                if meta == 'status-I':
                    val = book['all_metadata']['status'] == 'In-Progress'
                db.set_custom(book_id, val, label=label, commit=False)
-        
+
+        adapter = None
+        if prefs['allow_custcol_from_ini']:
+            configuration = Configuration(adapters.getConfigSectionFor(book['url']),options['fileform'])
+            configuration.readfp(StringIO(get_resources("plugin-defaults.ini")))
+            configuration.readfp(StringIO(options['personal.ini']))
+            adapter = adapters.getAdapter(configuration,book['url'])
+
+            # meta => custcol[,a|n|r]
+            # cliches=>\#acolumn,r
+            for line in adapter.getConfig('custom_columns_settings').splitlines():
+                if "=>" in line:
+                    (meta,custcol) = map( lambda x: x.strip(), line.split("=>") )
+                    flag='r'
+                    if "," in custcol:
+                        (custcol,flag) = map( lambda x: x.strip(), custcol.split(",") )
+
+                    print("meta:(%s) => custcol:(%s), flag(%s) "%(meta,custcol,flag))
+                    
+                    if meta not in book['all_metadata']:
+                        print("No value for %s, skipping custom column(%s) update."%(meta,custcol))
+                        continue
+                    
+                    if custcol not in custom_columns:
+                        print("No custom column(%s), skipping."%(custcol))
+                        continue
+                    else:
+                        coldef = custom_columns[custcol]
+                        label = coldef['label']
+                    
+                    if flag == 'r' or book['added']: # flag 'n' isn't actually needed--*always* set if configured and new book.
+                        db.set_custom(book_id, book['all_metadata'][meta], label=label, commit=False)
+
+                    if flag == 'a':
+                        vallist = []
+                        try:
+                            existing=db.get_custom(book_id,label=label,index_is_id=True)
+                            if isinstance(existing,list):
+                                vallist = existing
+                            elif existing:
+                                vallist = [existing]
+                        except:
+                            pass
+
+                        if book['all_metadata'][meta]:
+                            vallist = [book['all_metadata'][meta]]
+                            
+                        db.set_custom(book_id, ", ".join(vallist), label=label, commit=False)
+            
+                
        db.commit()

+        if 'Generate Cover' in self.gui.iactions and (book['added'] or not prefs['gcnewonly']):
+            
+            # force a refresh if generating cover so complex composite
+            # custom columns are current and correct
+            db.refresh_ids([book_id])
+                
+            gc_plugin = self.gui.iactions['Generate Cover']
+            setting_name = None
+            if prefs['allow_gc_from_ini']:
+                if not adapter: # might already have it from allow_custcol_from_ini
+                    configuration = Configuration(adapters.getConfigSectionFor(book['url']),options['fileform'])
+                    configuration.readfp(StringIO(get_resources("plugin-defaults.ini")))
+                    configuration.readfp(StringIO(options['personal.ini']))
+                    adapter = adapters.getAdapter(configuration,book['url'])
+
+                # template => regexp to match => GC Setting to use.
+                # generate_cover_settings:
+                # ${category} => Buffy:? the Vampire Slayer => Buffy
+                for line in adapter.getConfig('generate_cover_settings').splitlines():
+                    if "=>" in line:
+                        (template,regexp,setting) = map( lambda x: x.strip(), line.split("=>") )
+                        value = Template(template).safe_substitute(book['all_metadata']).encode('utf8')
+                        # print("%s(%s) => %s => %s"%(template,value,regexp,setting))
+                        if re.search(regexp,value):
+                            setting_name = setting
+                            break
+
+                if setting_name:
+                    print("Generate Cover Setting from generate_cover_settings(%s)"%line)
+                    if setting_name not in gc_plugin.get_saved_setting_names():
+                        print("GC Name %s not found, discarding! (check personal.ini for typos)"%setting_name)
+                        setting_name = None
+                
+            if not setting_name and book['all_metadata']['site'] in prefs['gc_site_settings']:
+                setting_name =  prefs['gc_site_settings'][book['all_metadata']['site']]
+
+            if not setting_name and 'Default' in prefs['gc_site_settings']:
+                setting_name =  prefs['gc_site_settings']['Default']
+                
+            if setting_name:
+                print("Running Generate Cover with settings %s."%setting_name)
+                realmi = db.get_metadata(book_id, index_is_id=True)
+                gc_plugin.generate_cover_for_book(realmi,saved_setting_name=setting_name)
+
+        ## if error column set.
+        if prefs['errorcol'] != '' and prefs['errorcol'] in custom_columns:
+            label = custom_columns[prefs['errorcol']]['label']
+            db.set_custom(book['calibre_id'], '', label=label, commit=True) # book['comment']
+
    def _get_clean_reading_lists(self,lists):
        if lists == None or lists.strip() == "" :
            return []
@@ -856,10 +1111,18 @@ make_firstimage_cover:true
            if len(lists) < 1 :
                message="<p>You configured FanFictionDownLoader to automatically update \"Send to Device\" Reading Lists, but you don't have any lists set?</p>"
                confirm(message,'fanfictiondownloader_no_send_lists', self.gui)
+
+            # Quick demo of how an 'allow send' list might work.
+            # Issues: allow list per send list?  Naming convention?  "send(allow)"
+            # allow_list = rl_plugin.get_book_list("Allow Send to Device")
+            # # intersection of book_ids & allow_list
+            # add_book_ids = list(set(book_ids) & set(allow_list))
+
            for l in lists:
                if l in rl_plugin.get_list_names():
                    #print("good send l:(%s)"%l)
                    rl_plugin.add_books_to_list(l,
+                                                #add_book_ids,
                                                book_ids,
                                                display_warnings=False)
                else:
@@ -867,19 +1130,19 @@ make_firstimage_cover:true
                        message="<p>You configured FanFictionDownLoader to automatically update Reading List '%s', but you don't have a list of that name?</p>"%l
                        confirm(message,'fanfictiondownloader_no_reading_list_%s'%l, self.gui)

-    def _find_existing_book_id(self,db,book,matchurl=True):
-        mi = MetaInformation(book["title"],(book["author"],)) # author is a list.
-        identicalbooks = db.find_identical_books(mi)
-        if matchurl: # only *really* identical if URL matches, too.
-            for ib in identicalbooks:
-                if self._get_story_url(db,ib) == book['url']:
-                    return ib
-        if identicalbooks:
-            return identicalbooks.pop()
-        return None
+    # def _find_existing_book_id(self,db,book,matchurl=True):
+    #     mi = MetaInformation(book["title"],book["author"]) # author is a list.
+    #     identicalbooks = db.find_identical_books(mi)
+    #     if matchurl: # only *really* identical if URL matches, too.
+    #         for ib in identicalbooks:
+    #             if self._get_story_url(db,ib) == book['url']:
+    #                 return ib
+    #     if identicalbooks:
+    #         return identicalbooks.pop()
+    #     return None
    
    def _make_mi_from_book(self,book):
-        mi = MetaInformation(book['title'],(book['author'],)) # author is a list.
+        mi = MetaInformation(book['title'],book['author']) # author is a list.
        mi.set_identifiers({'url':book['url']})
        mi.publisher = book['publisher']
        mi.tags = book['tags']
@@ -895,7 +1158,15 @@ make_firstimage_cover:true
        books = []
        uniqueurls = set()
        for url in urls:
+            # look here for [\d,\d] at end of url, and remove?
+            mc = re.match(r"^(?P<url>.*?)(?:\[(?P<begin>\d+)?(?P<comma>,)?(?P<end>\d+)?\])?$",url)
+            print("url:(%s) begin:(%s) end:(%s)"%(mc.group('url'),mc.group('begin'),mc.group('end')))
+            url = mc.group('url')
            book = self._convert_url_to_book(url)
+            book['begin'] = mc.group('begin')
+            book['end'] = mc.group('end')
+            if book['begin'] and not mc.group('comma'):
+                book['end'] = book['begin']
            if book['url'] in uniqueurls:
                book['good'] = False
                book['comment'] = "Same story already included."
@@ -908,8 +1179,9 @@ make_firstimage_cover:true
        book['good'] = True
        book['calibre_id'] = None
        book['title'] = 'Unknown'
-        book['author'] = 'Unknown'
-        book['author_sort'] = 'Unknown'
+        book['author_sort'] = book['author'] = ['Unknown'] # list
+        book['begin'] = None
+        book['end'] = None
            
        book['comment'] = ''
        book['url'] = ''
@@ -923,8 +1195,9 @@ make_firstimage_cover:true
        book['good'] = good
        book['calibre_id'] = idval
        book['title'] = 'Unknown'
-        book['author'] = 'Unknown'
-        book['author_sort'] = 'Unknown'
+        book['author_sort'] = book['author'] = ['Unknown'] # list
+        book['begin'] = None
+        book['end'] = None
            
        book['comment'] = ''
        book['url'] = ''
@@ -938,7 +1211,7 @@ make_firstimage_cover:true
        book['good'] = True
        book['calibre_id'] = mi.id
        book['title'] = mi.title
-        book['author'] = authors_to_string(mi.authors)
+        book['author'] = mi.authors
        book['author_sort'] = mi.author_sort
        book['comment'] = ''
        book['url'] = ""
@@ -953,6 +1226,7 @@ make_firstimage_cover:true
            book['comment'] = "No story URL found."
            book['good'] = False
            book['icon'] = 'search_delete_saved.png'
+            book['status'] = 'Not Found'
        else:
            # get normalized url or None.
            book['url'] = self._is_good_downloader_url(url)
@@ -961,6 +1235,7 @@ make_firstimage_cover:true
                book['comment'] = "URL is not a valid story URL."
                book['good'] = False
                book['icon']='dialog_error.png'
+                book['status'] = 'Bad URL'
    
    def _get_story_url(self, db, book_id):
        identifiers = db.get_identifiers(book_id,index_is_id=True)
@@ -977,16 +1252,20 @@ make_firstimage_cover:true
                if 'url' in identifiers:
                    #print("url from epub:"+identifiers['url'].replace('|',':'))
                    return identifiers['url'].replace('|',':')
-                # look for dc:source
-                return get_dcsource(existingepub)
+                # look for dc:source first, then scan HTML if 
+                link = get_dcsource(existingepub)
+                if link:
+                    return link
+                elif prefs['lookforurlinhtml']:
+                    return get_story_url_from_html(existingepub,self._is_good_downloader_url)
        return None

    def _is_good_downloader_url(self,url):
-        # this is the accepted way to 'check for existance'?  really?
+        # this is the accepted way to 'check for existance of a class variable'?  really?
        try:
            self.dummyconfig
        except AttributeError:
-            self.dummyconfig = SafeConfigParser()
+            self.dummyconfig = Configuration("test1.com","EPUB")
        # pulling up an adapter is pretty low over-head.  If
        # it fails, it's a bad url.
        try:
@@ -10,7 +10,6 @@ __docformat__ = 'restructuredtext en'

 import time, os, traceback

-from ConfigParser import SafeConfigParser
 from StringIO import StringIO

 from calibre.utils.ipc.server import Server
@@ -20,6 +19,7 @@ from calibre.utils.logging import Log
 from calibre_plugins.fanfictiondownloader_plugin.dialogs import (NotGoingToDownload,
    OVERWRITE, OVERWRITEALWAYS, UPDATE, UPDATEALWAYS, ADDNEW, SKIP, CALIBREONLY)
 from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapters, writers, exceptions
+from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.configurable import Configuration
 from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.epubutils import get_update_data

 # ------------------------------------------------------------------------------
@@ -87,7 +87,15 @@ def do_download_worker(book_list, options,
        print(job.details)

        if count >= total:
-            # All done!
+            # All done!  Output some lists for convenience of some users.
+            print("Successfully downloaded:")
+            for book in book_list:
+                if book['good']:
+                    print(book['title'])
+            print("\nUnsuccessful:")
+            for book in book_list:
+                if not book['good']:
+                    print(book['title'])
            break

    server.close()
@@ -101,19 +109,37 @@ def do_download_for_worker(book,options):
    when run as a worker job
    '''
    try:
+        # import logging
+        # logging.basicConfig(level=logging.DEBUG,format="%(levelname)s:%(filename)s(%(lineno)d):%(message)s")
+        
        book['comment'] = 'Download started...'

-        ffdlconfig = SafeConfigParser()
-        ffdlconfig.readfp(StringIO(get_resources("plugin-defaults.ini")))
-        ffdlconfig.readfp(StringIO(options['personal.ini']))
+        configuration = Configuration(adapters.getConfigSectionFor(book['url']),options['fileform'])
+        configuration.readfp(StringIO(get_resources("plugin-defaults.ini")))
+        configuration.readfp(StringIO(options['personal.ini']))
        
-        adapter = adapters.getAdapter(ffdlconfig,book['url'],options['fileform'])
+        if not options['updateepubcover'] and 'epub_for_update' in book and options['collision'] in (UPDATE, UPDATEALWAYS):
+            configuration.set("overrides","never_make_cover","true")
+
+        # images only for epub, even if the user mistakenly turned it
+        # on else where.
+        if options['fileform'] != "epub":
+            configuration.set("overrides","include_images","false")
+        
+        adapter = adapters.getAdapter(configuration,book['url'])
        adapter.is_adult = book['is_adult'] 
        adapter.username = book['username'] 
        adapter.password = book['password']
        
        story = adapter.getStoryMetadataOnly()
-        writer = writers.getWriter(options['fileform'],adapter.config,adapter)
+        if 'calibre_series' in book:
+            adapter.setSeries(book['calibre_series'][0],book['calibre_series'][1])
+            
+        # set PI version instead of default.
+        if 'version' in options:
+            story.setMetadata('version',options['version'])
+            
+        writer = writers.getWriter(options['fileform'],configuration,adapter)

        outfile = book['outfile']

@@ -125,19 +151,26 @@ def do_download_for_worker(book,options):
        ## checks were done earlier, it's new or not dup or newer--just write it.
        elif options['collision'] in (ADDNEW, SKIP, OVERWRITE, OVERWRITEALWAYS) or \
                ('epub_for_update' not in book and options['collision'] in (UPDATE, UPDATEALWAYS)):
+
+            adapter.setChaptersRange(book['begin'],book['end'])
+            
            print("write to %s"%outfile)
            writer.writeStory(outfilename=outfile, forceOverwrite=True)
            book['comment'] = 'Download %s completed, %s chapters.'%(options['fileform'],story.getMetadata("numChapters"))
            
        ## checks were done earlier, just update it.
        elif 'epub_for_update' in book and options['collision'] in (UPDATE, UPDATEALWAYS):
-            
+
            # update now handled by pre-populating the old images and
            # chapters in the adapter rather than merging epubs.
            urlchaptercount = int(story.getMetadata('numChapters'))
-            (url,chaptercount,
+            (url,
+             chaptercount,
             adapter.oldchapters,
-             adapter.oldimgs) = get_update_data(book['epub_for_update'])
+             adapter.oldimgs,
+             adapter.oldcover,
+             adapter.calibrebookmark,
+             adapter.logfile) = get_update_data(book['epub_for_update'])

            print("Do update - epub(%d) vs url(%d)" % (chaptercount, urlchaptercount))
            print("write to %s"%outfile)
@@ -156,6 +189,7 @@ def do_download_for_worker(book,options):
        book['good']=False
        book['comment']=unicode(e)
        book['icon']='dialog_error.png'
+        book['status'] = 'Error'
        print("Exception: %s:%s"%(book,unicode(e)))
        traceback.print_exc()

@@ -18,6 +18,10 @@
 ## [defaults] section applies to all formats and sites but may be
 ## overridden at several levels

+## Some sites also require the user to confirm they are adult for
+## adult content.  Uncomment by removing '#' in front of is_adult.
+#is_adult:true
+
 ## All available titlepage_entries and the label used for them:
 ## <entryname>_label:<label>
 ## Labels may be customized.
@@ -38,6 +42,7 @@ category_label:Category
 genre_label:Genre
 language_label:Language
 characters_label:Characters
+ships_label:Relationships
 series_label:Series
 ## Completed/In-Progress
 status_label:Status
@@ -62,13 +67,21 @@ authorId_label:Author ID
 ## show up in Calibre as tags.  Also carried into mobi when converted.
 extratags_label:Extra Tags
 ## The version of fanficdownloader
-##
 version_label:FFDL Version

+## Date formats used by FFDL.  Published and Update don't have time.
+## See http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+## Note that ini format requires % to be escaped as %%.
+dateCreated_format:%%Y-%%m-%%d %%H:%%M:%%S
+datePublished_format:%%Y-%%m-%%d
+dateUpdated_format:%%Y-%%m-%%d
+
 ## items to include in the title page
-## Empty entries will *not* appear, even if in the list.
+## Empty metadata entries will *not* appear, even if in the list.
+## You can include extra text or HTML that will be included as-is in
+## the title page. Eg: titlepage_entries: ...,<br />,summary,<br />,...
 ## All current formats already include title and author.
-titlepage_entries: series,category,genre,language,characters,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,description
+titlepage_entries: series,category,genre,language,characters,ships,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,description

 ## Try to collect series name and number of this story in series.
 ## Some sites (ab)use 'series' for reading lists and personal
@@ -119,11 +132,20 @@ allow_unsafe_filename: false

 ## entries to make epub subjects and calibre tags
 ## lastupdate creates two tags: "Last Update Year/Month: %Y/%m" and "Last Update: %Y/%m/%d"
-include_subject_tags: extratags, genre, category, characters, lastupdate, status
+include_subject_tags: extratags, genre, category, characters, ships, lastupdate, status

 ## extra tags (comma separated) to include, primarily for epub.
 extratags: FanFiction

+## extra categories, genres, characters, ships and warnings can be
+## configured.  Used primarily for sites that are dedicated to a genre
+## or 'ship and so don't included it for every story.
+#extracategories:
+#extragenres:
+#extracharacters:
+#extraships:
+#extrawarnings:
+
 ## number of seconds to sleep between calls to the story site.  May by
 ## useful if pulling large numbers of stories or if the site is slow.
 #slow_down_sleep_time:0.5
@@ -139,12 +161,14 @@ extratags: FanFiction
 ## for regexp details.
 ## Make sure to keep at least one space at the start of each line and
 ## to escape % to %%, if used.
+## Two or three part lines.  Two part effect everything.
+## Three part effect only those key(s) lists.
 #replace_metadata:
-# Sci-Fi=>SF
+# genre,category=>Sci-Fi=>SF
 # Puella Magi Madoka Magica.* => Madoka
 # Comedy=>Humor
 # Crossover: (.*)=>\1
-# (.*)Great(.*)=>\1Moderate\2
+# title=>(.*)Great(.*)=>\1Moderate\2
 # .*-Centered=>

 ## Some readers don't show horizontal rule (<hr />) tags correctly.
@@ -152,6 +176,45 @@ extratags: FanFiction
 ## doesn't work on some devices either.)
 #replace_hr: false

+## If set false, the summary will have all html stripped.
+## Both this and include_images must be true to get images in the
+## summary.
+keep_summary_html:true
+
+## Don't like the numbers at the start of chapter titles on some
+## sites?  You can use strip_chapter_numbers to strip them off.  Just
+## want to make them all look the same?  Strip them off, then add them
+## back on with add_chapter_numbers.  Don't like the way it strips
+## numbers or adds them back?  See chapter_title_strip_pattern and
+## chapter_title_add_pattern.
+strip_chapter_numbers:false
+add_chapter_numbers:false
+
+## (Two versions of chapter_title_strip_pattern are shown below.  You
+## should only have one uncommented.)
+## This version will remove the leading number from:
+## "1." => ""
+## "1. The Beginning" => "The Beginning"
+## "1: Start" => "Start"
+## "2, Chapter the second" => "Chapter the second"
+## etc
+chapter_title_strip_pattern:^[0-9]+[\.: -]+
+
+## This version will strip all of the above *plus* remove 'Chapter 1':
+## "Chapter 1" => ""
+## "1. Chapter 1" => ""
+## "1. Chapter 1, Bob's First Clue" => "Bob's First Clue"
+## "Chapter 2 - Pirates Place" => "Pirates Place"
+## etc
+#chapter_title_strip_pattern:^([0-9]+[\.: -]+)?(Chapter *[0-9]+[\.:, -]*)?
+
+## Uses a python template substitution.  The ${index} is the 'chapter'
+## number and ${title} is the chapter title, after applying
+## chapter_title_strip_pattern.  Those are the only variables available.
+## "The Beginning" => "1. The Beginning" 
+chapter_title_add_pattern:${index}. ${title}
+
+
 ## Each output format has a section that overrides [defaults]
 [html]

@@ -183,6 +246,9 @@ output_css:
 ## Add URLs since there aren't links.
 titlepage_entries: series,category,genre,language,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,storyUrl, authorUrl, description

+## Width to word wrap text output.  0 indicates no wrapping.
+wrap_width: 78
+
 ## use \r\n for line endings, the windows convention.  text output only.
 windows_eol: true

@@ -192,9 +258,24 @@ windows_eol: true
 zip_output: false

 ## epub carries the TOC in metadata.
-## mobi generated from epub will have a TOC at the end.
+## mobi generated from epub by calibre will have a TOC at the end.
 include_tocpage: false

+## include a Update Log page before the story text.  If included, the
+## log will be updated each time the epub is an all the metadata
+## fields that have changed since the last update (typically
+## dateUpdated,numChapters,numWords at a minimum) will be shown.
+## Great for tracking when chapters came out and when the description,
+## etc changed.
+include_logpage: false
+
+## items to include in the log page Empty metadata entries, or those
+## that haven't changed since the last update, will *not* appear, even
+## if in the list.  You can include extra text or HTML that will be
+## included as-is in each log entry. Eg: logpage_entries: ...,<br />,
+## summary,<br />,...
+logpage_entries: dateCreated,datePublished,dateUpdated,numChapters,numWords,status,title,author,description,category,genre,rating,warnings
+
 ## epub->mobi conversions typically don't like tables.
 titlepage_use_table: false

@@ -209,10 +290,12 @@ background_color: ffffff
 ## Allow customization of CSS.  Make sure to keep at least one space
 ## at the start of each line and to escape % to %%.  Also need
 ## background_color to be in the same section, if included in CSS.
+## 'adobe-text-layout: optimizeSpeed;' prevents hyphenation on newer Nooks
 output_css: 
 body { background-color: #%(background_color)s;
        text-align: justify;
-        margin: 2%%; }
+        margin: 2%%;
+	adobe-text-layout: optimizeSpeed; }
 pre { font-size: x-small; }
 sml { font-size: small; }
 h1 { text-align: center; }
@@ -239,11 +322,6 @@ output_css:
 ## stories.  Images will be converted to jpg for size if possible.
 #include_images:false

-## If not set, the summary will have all html stripped for safety.
-## Both this and include_images must be true to get images in the
-## summary.
-#keep_summary_html:false
-
 ## If set, the first image found will be made the cover image.  If
 ## keep_summary_html is true, any images in summary will be before any
 ## in chapters.
@@ -296,6 +374,32 @@ nook_img_fix:true
 ## URLs like: http://test1.com?sid=12345
 [test1.com]
 extratags: FanFiction,Testing
+extracategories:Fafner
+extragenres:Romance,Fluff
+extracharacters:Reginald Smythe-Smythe,Mokona,Harry P.
+extraships:Smythe-Smythe/Mokona
+extrawarnings:Extreme Bogosity
+
+extra_valid_entries:metaA,metaB,metaC,listX,listY,listZ,compositeJ,compositeK,compositeL
+
+include_in_compositeJ:dateCreated
+include_in_compositeK:metaC,listX,compositeL,compositeJ,compositeK,listZ
+include_in_compositeL:ships,metaA,listZ,datePublished,dateUpdated,
+
+extra_titlepage_entries: metaA,metaB,metaC,listX,listY,listZ,compositeJ,compositeK,compositeL
+extra_logpage_entries: metaA,metaB,metaC,listX,listY,listZ,compositeJ,compositeK,compositeL
+extra_subject_tags: metaA,metaB,metaC
+
+replace_metadata:
+ compositeL=>Val=>VALUE
+ series,extratags=>Test=>Plan
+ Puella Magi Madoka Magica.* => Madoka
+ Comedy=>Humor
+ Crossover: (.*)=>\1
+ (.*)Great(.*)=>\1Moderate\2
+ .*-Centered=>
+ characters=>Harry P\.=>Harry Potter
+

 ## If necessary, you can define [<site>:<format>] sections to
 ## customize the formats differently for the same site.  Overrides
@@ -306,7 +410,9 @@ extratags: FanFiction,Testing,Text
 [test1.com:html]
 extratags: FanFiction,Testing,HTML

-[castlefans.org]
+[archive.skyehawke.com]
+
+[archiveofourown.org]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -319,7 +425,159 @@ extratags: FanFiction,Testing,HTML
 ## personal.ini, not defaults.ini.
 #is_adult:true

+## AO3 adapter defines a few extra metadata entries.
+extra_valid_entries:fandoms,freefromtags,ao3categories,comments,kudos,hits,bookmarks
+fandoms_label:Fandoms
+freefromtags_label:Freeform Tags
+ao3categories_label:AO3 Categories
+comments_label:Comments
+kudos_label:Kudos
+hits_label:Hits
+bookmarks:Bookmarks
+
+## adds to titlepage_entries instead of replacing it.
+#extra_titlepage_entries: fandoms,freefromtags,ao3categories,comments,kudos,hits,bookmarks
+
+## adds to include_subject_tags instead of replacing it.
+#extra_subject_tags:fandoms,freefromtags,ao3categories
+
+[ashwinder.sycophanthex.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+extracharacters:Severus Snape,Hermione Granger
+extraships:Severus Snape/Hermione Granger
+
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+[bloodties-fans.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Blood Ties
+
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
+[castlefans.org]
+## Site dedicated to these categories/characters/ships
+extracategories:Castle
+
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
+[chaos.sycophanthex.com]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+## some sites include images that we don't ever want becoming the
+## cover image.  This lets you exclude them.
+cover_exclusion_regexp:/images/.*?ribbon.gif
+
+[dark-solace.org]
+## Site dedicated to these categories/characters/ships
+extracategories:Buffy: The Vampire Slayer
+extracharacters:Buffy, Spike
+extraships:Spike/Buffy
+
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+[dramione.org]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+extracharacters:Draco Malfoy,Hermione Granger
+extraships:Draco Malfoy/Hermione Granger
+
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
+## Some adapters collect additional meta information beyond the
+## standard ones.  They need to be defined in extra_valid_entries to
+## tell the rest of the FFDL system about them.  They can be used in
+## include_subject_tags, titlepage_entries, extra_titlepage_entries,
+## logpage_entries, extra_logpage_entries, and include_in_* config
+## items.  You can also add additional entries here to build up
+## composite metadata entries.  dramione.org, for example, adds
+## 'cliches' and then defines as the composite of hermiones,dracos in
+## include_in_cliches.
+extra_valid_entries:themes,hermiones,dracos,timeline,cliches
+include_in_cliches:hermiones,dracos
+
+## For another example, you could, by uncommenting this line, include
+## themes in with genre metadata.
+#include_in_genre:genre, themes
+
+## You can give each new valid entry a specific label for use on
+## titlepage and logpage.  If not defined, it will simply be the 
+themes_label:Themes
+hermiones_label:Hermiones
+dracos_label:Dracos
+timeline_label:Timeline
+cliches_label:Character Cliches
+
+## extra_titlepage_entries (and extra_logpage_entries) *add* to
+## titlepage_entries (and logpage_entries) so you can add site
+## specific entries to titlepage/logpage without having to copy the
+## entire titlepage_entries line.  (But if you want them higher than
+## the end, you will need to copy titlepage_entries.)
+#extra_titlepage_entries: themes,hermiones,dracos,timeline,cliches
+
+## adds to include_subject_tags instead of replacing it.
+#extra_subject_tags: themes,timeline,cliches
+
+[erosnsappho.sycophanthex.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+## some sites include images that we don't ever want becoming the
+## cover image.  This lets you exclude them.
+cover_exclusion_regexp:/images/.*?ribbon.gif
+
 [fanfiction.mugglenet.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -333,6 +591,10 @@ extratags: FanFiction,Testing,HTML
 #is_adult:true

 [fanfiction.portkey.org]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+extraships:Harry Potter/Hermione Granger
+
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
@@ -346,18 +608,202 @@ extratags: FanFiction,Testing,HTML
 #username:YourName
 #password:yourpassword

-[nfacommunity.com]
+[finestories.com]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+[grangerenchanted.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+extracharacters:Hermione Granger
+
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
+[hlfiction.net]
+## Site dedicated to these categories/characters/ships
+extracategories:Highlander
+
+[ksarchive.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Star Trek
+extracharacters:Kirk,Spock
+extraships:Kirk/Spock
+
+[lumos.sycophanthex.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true

+[merlinfic.dtwins.co.uk]
+## Site dedicated to these categories/characters/ships
+extracategories:Merlin
+
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
+[national-library.net]
+## Site dedicated to these categories/characters/ships
+extracategories:West Wing
+
+[ncisfic.com]
+## Site dedicated to these categories/characters/ships
+extracategories:NCIS
+
+[nfacommunity.com]
+## Site dedicated to these categories/characters/ships
+extracategories:NCIS
+
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+[nha.magical-worlds.us]
+## Site dedicated to these categories/characters/ships
+extracategories:Buffy: The Vampire Slayer
+extracharacters:Willow
+
+[occlumency.sycophanthex.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+extracharacters:Severus Snape
+
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+[onedirectionfanfiction.com]
+## Site dedicated to these categories/characters/ships
+extracategories:One Direction
+
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
+[ponyfictionarchive.net]
+## Site dedicated to these categories/characters/ships
+extracategories:My Little Pony: Friendship is Magic
+
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+[pretendercentre.com]
+## Site dedicated to these categories/characters/ships
+extracategories:The Pretender
+
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+[samdean.archive.nu]
+## Site dedicated to these categories/characters/ships
+extracategories:Supernatural
+extracharacters:Sam,Dean
+extraships:Sam/Dean
+
+[sg1-heliopolis.com]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+[stargate-atlantis.org]
+## Site dedicated to these categories/characters/ships
+extracategories:Stargate: Atlantis
+
+[svufiction.com]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
+[thehexfiles.net]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+extracharacters:Draco Malfoy,Harry Potter
+extraships:Harry Potter/Draco Malfoy
+
+[thehookupzone.net]
+## Site dedicated to these categories/characters/ships
+extracategories:Criminal Minds
+
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
+[themasque.net]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
 ## Some sites also require the user to confirm they are adult for
 ## adult content.  In commandline version, this should go in your
 ## personal.ini, not defaults.ini.
 #is_adult:true

 [thequidditchpitch.org]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -371,22 +817,82 @@ extratags: FanFiction,Testing,HTML
 #is_adult:true

 [www.adastrafanfic.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Star Trek
+
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true

-[www.archiveofourown.org]
+[www.dracoandginny.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+extracharacters:Draco Malfoy,Ginny Weasley
+extraships:Draco Malfoy/Ginny Weasley
+
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
 ## Some sites also require the user to confirm they are adult for
 ## adult content.  In commandline version, this should go in your
 ## personal.ini, not defaults.ini.
 #is_adult:true

+[www.thealphagate.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Stargate: SG-1
+
+[www.checkmated.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+[www.destinysgateway.com]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+[www.dokuga.com]
+## Site dedicated to these categories/characters/ships
+extracategories:InuYasha
+extracharacters:Sesshoumaru,Kagome
+extraships:Sesshoumaru/Kagome
+
 [www.fanfiction.net]
+## fanfiction.net's 'cover' images are really just tiny thumbnails.
+## Change this to false to use them anyway.
+never_make_cover: true
+
+## Extra metadata that this adapter knows about.  See [dramione.org]
+## for examples of how to use them.
+extra_valid_entries:reviews,favs,follows
+
+[www.fanfiktion.de]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword

 [www.ficbook.net]

 [www.fictionalley.org]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
@@ -399,6 +905,10 @@ output_filename: ${title}-${siteabbrev}_${authorId}_${storyId}${formatext}
 ## Clear FanFiction from defaults, fictionpress.com is original fiction.
 extratags:

+## Extra metadata that this adapter knows about.  See [dramione.org]
+## for examples of how to use them.
+extra_valid_entries:reviews,favs,follows
+
 [www.ficwad.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -408,6 +918,9 @@ extratags:
 #password:yourpassword

 [www.fimfiction.net]
+## Site dedicated to these categories/characters/ships
+extracategories:My Little Pony: Friendship is Magic
+
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
@@ -419,12 +932,49 @@ extratags:
 #fail_on_password: false

 [www.harrypotterfanfiction.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true

 [www.hpfandom.net]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+[www.hpfanficarchive.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
+[www.ik-eternal.net]
+## Site dedicated to these categories/characters/ships
+extracategories:InuYasha
+extracharacters:InuYasha,Kagome
+extraships:InuYasha/Kagome
+
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
+[www.libraryofmoria.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Lord of the Rings
+
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
@@ -433,6 +983,9 @@ extratags:
 [www.mediaminer.org]

 [www.midnightwhispers.ca]
+## Site dedicated to these categories/characters/ships
+extracategories:Queer as Folk
+
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
@@ -442,9 +995,77 @@ extratags:
 ## cover image.  This lets you exclude them.
 cover_exclusion_regexp:/stories/999/images/.*?_trophy.png

+[www.ncisfiction.com]
+## Site dedicated to these categories/characters/ships
+extracategories:NCIS
+
+[www.phoenixsong.net]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+extraships:Harry Potter/Ginny Weasley
+
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
 [www.potionsandsnitches.net]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
+[www.prisonbreakfic.net]
+## Site dedicated to these categories/characters/ships
+extracategories:Prison Break
+
+[www.scarvesandcoffee.net]
+## Site dedicated to these categories/characters/ships
+extracategories:Glee
+extracharacters:Kurt Hummel,Blaine Anderson
+
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+[www.sinful-desire.org]
+## Site dedicated to these categories/characters/ships
+extracategories:Supernatural
+
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true

 [www.siye.co.uk]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+extracharacters:Harry Potter,Ginny Weasley
+extraships:Harry Potter/Ginny Weasley
+
+[www.squidge.org/peja]
+# www.squidge.org/peja calls it Fandom <shrug>
+category_label:Fandom
+# Remove numWords -- www.squidge.org/peja word counts are inaccurate
+titlepage_entries: series,category,genre,language,characters,ships,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,site,description
+
+[www.squidge.org/peja:txt]
+## Add URLs since there aren't links.
+# Remove numWords -- www.squidge.org/peja word counts are inaccurate
+titlepage_entries: series,category,genre,language,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,site,storyUrl, authorUrl, description
+
+[www.storiesofarda.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Lord of the Rings
+
+[www.thepetulantpoetess.com]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword

 [www.thewriterscoffeeshop.com]
 ## Some sites require login (or login for some rated stories) The
@@ -475,7 +1096,14 @@ collect_series: false
 #username:YourName
 #password:yourpassword

+[www.twilightarchives.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Twilight
+
 [www.twilighted.net]
+## Site dedicated to these categories/characters/ships
+extracategories:Twilight
+
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -487,6 +1115,9 @@ collect_series: false
 collect_series: false

 [www.twiwrite.net]
+## Site dedicated to these categories/characters/ships
+extracategories:Twilight
+
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -497,8 +1128,52 @@ collect_series: false
 ## twiwrite.net (ab)uses series as personal reading lists.
 collect_series: false

+[www.walkingtheplank.org]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+extracharacters:Severus Snape,Harry Potter
+extraships:Severus Snape/Harry Potter
+
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
 [www.whofic.com]

+[www.wizardtales.net]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
+[www.wolverineandrogue.com]
+## Site dedicated to these categories/characters/ships
+extracategories:X-Men Movie
+extracharacters:Wolverine,Rogue
+
+[www.wraithbait.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Stargate: Atlantis
+
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+[www.yourfanfiction.com]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
 [overrides]
 ## It may sometimes be useful to override all of the specific format,
 ## site and site:format sections in your private configuration.  For
@@ -16,8 +16,6 @@
 #

 import logging
-## XXX cli option for logging level.
-logging.basicConfig(level=logging.DEBUG,format="%(levelname)s:%(filename)s(%(lineno)d):%(message)s")

 import sys, os
 from os.path import normpath, expanduser, isfile, join
@@ -29,7 +27,9 @@ import ConfigParser
 from subprocess import call

 from fanficdownloader import adapters,writers,exceptions
+from fanficdownloader.configurable import Configuration
 from fanficdownloader.epubutils import get_dcsource_chaptercount, get_update_data
+from fanficdownloader.geturls import get_urls_from_page

 if sys.version_info < (2, 5):
    print "This program requires Python 2.5 or newer."
@@ -64,19 +64,42 @@ def main():
   parser.add_option("-u", "--update-epub",
                     action="store_true", dest="update",
                     help="Update an existing epub with new chapter, give epub filename instead of storyurl.",)
+   parser.add_option("--update-cover",
+                     action="store_true", dest="updatecover",
+                     help="Update cover in an existing epub, otherwise existing cover (if any) is used on update.  Only valid with --update-epub.",)
   parser.add_option("--force",
                     action="store_true", dest="force",
                     help="Force overwrite or update of an existing epub, download and overwrite all chapters.",)
+   parser.add_option("-l", "--list",
+                     action="store_true", dest="list",
+                     help="Get list of valid story URLs from page given.",)
+   parser.add_option("-d", "--debug",
+                     action="store_true", dest="debug",
+                     help="Show debug output while downloading.",)
   
   (options, args) = parser.parse_args()

+   if options.debug:
+       logging.basicConfig(level=logging.DEBUG,format="%(levelname)s:%(filename)s(%(lineno)d):%(message)s")
+   else:
+       logging.basicConfig(level=logging.INFO,format="%(levelname)s:%(filename)s(%(lineno)d):%(message)s")
+
+   
   if len(args) != 1:
       parser.error("incorrect number of arguments")

   if options.update and options.format != 'epub':
       parser.error("-u/--update-epub only works with epub")

-   config = ConfigParser.SafeConfigParser()
+   ## Attempt to update an existing epub.
+   if options.update:
+       (url,chaptercount) = get_dcsource_chaptercount(args[0])
+       print "Updating %s, URL: %s" % (args[0],url)
+       output_filename = args[0]
+   else:
+       url = args[0]
+           
+   configuration = Configuration(adapters.getConfigSectionFor(url),options.format)

   conflist = []
   homepath = join(expanduser("~"),".fanficdownloader")
@@ -95,41 +118,57 @@ def main():
       conflist.extend(options.configfile)
      
   logging.debug('reading %s config file(s), if present'%conflist)
-   config.read(conflist)
+   configuration.read(conflist)
+
+   print("has include_in_tags?%s"%configuration.hasConfig("include_in_tags"))

   try:
-       config.add_section("overrides")
+       configuration.add_section("overrides")
   except ConfigParser.DuplicateSectionError:
       pass

   if options.force:
-       config.set("overrides","always_overwrite","true")
+       configuration.set("overrides","always_overwrite","true")
+
+   if options.update:
+       configuration.set("overrides","output_filename",args[0])
+       
+   if options.update and not options.updatecover:
+       configuration.set("overrides","never_make_cover","true")
+
+   # images only for epub, even if the user mistakenly turned it
+   # on else where.
+   if options.format != "epub":
+       configuration.set("overrides","include_images","false")
       
   if options.options:
       for opt in options.options:
           (var,val) = opt.split('=')
-           config.set("overrides",var,val)
+           configuration.set("overrides",var,val)
+
+   if options.list:
+       retlist = get_urls_from_page(args[0], configuration)
+       print "\n".join(retlist)
+               
+       return

   try:
-       ## Attempt to update an existing epub.
-       if options.update:
-           (url,chaptercount) = get_dcsource_chaptercount(args[0])
-           print "Updating %s, URL: %s" % (args[0],url)
-           output_filename = args[0]
-           config.set("overrides","output_filename",args[0])
-       else:
-           url = args[0]

-       adapter = adapters.getAdapter(config,url,options.format)
+       adapter = adapters.getAdapter(configuration,url)

       ## Check for include_images and absence of PIL, give warning.
       if adapter.getConfig('include_images'):
           try:
-               import Image
+               from calibre.utils.magick import Image
+               logging.debug("Using calibre.utils.magick")
           except:
-               print "You have include_images enabled, but Python Image Library(PIL) isn't found.\nImages will be included full size in original format.\nContinue? (y/n)?"
-               if not sys.stdin.readline().strip().lower().startswith('y'):
-                   return
+               try:
+                   import Image
+                   logging.debug("Using PIL")
+               except:
+                   print "You have include_images enabled, but Python Image Library(PIL) isn't found.\nImages will be included full size in original format.\nContinue? (y/n)?"
+                   if not sys.stdin.readline().strip().lower().startswith('y'):
+                       return
               

       ## three tries, that's enough if both user/pass & is_adult needed,
@@ -165,11 +204,15 @@ def main():
                   # update now handled by pre-populating the old
                   # images and chapters in the adapter rather than
                   # merging epubs.
-                   (url,chaptercount,
+                   (url,
+                    chaptercount,
                    adapter.oldchapters,
-                    adapter.oldimgs) = get_update_data(args[0])
+                    adapter.oldimgs,
+                    adapter.oldcover,
+                    adapter.calibrebookmark,
+                    adapter.logfile) = get_update_data(args[0])

-                   writeStory(config,adapter,"epub")
+                   writeStory(configuration,adapter,"epub")
                   
       else:
           # regular download
@@ -178,7 +221,7 @@ def main():
           
           adapter.setChaptersRange(options.begin,options.end)
           
-           output_filename=writeStory(config,adapter,options.format,options.metaonly)
+           output_filename=writeStory(configuration,adapter,options.format,options.metaonly)
       
       if not options.metaonly and adapter.getConfig("post_process_cmd"):
           metadata = adapter.story.metadata
@@ -194,7 +237,7 @@ def main():
       print dne
   except exceptions.UnknownSite, us:
       print us
-   
+
 if __name__ == "__main__":
    #import time
    #start = time.time()
@@ -1,40 +1,103 @@
 ## This is an example of what your personal configuration might look
-## like.
+## like.  Uncomment options by removing the '#' in front of them.

 [defaults]
 ## Some sites also require the user to confirm they are adult for
-## adult content.  In commandline version, this should go in your
-## personal.ini, not defaults.ini.
+## adult content.  Uncomment by removing '#' in front of is_adult.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
 #is_adult:true

+## Don't like the numbers at the start of chapter titles on some
+## sites?  You can use strip_chapter_numbers to strip them off.  Just
+## want to make them all look the same?  Strip them off, then add them
+## back on with add_chapter_numbers.  Don't like the way it strips
+## numbers or adds them back?  See chapter_title_strip_pattern and
+## chapter_title_add_pattern.
+#strip_chapter_numbers:true
+#add_chapter_numbers:true
+
+[epub]
+## include images from img tags in the body and summary of stories.
+## Images will be converted to jpg for size if possible.  Images work
+## in epub format only.  To get mobi or other format with images,
+## download as epub and use Calibre to convert.
+#include_images:true
+
+## If not set, the summary will have all html stripped for safety.
+## Both this and include_images must be true to get images in the
+## summary.
+#keep_summary_html:true
+
+## If set, the first image found will be made the cover image.  If
+## keep_summary_html is true, any images in summary will be before any
+## in chapters.
+#make_firstimage_cover:true
+
+## Resize images down to width, height, preserving aspect ratio.
+## Nook size, with margin.
+#image_max_size: 580, 725
+
+## Change image to grayscale, if graphics library allows, to save
+## space.
+#grayscale_images: false
+
+
 ## Most common, I expect will be using this to save username/passwords
-## for different sites.
+## for different sites.  Here are a few examples.  See defaults.ini
+## for the full list.
+
 [www.twilighted.net]
 #username:YourPenname
 #password:YourPassword
+## default is false
+#collect_series: true

 [www.ficwad.com]
 #username:YourUsername
 #password:YourPassword

+[www.twiwrite.net]
+#username:YourName
+#password:yourpassword
+## default is false
+#collect_series: true
+
 [www.adastrafanfic.com]
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.
 #is_adult:true

-## The [defaults] section here will override the system [defaults],
-## but not format, site for site:format sections.
-[defaults]
-## Directories only useful in commandline or zip files.
-#output_filename: books/${title}-${siteabbrev}_${storyId}${formatext}
-#output_filename: books/${site}/${authorId}/${title}-${storyId}${formatext}
+[www.thewriterscoffeeshop.com]
+#username:YourName
+#password:yourpassword
+#is_adult:true
+## default is false
+#collect_series: true
+
+[www.fictionalley.org]
+#is_adult:true
+
+[www.harrypotterfanfiction.com]
+#is_adult:true
+
+[www.fimfiction.net]
+#is_adult:true
+#fail_on_password: false
+
+[www.tthfanfic.org]
+#is_adult:true
+## tth is a little unusual--it doesn't require user/pass, but the site
+## keeps track of which chapters you've read and won't send another
+## update until it thinks you're up to date.  This way, on download,
+## it thinks you're up to date.
+#username:YourName
+#password:yourpassword

-## For example, zip_output here will turn on zip for html and txt, but
-## not epub because the system [epub] section explicitly says
-## zip_output: false (epubs *are* specially formated zip files.)
-#zip_output: true
-#zip_filename: ${title}-${siteabbrev}_${storyId}${formatext}.zip

 ## This section will override anything in the system defaults or other
 ## sections here.
 [overrides]
+## default varies by site.  Set true here to force all sites to
+## collect series.
+#collect_series: true
@@ -52,6 +52,54 @@ import adapter_nfacommunitycom
 import adapter_midnightwhispersca
 import adapter_ksarchivecom
 import adapter_archiveskyehawkecom
+import adapter_squidgeorgpeja
+import adapter_libraryofmoriacom
+import adapter_wraithbaitcom
+import adapter_checkmatedcom
+import adapter_chaossycophanthexcom
+import adapter_dramioneorg
+import adapter_erosnsapphosycophanthexcom
+import adapter_lumossycophanthexcom
+import adapter_occlumencysycophanthexcom
+import adapter_phoenixsongnet
+import adapter_walkingtheplankorg
+import adapter_ashwindersycophanthexcom
+import adapter_thehexfilesnet
+import adapter_dokugacom
+import adapter_iketernalnet
+import adapter_onedirectionfanfictioncom
+import adapter_prisonbreakficnet
+import adapter_storiesofardacom
+import adapter_yourfanfictioncom
+import adapter_samdeanarchivenu
+import adapter_destinysgatewaycom
+import adapter_ncisfictioncom
+import adapter_stargateatlantisorg
+import adapter_thealphagatecom
+import adapter_fanfiktionde
+import adapter_ponyfictionarchivenet
+import adapter_sg1heliopoliscom
+import adapter_ncisficcom
+import adapter_nationallibrarynet
+import adapter_themasquenet
+import adapter_pretendercentrecom
+import adapter_darksolaceorg
+import adapter_finestoriescom
+import adapter_hpfanficarchivecom
+import adapter_svufictioncom
+import adapter_twilightarchivescom
+import adapter_wizardtalesnet
+import adapter_nhamagicalworldsus
+import adapter_hlfictionnet
+import adapter_grangerenchantedcom
+import adapter_dracoandginnycom
+import adapter_scarvesandcoffeenet
+import adapter_thepetulantpoetesscom
+import adapter_wolverineandroguecom
+import adapter_sinfuldesireorg
+import adapter_merlinficdtwinscouk
+import adapter_thehookupzonenet
+import adapter_bloodtiesfancom

 ## This bit of complexity allows adapters to be added by just adding
 ## importing.  It eliminates the long if/else clauses we used to need
@@ -69,10 +117,32 @@ for x in imports():
    if "fanficdownloader.adapters.adapter_" in x:
        #print x
        __class_list.append(sys.modules[x].getClass())
-            
-def getAdapter(config,url,fileform=None):
+
+def getAdapter(config,url):
+
+    logging.debug("trying url:"+url)
+    (cls,fixedurl) = getClassFor(url)
+    logging.debug("fixedurl:"+fixedurl)
+    if cls:
+        adapter = cls(config,fixedurl) # raises InvalidStoryURL
+        return adapter
+    # No adapter found.
+    raise exceptions.UnknownSite( url, [cls.getSiteDomain() for cls in __class_list] )
+
+def getConfigSections():
+    return [cls.getConfigSection() for cls in __class_list]
+
+def getConfigSectionFor(url):
+    (cls,fixedurl) = getClassFor(url)
+    if cls:
+        return cls.getConfigSection()
+    
+    # No adapter found.
+    raise exceptions.UnknownSite( url, [cls.getSiteDomain() for cls in __class_list] )
+
+def getClassFor(url):
    ## fix up leading protocol.
-    fixedurl = re.sub(r"(?i)^[htp]+[:/]+","http://",url.strip())
+    fixedurl = re.sub(r"(?i)^[htps]+[:/]+","http://",url.strip())
    if not fixedurl.startswith("http"):
        fixedurl = "http://%s"%url
    ## remove any trailing '#' locations.
@@ -87,20 +157,20 @@ def getAdapter(config,url,fileform=None):
    if( domain != parsedUrl.netloc ):
        fixedurl = fixedurl.replace(parsedUrl.netloc,domain)

-    logging.debug("site:"+domain)
-    cls = getClassFor(domain)
+    cls = getClassFromList(domain)
+    if not cls and domain.startswith("www."):
+        domain = domain.replace("www.","")
+        logging.debug("trying site:without www: "+domain)
+        cls = getClassFromList(domain)
+        fixedurl = fixedurl.replace("http://www.","http://")
    if not cls:
        logging.debug("trying site:www."+domain)
-        cls = getClassFor("www."+domain)
+        cls = getClassFromList("www."+domain)
        fixedurl = fixedurl.replace("http://","http://www.")
-    if cls:
-        adapter = cls(config,fixedurl) # raises InvalidStoryURL
-        adapter.setSectionOrder(adapter.getSiteDomain(),fileform)
-        return adapter
-    # No adapter found.
-    raise exceptions.UnknownSite( url, [cls.getSiteDomain() for cls in __class_list] )
-
-def getClassFor(domain):
+        
+    return (cls,fixedurl)
+    
+def getClassFromList(domain):
    for cls in __class_list:
        if cls.matchesSite(domain):
            return cls
@@ -37,7 +37,6 @@ class AdAstraFanficComSiteAdapter(BaseSiteAdapter):
                               # Most sites that claim to be
                               # iso-8859-1 (and some that claim to be
                               # utf8) are really windows-1252.
-        self.story.addToList("category","Star Trek")
        self.is_adult=False
        
        # get storyId from url--url validation guarantees query is only sid=1234
@@ -42,13 +42,25 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
                               # utf8) are really windows-1252.
 							   
 							   
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
        
        # get storyId from url--url validation guarantees query is only sid=1234
        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/works/'+self.story.getMetadata('storyId'))
+        # get storyId from url--url validation guarantees query correct
+        m = re.match(self.getSiteURLPattern(),url)
+        if m:
+            self.story.setMetadata('storyId',m.group('id'))
+            logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+            # normalized story URL.
+            self._setURL('http://' + self.getSiteDomain() + '/works/'+self.story.getMetadata('storyId'))
+        else:
+            raise exceptions.InvalidStoryURL(url,
+                                             self.getSiteDomain(),
+                                             self.getSiteExampleURLs())
        
        # Each adapter needs to have a unique site abbreviation.
        self.story.setMetadata('siteabbrev','ao3')
@@ -60,13 +72,51 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
    @staticmethod # must be @staticmethod, don't remove it.
    def getSiteDomain():
        # The site domain.  Does have www here, if it uses it.
-        return 'www.archiveofourown.org'
+        return 'archiveofourown.org'

    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/works/123456"
+        return "http://"+self.getSiteDomain()+"/works/123456 http://"+self.getSiteDomain()+"/collections/Some_Archive/works/123456"

    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/works/")+r"\d+(/chapters/\d+)?/?$"
+        # http://archiveofourown.org/collections/Smallville_Slash_Archive/works/159770
+        return re.escape("http://")+re.escape(self.getSiteDomain())+r"(/collections/[^/]+)?/works/(?P<id>\d+)"
+        
+    ## Login
+    def needToLoginCheck(self, data):
+        if 'This work is only available to registered users of the Archive.' in data \
+                or "The password or user name you entered doesn't match our records" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url, data):
+
+        params = {}
+        if self.password:
+            params['user_session[login]'] = self.username
+            params['user_session[password]'] = self.password
+        else:
+            params['user_session[login]'] = self.getConfig("username")
+            params['user_session[password]'] = self.getConfig("password")
+        params['user_session[remember_me]'] = '1'
+        params['commit'] = 'Log in'
+        #params['utf8'] = u'✓'#u'\x2713' # gets along with out it, and it confuses the encoder.
+        params['authenticity_token'] = data.split('input name="authenticity_token" type="hidden" value="')[1].split('" /></div>')[0]
+
+        loginUrl = 'http://' + self.getSiteDomain() + '/user_sessions'
+        logging.info("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['user_session[login]']))
+    
+        d = self._postUrl(loginUrl, params)
+        #logging.info(d)
+    
+        if "Successfully logged in" not in d : #Member Account
+            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['user_session[login]']))
+            raise exceptions.FailedToLogin(url,params['user_session[login]'])
+            return False
+        else:
+            return True

    ## Getting the chapter list and the meta data, plus 'is adult' checking.
    def extractChapterUrlsAndMetadata(self):
@@ -76,13 +126,14 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
        else:
            addurl=""

-        meta = self.url+addurl
+        metaurl = self.url+addurl
        url = self.url+'/navigate'+addurl
-        logging.debug("URL: "+meta)
+        logging.info("url: "+url)
+        logging.info("metaurl: "+metaurl)

        try:
            data = self._fetchUrl(url)
-            meta = self._fetchUrl(meta)
+            meta = self._fetchUrl(metaurl)

            if "This work could have adult content. If you proceed you have agreed that you are willing to see such content." in meta:
                raise exceptions.AdultCheckRequired(self.url)
@@ -92,23 +143,33 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
                raise exceptions.StoryDoesNotExist(self.meta)
            else:
                raise e
+                
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url,data)
+            data = self._fetchUrl(url)
+            meta = self._fetchUrl(metaurl)
            
        # use BeautifulSoup HTML parser to make everything easier to find.
        soup = bs.BeautifulSoup(data)
        metasoup = bs.BeautifulSoup(meta)
-        # print data

        # Now go hunting for all the meta data and the chapter list.
        
        ## Title
-        a = soup.find('a', href=re.compile(r"^/works/\w+"))
+        a = soup.find('a', href=re.compile(r"^/works/\d+$"))
        self.story.setMetadata('title',a.string)
 		
        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"^/users/\w+/pseuds/\w+"))
-        self.story.setMetadata('authorId',a['href'].split('/')[2])
-        self.story.setMetadata('authorUrl','http://'+self.host+a['href'])
-        self.story.setMetadata('author',a.text)
+        alist = soup.findAll('a', href=re.compile(r"^/users/\w+/pseuds/\w+"))
+        if len(alist) < 1: # ao3 allows for author 'Anonymous' with no author link.
+            self.story.setMetadata('author','Anonymous')
+            self.story.setMetadata('authorUrl',self.url)
+        else:
+            for a in alist:
+                self.story.addToList('authorId',a['href'].split('/')[2])
+                self.story.addToList('authorUrl','http://'+self.host+a['href'])
+                self.story.addToList('author',a.text)

        # Find the chapters:
        chapters=soup.findAll('a', href=re.compile(r'/works/'+self.story.getMetadata('storyId')+"/chapters/\d+$"))
@@ -126,7 +187,7 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):

        a = metasoup.find('blockquote',{'class':'userstuff'})
        if a != None:
-            self.setDescription(url,a.text)
+            self.setDescription(url,a)
            #self.story.setMetadata('description',a.text)
 		
        a = metasoup.find('dd',{'class':"rating tags"})
@@ -136,6 +197,7 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
        a = metasoup.find('dd',{'class':"fandom tags"})
        fandoms = a.findAll('a',{'class':"tag"})
        for fandom in fandoms:
+            self.story.addToList('fandoms',fandom.string)
            self.story.addToList('category',fandom.string)
 		
        a = metasoup.find('dd',{'class':"warning tags"})
@@ -151,12 +213,15 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
        if a != None:
            genres = a.findAll('a',{'class':"tag"})
            for genre in genres:
+                self.story.addToList('freefromtags',genre.string)
                self.story.addToList('genre',genre.string)
+                
        a = metasoup.find('dd',{'class':"category tags"})
        if a != None:
            genres = a.findAll('a',{'class':"tag"})
            for genre in genres:
                if genre != "Gen":
+                    self.story.addToList('ao3categories',genre.string)
                    self.story.addToList('genre',genre.string)
 		
        a = metasoup.find('dd',{'class':"character tags"})
@@ -166,9 +231,9 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
                self.story.addToList('characters',char.string)
        a = metasoup.find('dd',{'class':"relationship tags"})
        if a != None:
-            chars = a.findAll('a',{'class':"tag"})
-            for char in chars:
-                self.story.addToList('characters',char.string)
+            ships = a.findAll('a',{'class':"tag"})
+            for ship in ships:
+                self.story.addToList('ships',ship.string)
 		

        stats = metasoup.find('dl',{'class':'stats'})
@@ -181,6 +246,18 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
            if 'Words:' in label:
                self.story.setMetadata('numWords', value)
 				
+            if 'Comments:' in label:
+                self.story.setMetadata('comments', value)
+				
+            if 'Kudos:' in label:
+                self.story.setMetadata('kudos', value)
+				
+            if 'Hits:' in label:
+                self.story.setMetadata('hits', value)
+				
+            if 'Bookmarks:' in label:
+                self.story.setMetadata('bookmarks', value)
+				
            if 'Chapters:' in label:
                if value.split('/')[0] == value.split('/')[1]:
                    self.story.setMetadata('status', 'Completed')
@@ -0,0 +1,251 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return AshwinderSycophantHexComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class AshwinderSycophantHexComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+		
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','asph')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%m/%d/%Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'ashwinder.sycophanthex.com'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'This story contains adult content and/or themes.' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['rememberme'] = '1'
+        params['sid'] = ''
+        params['intent'] = ''
+        params['submit'] = 'Submit'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/user.php'
+        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "Logout" not in d : #Member Account
+            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+            
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
+		
+        try:
+            # in case link points somewhere other than the first chapter
+            a = soup.findAll('option')[1]['value']
+            self.story.setMetadata('storyId',a.split('=',)[1])
+            url = 'http://'+self.host+'/'+a
+            soup = bs.BeautifulSoup(self._fetchUrl(url))
+        except:
+            pass
+		
+        for info in asoup.findAll('table', {'width' : '100%', 'bordercolor' : re.compile(r'#')}):
+            a = info.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+            if a != None:
+                self.story.setMetadata('title',a.string)
+                break
+		
+
+        # Find the chapters:
+        chapters=soup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+&i=1$'))
+        if len(chapters) == 0:
+            self.chapterUrls.append((self.story.getMetadata('title'),url))
+        else:
+            for chapter in chapters:
+                # just in case there's tags, like <i> in chapter titles.
+                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d):
+            try:
+                return d.name
+            except:
+                return ""
+				
+        cats = info.findAll('a',href=re.compile('categories.php'))
+        for cat in cats:
+            self.story.addToList('category',cat.string)
+					
+        a = info.find('a', href=re.compile(r'reviews.php\?sid='+self.story.getMetadata('storyId')))
+        val = a.nextSibling
+        svalue = ""
+        while not defaultGetattr(val) == 'br':
+            val = val.nextSibling
+        val = val.nextSibling
+        while not defaultGetattr(val) == 'table':
+            svalue += str(val)
+            val = val.nextSibling
+        self.setDescription(url,svalue)
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = info.findAll('b')
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = stripHTML(labelspan)
+
+            if 'Rating' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word Count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Genres' in label:
+                genres = value.string.split(', ')
+                for genre in genres:
+                    if genre != 'none':
+                        self.story.addToList('genre',genre)
+
+            if 'Warnings' in label:
+                warnings = value.string.split(', ')
+                for warning in warnings:
+                    if warning != ' none':
+                        self.story.addToList('warnings',warning)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+		
+        data = self._fetchUrl(url)
+
+        soup = bs.BeautifulSoup(data, selfClosingTags=('br','hr','span','center')) # some chapters seem to be hanging up on those tags, so it is safer to close them
+        
+        story = soup.find('div', {"align" : "left"})
+
+        if None == story:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,story)
@@ -0,0 +1,333 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+# By virtue of being recent and requiring both is_adult and user/pass,
+# adapter_fanficcastletvnet.py is the best choice for learning to
+# write adapters--especially for sites that use the eFiction system.
+# Most sites that have ".../viewstory.php?sid=123" in the story URL
+# are eFiction.
+
+# For non-eFiction sites, it can be considerably more complex, but
+# this is still a good starting point.
+
+# In general an 'adapter' needs to do these five things:
+
+# - 'Register' correctly with the downloader
+# - Site Login (if needed)
+# - 'Are you adult?' check (if needed--some do one, some the other, some both)
+# - Grab the chapter list
+# - Grab the story meta-data (some (non-eFiction) adapters have to get it from the author page)
+# - Grab the chapter texts
+
+# Search for XXX comments--that's where things are most likely to need changing.
+
+# This function is called by the downloader in all adapter_*.py files
+# in this dir to register the adapter class.  So it needs to be
+# updated to reflect the class below it.  That, plus getSiteDomain()
+# take care of 'Registering'.
+def getClass():
+    return BloodTiesFansComAdapter # XXX
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class BloodTiesFansComAdapter(BaseSiteAdapter): # XXX
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
+        self._setURL('http://' + self.getSiteDomain() + '/fiction/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','btf') # XXX
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%b %d, %Y" # XXX
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'bloodties-fans.com' # XXX
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/fiction/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/fiction/viewstory.php?sid=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Registered Users Only' in data \
+                or 'There is no such account on our website' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['cookiecheck'] = '1'
+        params['submit'] = 'Submit'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/fiction/user.php?action=login'
+        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "Member Account" not in d : #Member Account
+            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+
+            # Furthermore, there's a couple sites now with more than
+            # one warning level for different ratings.  And they're
+            # fussy about it.  midnightwhispers has three: 4, 2 & 1.
+            # we'll try 1 first.
+            addurl = "&ageconsent=ok&warning=4" # XXX
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+            
+        # The actual text that is used to announce you need to be an
+        # adult varies from site to site.  Again, print data before
+        # the title search to troubleshoot.
+            
+        # Since the warning text can change by warning level, let's
+        # look for the warning pass url.  nfacommunity uses
+        # &amp;warning= -- actually, so do other sites.  Must be an
+        # eFiction book.
+            
+        # viewstory.php?sid=561&amp;warning=4
+        # viewstory.php?sid=561&amp;warning=1
+        # viewstory.php?sid=561&amp;warning=2
+        #print data
+        #m = re.search(r"'viewstory.php\?sid=1882(&amp;warning=4)'",data)
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&amp;ageconsent=ok)?&amp;warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid &amp; error in url.
+                addurl = addurl.replace("&amp;","&")
+                url = self.url+'&index=1'+addurl
+                logging.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/fiction/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                catstext = [cat.string for cat in cats]
+                for cat in catstext:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                charstext = [char.string for char in chars]
+                for char in charstext:
+                    self.story.addToList('characters',char.string)
+
+            ## Not all sites use Genre, but there's no harm to
+            ## leaving it in.  Check to make sure the type_id number
+            ## is correct, though--it's site specific.
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
+                genrestext = [genre.string for genre in genres]
+                self.genre = ', '.join(genrestext)
+                for genre in genrestext:
+                    self.story.addToList('genre',genre.string)
+
+            ## Not all sites use Warnings, but there's no harm to
+            ## leaving it in.  Check to make sure the type_id number
+            ## is correct, though--it's site specific.
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
+                warningstext = [warning.string for warning in warnings]
+                self.warning = ', '.join(warningstext)
+                for warning in warningstext:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/fiction/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -80,11 +80,6 @@ class CastleFansOrgAdapter(BaseSiteAdapter): # XXX
        # Each adapter needs to have a unique site abbreviation.
        self.story.setMetadata('siteabbrev','cslf') # XXX

-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Castle") # XXX
-
        # The date format will vary from site to site.
        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
        self.dateformat = "%b %d, %Y" # XXX
@@ -179,13 +174,14 @@ class CastleFansOrgAdapter(BaseSiteAdapter): # XXX
        # print data

        # Now go hunting for all the meta data and the chapter list.
-        
+
+        pagetitle = soup.find('div',{'id':'pagetitle'})
        ## Title
-        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        a = pagetitle.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
        self.story.setMetadata('title',a.string)
        
        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        a = pagetitle.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
        self.story.setMetadata('authorId',a['href'].split('=')[1])
        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
        self.story.setMetadata('author',a.string)
@@ -0,0 +1,234 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return ChaosSycophantHexComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class ChaosSycophantHexComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+		
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','csph')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%m/%d/%Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'chaos.sycophanthex.com'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=19"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        # The actual text that is used to announce you need to be an
+        # adult varies from site to site.  Again, print data before
+        # the title search to troubleshoot.
+        if "Age Consent Required" in data:
+            raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        pt = soup.find('div', {'id' : 'pagetitle'})
+        a = pt.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.text)
+        
+        # Find authorid and URL from... author url.
+        a = pt.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        rating=pt.text.split('(')[1].split(')')[0]
+        self.story.setMetadata('rating', rating)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+				
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+		
+        labels = soup.findAll('span',{'class':'label'})
+		
+        value = labels[0].previousSibling
+        svalue = ""
+        while value != None:
+            val = value
+            value = value.previousSibling
+        while not defaultGetattr(val,'class') == 'label':
+            svalue += str(val)
+            val = val.nextSibling
+        self.setDescription(url,svalue)
+		
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value.split(' -')[0])
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Complete' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value.split(' -')[0]), self.dateformat))
+            
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -0,0 +1,236 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+
+def getClass():
+    return CheckmatedComAdapter
+
+
+class CheckmatedComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+
+        self._setURL('http://' + self.getSiteDomain() + '/story.php?story='+self.story.getMetadata('storyId'))
+
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','chm')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%b %d, %Y"
+
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.checkmated.com'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/story.php?story=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/story.php?story=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites. This story is in The Bedchamber
+    def needToLoginCheck(self, data):
+        if 'This story is in The Bedchamber' in data \
+                or 'That username is not in our database' in data \
+                or "That password is not correct, please try again" in data:
+            return True
+        else:
+            return False
+
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['name'] = self.username
+            params['pass'] = self.password
+        else:
+            params['name'] = self.getConfig("username")
+            params['pass'] = self.getConfig("password")
+        params['login'] = 'yes'
+        params['submit'] = 'login'
+
+        loginUrl = 'http://' + self.getSiteDomain()+'/login.php'
+        d = self._fetchUrl(loginUrl,params)
+        e = self._fetchUrl(url)
+
+        if "Welcome back," not in d : #Member Account
+            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['name']))
+            raise exceptions.FailedToLogin(url,params['name'])
+            return False
+        elif "This story is in The Bedchamber" in e:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Your account does not have sufficient priviliges to read this story.")
+            return False
+        else:
+            return True
+
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+
+        # The actual text that is used to announce you need to be an
+        # adult varies from site to site.  Again, print data before
+        # the title search to troubleshoot.
+
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+
+        ## Title
+        a = soup.findAll('span', {'class' : 'storytitle'})
+        self.story.setMetadata('title',a[0].string)
+
+        # Find authorid and URL from... author url.
+        a = a[1].find('a', href=re.compile(r"authors.php\?name\=\w+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        a = soup.find('select', {'name' : 'chapter'})
+        if a == None:
+            self.chapterUrls.append((self.story.getMetadata('title'),url))
+        else:
+            for chapter in a.findAll('option'):
+                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/story.php?story='+self.story.getMetadata('storyId')+'&chapter='+chapter['value']))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+
+        # website does not keep track of word count, and there is no convenient way to calculate it
+
+        summary = soup.find('fieldset')
+        summary.find('legend').extract()
+        summary.name='div'
+        self.setDescription(url,summary)
+
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        table = soup.findAll('div', {'class' : 'text'})[1]
+        for labels in table.findAll('tr'):
+            value = labels.findAll('td')[1]
+            label = labels.findAll('td')[0]
+
+
+            if 'Rating' in stripHTML(label):
+                self.story.setMetadata('rating', stripHTML(value))
+
+            if 'Ship' in stripHTML(label):
+                if value.string != "none/none":
+                    self.story.addToList('ships',value.string)
+                for char in value.string.split('/'):
+                    if char != 'none':
+                        self.story.addToList('characters',char)
+
+            if 'Status' in stripHTML(label):
+                if value.find('img', {'src' : 'img/incomplete.gif'}) == None:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in stripHTML(label):
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+
+            if 'Updated' in stripHTML(label):
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        a = self._fetchUrl(self.story.getMetadata('authorUrl')+'&cat=stories')
+        for story in bs.BeautifulSoup(a).findAll('table', {'class' : 'storyinfo'}):
+            a = story.find('a', href=re.compile(r"review.php\?s\="+self.story.getMetadata('storyId')+'&act=view'))
+            if a != None:
+                for labels in story.findAll('tr'):
+                    value = labels.findAll('td')[1]
+                    label = labels.findAll('td')[0]
+                    if 'genre' in stripHTML(label):
+                        for genre in value.findAll('img'):
+                            self.story.addToList('genre',genre['title'])
+
+
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+
+        div = soup.find('div', {'id' : 'resizeableText'})
+        div.find('div', {'class' : 'storyTools'}).extract()
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+
+        return self.utf8FromSoup(url,div)
@@ -0,0 +1,274 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return DarkSolaceOrgAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class DarkSolaceOrgAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+		
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/elysian/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','dksl')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%d %B %Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'dark-solace.org'
+        
+    @classmethod
+    def getAcceptDomains(cls):
+        return ['www.dark-solace.org','dark-solace.org']
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/elysian/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://")+"(www\.)?"+re.escape(self.getSiteDomain()+"/elysian/viewstory.php?sid=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'This story contains adult content not suitable for children' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['rememberme'] = '1'
+        params['sid'] = ''
+        params['intent'] = ''
+        params['submit'] = 'Submit'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/elysian/user.php'
+        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "User Account Page" not in d : #Member Account
+            logging.info("Failed to login to URL %s as %s, or have no authorization to access the story" % (loginUrl, params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            addurl="&ageconsent=ok"
+            self.performLogin(url)
+            data = self._fetchUrl(url+addurl)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title and author
+        a = soup.find('div', {'id' : 'pagetitle'})
+        
+        aut = a.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',aut['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/elysian/'+aut['href'])
+        self.story.setMetadata('author',aut.string)
+        aut.extract()
+        
+        self.story.setMetadata('title',a.string[:(len(a.string)-3)])
+
+        # Find the chapters:
+        chapters=soup.find('select', {'name' : 'chapter'})
+        if chapters != None:
+            for chapter in chapters.findAll('option'):
+                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/elysian/viewstory.php?sid='+self.story.getMetadata('storyId')+'&chapter='+chapter['value']))
+        else:
+            self.chapterUrls.append((self.story.getMetadata('title'),url))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+        
+        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
+        
+        for list in asoup.findAll('div', {'class' : re.compile('listbox\s+')}):
+            a = list.find('a', href=re.compile(r'viewstory.php\?sid='))
+            if a != None:
+                if 'viewstory.php?sid='+self.story.getMetadata('storyId') in a['href']:
+                    break
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+                
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = list.findAll('span', {'class' : 'classification'})
+        for labelspan in labels:
+            label = labelspan.text
+            value = labelspan.nextSibling
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not (defaultGetattr(value,'class') == 'classification' or "Chapters: " in stripHTML(value)):
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value[:len(value)-2])
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'categories.php\?catid=\d+'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                for char in value.string.split(', '):
+                    if not 'None' in char:
+                        self.story.addToList('characters',char)
+
+            if 'Genre' in label:
+                for genre in value.string.split(', '):
+                    if not 'None' in genre:
+                        self.story.addToList('genre',genre)
+
+            if 'Warnings' in label:
+                for warning in value.string.split(', '):
+                    if not 'None' in warning:
+                        self.story.addToList('warnings',warning)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = list.find('a', href=re.compile(r"series.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/elysian/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            i=1
+            for a in storyas:
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if ('viewstory.php?sid='+self.story.getMetadata('storyId')) in a['href']:
+                        self.setSeries(series_name, i)
+                        break
+                    i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -0,0 +1,240 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return DestinysGatewayComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class DestinysGatewayComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','dgrfa')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%b %d %Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.destinysgateway.com'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&warning=4"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&amp;ageconsent=ok)?&amp;warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid &amp; error in url.
+                addurl = addurl.replace("&amp;","&")
+                url = self.url+'&index=1'+addurl
+                logging.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -0,0 +1,237 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return DokugaComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class DokugaComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[3])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+
+        # www.dokuga.com has two 'sections', shown in URL as
+        # 'fanfiction' and 'spark' that change how things should be
+        # handled.
+        # http://www.dokuga.com/fanfiction/story/7528/1
+        # http://www.dokuga.com/spark/story/7299/1
+        self.section=self.parsedUrl.path.split('/',)[1]
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/'+self.parsedUrl.path.split('/',)[1]+'/story/'+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','dkg')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        if 'fanfiction' in self.section:
+            self.dateformat = "%d %b %Y"
+        else:
+            self.dateformat = "%m-%d-%y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.dokuga.com'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/fanfiction/story/1234/1 http://"+self.getSiteDomain()+"/spark/story/1234/1"
+
+    def getSiteURLPattern(self):
+        return r"http://"+self.getSiteDomain()+"/(fanfiction|spark)?/story/\d+/?\d+?$"
+
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title and author
+        a = soup.find('div', {'align' : 'center'}).find('h3')
+        
+        # Find authorid and URL from... author url.
+        aut = a.find('a')
+        self.story.setMetadata('authorId',aut['href'].split('=')[1])
+        alink='http://'+self.host+aut['href']
+        self.story.setMetadata('authorUrl','http://'+self.host+aut['href'])
+        self.story.setMetadata('author',aut.string)
+        aut.extract()
+        
+        a = a.string[:(len(a.string)-4)]
+        self.story.setMetadata('title',a)
+
+        # Find the chapters:
+        chapters = soup.find('select').findAll('option')
+        if len(chapters)==1:
+            self.chapterUrls.append((self.story.getMetadata('title'),'http://'+self.host+'/'+self.section+'/story/'+self.story.getMetadata('storyId')+'/1'))
+        else:
+            for chapter in chapters:
+                # just in case there's tags, like <i> in chapter titles. /fanfiction/story/7406/1
+                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+self.section+'/story/'+self.story.getMetadata('storyId')+'/'+chapter['value']))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+        
+        asoup = bs.BeautifulSoup(self._fetchUrl(alink))
+        
+        if 'fanfiction' in self.section:
+            asoup=asoup.find('div', {'id' : 'cb_tabid_52'}).find('div')
+        
+            #grab the rest of the metadata from the author's page
+            for div in asoup.findAll('div'):
+                nav=div.find('a', href=re.compile(r'/fanfiction/story/'+self.story.getMetadata('storyId')+"/1$"))
+                if nav != None:
+                    break
+            div=div.nextSibling
+            self.setDescription(url,div)
+        
+            div=div.nextSibling
+            
+            a=div.text.split('Rating: ')
+            if len(a) == 2: self.story.setMetadata('rating', a[1].split('&')[0])
+        
+            a=div.text.split('Status: ')
+            if len(a)==2:
+                iscomp=a[1].split('&')[0]
+                if 'Complete' in iscomp:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+            
+            a=div.text.split('Category: ')
+            if len(a) == 2: self.story.addToList('category', a[1].split('&')[0])
+            self.story.addToList('category', 'Fanfiction')
+        
+            a=div.text.split('Created: ')
+            if len(a) == 2: self.story.setMetadata('datePublished', makeDate(stripHTML(a[1].split('&')[0]), self.dateformat))
+        
+            a=div.text.split('Updated: ')
+            if len(a) == 2: self.story.setMetadata('dateUpdated', makeDate(stripHTML(a[1]), self.dateformat))
+            
+            div=div.nextSibling.nextSibling
+            a=div.text.split('Words: ')
+            if len(a) == 2: self.story.setMetadata('numWords', a[1].split('&')[0])
+        
+            a=div.text.split('Genre: ')
+            if len(a) == 2:
+                for genre in a[1].split('&')[0].split(', '):
+                    self.story.addToList('genre',genre)
+        
+        else:
+            asoup=asoup.find('div', {'id' : 'maincol'}).find('div', {'class' : 'padding'})
+            for div in asoup.findAll('div'):
+                nav=div.find('a', href=re.compile(r'/spark/story/'+self.story.getMetadata('storyId')+"/1$"))
+                if nav != None:
+                    break
+                    
+            div=div.nextSibling.nextSibling
+            self.setDescription(url,div)
+            self.story.addToList('category', 'Spark')
+            
+            div=div.nextSibling.nextSibling
+            a=div.text.split('Rating: ')
+            if len(a) == 2: self.story.setMetadata('rating', a[1].split(' - ')[0])
+            
+            a=div.text.split('Status: ')
+            if len(a)==2:
+                iscomp=a[1].split(' - ')[0]
+                if 'Complete' in iscomp:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+            
+            a=div.text.split('Genre: ')
+            if len(a)==2:
+                for genre in a[1].split(' - ')[0].split('/'):
+                    self.story.addToList('genre',genre)
+                
+            div=div.nextSibling.nextSibling
+            
+            a=div.text.split('Updated: ')
+            if len(a)==2:
+                date=a[1].split('      -')[0]
+                self.story.setMetadata('dateUpdated', makeDate(date, self.dateformat))
+        
+                # does not have published date anywhere
+                self.story.setMetadata('datePublished', makeDate(date, self.dateformat))
+            
+            a=div.text.split('Words ')
+            if len(a)==2: self.story.setMetadata('numWords', a[1])
+            
+        
+
+
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'chtext'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -0,0 +1,298 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return DracoAndGinnyComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class DracoAndGinnyComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','dcagn')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%b %d, %Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.dracoandginny.com'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Registered Users Only' in data \
+                or 'There is no such account on our website' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['cookiecheck'] = '1'
+        params['submit'] = 'Submit'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
+        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "Member Account" not in d : #Member Account
+            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=2"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&amp;ageconsent=ok)?&amp;warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid &amp; error in url.
+                addurl = addurl.replace("&amp;","&")
+                url = self.url+'&index=1'+addurl
+                logging.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        content=soup.find('div',{'class' : 'listbox'})
+        
+        self.setDescription(url,content.find('blockquote'))
+        
+        for genre in content.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')):
+            self.story.addToList('genre',genre.string)
+
+        for warning in content.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')):
+            self.story.addToList('warnings',warning.string)
+        
+        labels = content.findAll('b')
+        
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value.split(' |')[0])
+                
+            if 'Rating' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value.nextSibling).split(' |')[0], self.dateformat))
+            
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            i=1
+            for a in storyas:
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        break
+                    i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'class' : 'listbox'})
+        
+        if None == div:
+            div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -0,0 +1,296 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return DramioneOrgAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class DramioneOrgAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','drmn')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%d %B %Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'dramione.org'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Registered Users Only' in data \
+                or 'There is no such account on our website' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['cookiecheck'] = '1'
+        params['submit'] = 'Submit'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
+        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "Member Account" not in d : #Member Account
+            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&warning=5"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+
+        # The actual text that is used to announce you need to be an
+        # adult varies from site to site.  Again, print data before
+        # the title search to troubleshoot.
+        if "Stories that are suitable for ages 16 and older" in data:
+            raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Use banner as cover if found
+        coverurl = ''
+        img = soup.find('img',{'class':'banner'})
+        if img:
+            coverurl = img['src']
+            #print "Cover: "+coverurl
+        a = soup.find(text="This story has a banner; click to view.")
+        if a:
+            #print "A: "+ ', '.join("(%s, %s)" %tup for tup in a.parent.attrs)
+            coverurl = a.parent['href']
+            #print "Cover: "+coverurl
+        if coverurl:
+            self.setCoverImage(url,coverurl)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+		
+        genres=soup.findAll('a', {'class' : "tag-1"})
+        for genre in genres:
+            self.story.addToList('genre',genre.string)
+
+        warnings=soup.findAll('a', {'class' : "tag-2"})
+        for warning in warnings:
+            self.story.addToList('warnings',warning.string)
+
+        themes=soup.findAll('a', {'class' : "tag-3"})
+        for theme in themes:
+            self.story.addToList('themes',theme.string)
+
+        hermiones=soup.findAll('a', {'class' : "tag-4"})
+        for hermione in hermiones:
+            self.story.addToList('hermiones',hermione.string)
+
+        dracos=soup.findAll('a', {'class' : "tag-5"})
+        for draco in dracos:
+            self.story.addToList('dracos',draco.string)
+
+        timelines=soup.findAll('a', {'class' : "tag-6"})
+        for timeline in timelines:
+            self.story.addToList('timeline',timeline.string)
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                value=re.sub(r"(\d+)(st|nd|rd|th)",r"\1",value)
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                value=re.sub(r"(\d+)(st|nd|rd|th)",r"\1",value)
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -0,0 +1,252 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return ErosnSapphoSycophantHexComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class ErosnSapphoSycophantHexComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+		
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','essph')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%d/%m/%y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'erosnsappho.sycophanthex.com'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=18"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&amp;ageconsent=ok)?&amp;warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid &amp; error in url.
+                addurl = addurl.replace("&amp;","&")
+                url = self.url+'&index=1'+addurl
+                logging.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        pt = soup.find('div', {'id' : 'pagetitle'})
+        a = pt.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.text)
+        
+        # Find authorid and URL from... author url.
+        a = pt.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        rating=pt.text.split('(')[1].split(')')[0]
+        self.story.setMetadata('rating', rating)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+				
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+		
+        labels = soup.findAll('span',{'class':'label'})
+		
+        value = labels[0].previousSibling
+        svalue = ""
+        while value != None:
+            val = value
+            value = value.previousSibling
+        while not defaultGetattr(val,'class') == 'label':
+            svalue += str(val)
+            val = val.nextSibling
+        self.setDescription(url,svalue)
+		
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value.split(' -')[0])
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Complete' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value.split(' -')[0]), self.dateformat))
+            
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            i=1
+            for a in storyas:
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        break
+                    i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -19,13 +19,19 @@ import time
 import logging
 import re
 import urllib2
+from urllib import unquote_plus
 import time

 from .. import BeautifulSoup as bs
 from .. import exceptions as exceptions
+from ..htmlcleanup import stripHTML

 from base_adapter import BaseSiteAdapter,  makeDate

+ffnetgenres=["Adventure", "Angst", "Crime", "Drama", "Family", "Fantasy", "Friendship", "General",
+             "Horror", "Humor", "Hurt-Comfort", "Mystery", "Parody", "Poetry", "Romance", "Sci-Fi",
+             "Spiritual", "Supernatural", "Suspense", "Tragedy", "Western"]
+
 class FanFictionNetSiteAdapter(BaseSiteAdapter):

    def __init__(self, config, url):
@@ -82,7 +88,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
            else:
                raise e
            
-        if "Unable to locate story with id of " in data:
+        if "Unable to locate story" in data:
            raise exceptions.StoryDoesNotExist(url)

        # some times "Chapter not found...", sometimes "Chapter text not found..."
@@ -117,64 +123,100 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
        self.story.setMetadata('authorUrl','http://'+self.host+a['href'])
        self.story.setMetadata('author',a.string)

-            
-        # start by finding a script towards the bottom that has a
-        # bunch of useful stuff in it.
-            
-        # var storyid = 6577076;
-        # var chapter = 1;
-        # var chapters = 17;
-        # var words = 42787;
-        # var userid = 2645830;
-        # var title = 'The+Invitation';
-        # var title_t = 'The Invitation';
-        # var summary = 'Dudley Dursley would be the first to say he lived a very normal life. But what happens when he gets invited to his cousin Harry Potter\'s wedding? Will Dudley get the courage to apologize for the torture he caused all those years ago? Harry/Ginny story.';
-        # var categoryid = 224;
-        # var cat_title = 'Harry Potter';
-        # var datep = '12-21-10';
-        # var dateu = '04-06-11';
-        # var author = 'U n F a b u l o u s M e';
+        ## Pull some additional data from html.

-        for script in soup.findAll('script', src=None):
-            if not script:
-                continue
-            if not script.string:
-                continue
-            if 'var storyid' in script.string:
-                for line in script.string.split('\n'):
-                    m = re.match(r"^ +var ([^ ]+) = '?(.*?)'?;\r?$",line)
-                    if m == None : continue
-                    var,value = m.groups()
-                    # remove javascript escaping from values.
-                    value = re.sub(r'\\(.)',r'\1',value)
-                    #print var,value
-                    if 'words' in var:
-                        self.story.setMetadata('numWords', value)
-                    if 'title_t' in var:
-                        self.story.setMetadata('title', value)
-                    if 'summary' in var:
-                        self.setDescription(url,value)
-                        #self.story.setMetadata('description', value)
-                    if 'datep' in var:
-                        self.story.setMetadata('datePublished',makeDate(value, '%m-%d-%y'))
-                    if 'dateu' in var:
-                        self.story.setMetadata('dateUpdated',makeDate(value, '%m-%d-%y'))
-                    if 'cat_title' in var:
-                        if "Crossover" in value:
-                            value = re.sub(r' Crossover$','',value)
-                            for c in value.split(' and '):
-                                self.story.addToList('category',c)
-                                # Screws up when the category itself
-                                # contains ' and '.  But that's rare
-                                # and the only alternative is to find
-                                # the 'Crossover' category URL and
-                                # parse that page to search for <a>
-                                # with href /crossovers/(name)/(num)/
-				# <a href="/crossovers/Harry_Potter/224/">Harry Potter</a>
-				# <a href="/crossovers/Naruto/1402/">Naruto</a>
-                        else:
-                            self.story.addToList('category',value)
-                break # for script in soup.findAll('script', src=None):
+        ## ffnet shows category two ways
+        ## 1) class(Book, TV, Game,etc) >> category(Harry Potter, Sailor Moon, etc)
+        ## 2) cat1_cat2_Crossover
+        ## For 1, use the second link.
+        ## For 2, fetch the crossover page and pull the two categories from there.
+
+        categories = soup.findAll('a',{'class':'xcontrast_txt'})
+        if len(categories) > 1:
+            self.story.addToList('category',stripHTML(categories[-1]))
+        elif 'Crossover' in categories[0]['href']:
+            caturl = "http://%s%s"%(self.getSiteDomain(),categories[0]['href'])
+            catsoup = bs.BeautifulSoup(self._fetchUrl(caturl))
+            for a in catsoup.findAll('a',href=re.compile(r"^/crossovers/")):
+                self.story.addToList('category',stripHTML(a))
+            
+        a = soup.find('a', href='http://www.fictionratings.com/')
+        rating = a.string
+        if 'Fiction' in rating: # if rating has 'Fiction ', strip that out for consistency with past.
+            rating = rating[8:]
+            
+        self.story.setMetadata('rating',rating)
+
+        # after Rating, the same bit of text containing id:123456 contains
+        # Complete--if completed.
+        gui_table1i = soup.find(id="gui_table1i")
+
+        self.story.setMetadata('title', stripHTML(gui_table1i.find('b'))) # title appears to be only(or at least first) bold tag in gui_table1i
+
+        summarydiv = gui_table1i.find('div',{'style':'margin-top:2px'})
+        if summarydiv:
+            self.setDescription(url,stripHTML(summarydiv))
+            
+        
+        metatext = stripHTML(gui_table1i.find('div', {'style':'color:gray;'})).replace('Hurt/Comfort','Hurt-Comfort')
+        metalist = metatext.split(" - ")
+        #print("metatext:(%s)"%metalist)
+
+        # Rated: Fiction K - English - Words: 158,078 - Published: 02-04-11
+
+        # rating is obtained above more robustly.
+        if metalist[0].startswith('Rated:'):
+            metalist=metalist[1:]
+
+        # next is assumed to be language.
+        self.story.setMetadata('language',metalist[0])
+        metalist=metalist[1:]
+
+        # next might be genre.
+        genrelist = metalist[0].split('/') # Hurt/Comfort already changed above.
+        goodgenres=True
+        for g in genrelist:
+            print("g:(%s)"%g)
+            if g.strip() not in ffnetgenres:
+                print("g not in ffnetgenres")
+                goodgenres=False
+        if goodgenres:
+            self.story.extendList('genre',genrelist)
+            metalist=metalist[1:]
+
+        donechars = False
+        while len(metalist) > 0:
+            if  metalist[0].startswith('Chapters') or metalist[0].startswith('Status') or metalist[0].startswith('id:'):
+                pass
+            elif  metalist[0].startswith('Reviews'):
+                self.story.setMetadata('reviews',metalist[0].split(':')[1].strip())
+            elif  metalist[0].startswith('Favs:'):
+                self.story.setMetadata('favs',metalist[0].split(':')[1].strip())
+            elif  metalist[0].startswith('Follows:'):
+                self.story.setMetadata('follows',metalist[0].split(':')[1].strip())
+            elif  metalist[0].startswith('Updated'):
+                self.story.setMetadata('dateUpdated',makeDate(metalist[0].split(':')[1].strip(), '%m-%d-%y'))
+            elif  metalist[0].startswith('Published'):
+                self.story.setMetadata('datePublished',makeDate(metalist[0].split(':')[1].strip(), '%m-%d-%y'))
+            elif  metalist[0].startswith('Words'):
+                self.story.setMetadata('numWords',metalist[0].split(':')[1].strip())
+            elif not donechars:
+                self.story.extendList('characters',metalist[0].split('&'))
+                donechars = True
+            metalist=metalist[1:]
+
+        # next might be characters, otherwise Reviews, Updated, Published, Words
+        # if not ( metalist[0].startswith('Reviews') or metalist[0].startswith('Updated') or metalist[0].startswith('Published') or metalist[0].startswith('Words') or metalist[0].startswith('Chapters') ):
+        #     self.story.extendList('characters',metalist[0].split('&'))
+        
+        if 'Status: Complete' in metatext:
+            self.story.setMetadata('status', 'Completed')
+        else:
+            self.story.setMetadata('status', 'In-Progress')
+
+        img = soup.find('img',{'class':'cimage'})
+        if img:
+            self.setCoverImage(url,img['src'])
            
        # Find the chapter selector 
        select = soup.find('select', { 'name' : 'chapter' } )
@@ -195,58 +237,6 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):

        self.story.setMetadata('numChapters',len(self.chapterUrls))

-        ## Pull some additional data from html.  Find Rating and look around it.
-
-        a = soup.find('a', href='http://www.fictionratings.com/')
-        self.story.setMetadata('rating',a.string)
-
-        # used below to get correct characters.
-        metatext = a.findNext(text=re.compile(r' - Reviews:'))
-        if metatext == None: # indicates there's no Reviews, look for id: instead.
-            metatext = a.findNext(text=re.compile(r' - id:'))
-
-        # after Rating, the same bit of text containing id:123456 contains
-        # Complete--if completed.
-        if 'Complete' in a.findNext(text=re.compile(r'id:'+self.story.getMetadata('storyId'))):
-            self.story.setMetadata('status', 'Completed')
-        else:
-            self.story.setMetadata('status', 'In-Progress')
-
-        # Parse genre(s) from <meta name="description" content="..."
-        # <meta name="description" content="A Transformers/Beast Wars  - Humor fanfiction with characters Prowl & Sideswipe. Story summary: Sideswipe is bored. Prowl appears to be so, too  or at least, Sideswipe thinks he looks bored . So Sideswipe entertains them. After all, what's more fun than a race? Song-fic.">
-        # <meta name="description" content="Chapter 1 of a Transformers/Beast Wars  - Adventure/Friendship fanfiction with characters Bumblebee. TFA: What would you do if you was being abused all you life? Follow NightRunner as she goes through her spark breaking adventure of getting away from her father..">
-        # (fp)<meta name="description" content="Chapter 1 of a Sci-Fi  - Adventure/Humor fiction. Felix Max was just your regular hyperactive kid until he accidently caused his own fathers death. Now he has meta-humans trying to hunt him down with a corrupt goverment to back them up. Oh, and did I mention he has no Powers yet?.">
-        # <meta name="description" content="Chapter 1 of a Bleach  - Adventure/Angst fanfiction with characters Ichigo K. & Neliel T. O./Nel. Time travel with a twist. Time can be a real bi***. Ichigo finds that fact out when he accidentally goes back in time. Is this his second chance or is fate just screwing with him. Not a crack fic.IchixNelXHime.">
-        # <meta name="description" content="Chapter 1 of a Harry Potter and Transformers  - Humor/Adventure crossover fanfiction  with characters: Harry P. & Ironhide. IT’s one thing to be tossed thru the Veil for something he didn’t do. It was quite another to wake in his animigus form in a world not his own. Harry just knew someone was laughing at him somewhere. Mech/Mech pairings inside..">
-        m = re.match(r"^(?:Chapter \d+ of a|A) (?:.*?)  (?:- (?P<genres>.*?) )?(?:crossover )?(?:fan)?fiction(?P<chars>[ ]+with characters)?",
-                     soup.find('meta',{'name':'description'})['content'])
-        if m != None:
-            genres=m.group('genres')
-            if genres != None:
-                # Hurt/Comfort is one genre.
-                genres=re.sub('Hurt/Comfort','Hurt-Comfort',genres)
-                for g in genres.split('/'):
-                    self.story.addToList('genre',g)
-
-            if m.group('chars') != None:
-
-                # At this point we've proven that there's character(s)
-                # We can't reliably parse characters out of meta name="description".
-                # There's no way to tell that "with characters Ichigo K. & Neliel T. O./Nel. " ends at "Nel.", not "T."
-                # But we can pull them from the reviewstext line, now that we know about existance of chars.
-                # reviewstext can take form of:
-                # - English -  Shinji H. - Updated: 01-13-12 - Published: 12-20-11 - id:7654123
-                # - English - Adventure/Angst -  Ichigo K. & Neliel T. O./Nel - Reviews:
-                # - English - Humor/Adventure -  Harry P. & Ironhide - Reviews:
-                mc = re.match(r" - (?P<lang>[^ ]+ - )(?P<genres>[^ ]+ - )? (?P<chars>.+?) - (Reviews|Updated|Published)",
-                              metatext)
-                chars = mc.group("chars")
-                for c in chars.split(' & '):
-                    self.story.addToList('characters',c)
-        m = re.match(r" - (?P<lang>[^ ]+)",metatext)
-        if m.group('lang') != None:
-            self.story.setMetadata('language',m.group('lang'))
-                    
        return

    def getChapterText(self, url):
@@ -256,19 +246,26 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
                        ## additional to what ever the
                        ## slow_down_sleep_time setting is.
        data = self._fetchUrl(url)
+
+        # some ancient stories have body tags inside them that cause
+        # soup parsing to discard the content.  For story text we
+        # don't care about anything before "<div class='storytextp"
+        # (there's a space after storytextp, so no close quote(')) and
+        # this kills any body tags.
+        data = data[data.index("<div class='storytextp"):]
+        data.replace("<body","<notbody").replace("<BODY","<NOTBODY")
+        
        soup = bs.BeautifulSoup(data)

        ## Remove the 'share' button.
        sharediv = soup.find('div', {'class' : 'a2a_kit a2a_default_style'})
        if sharediv:
            sharediv.extract()
-        else:
-            logging.debug('share button div not found')
        
-        div = soup.find('div', {'id' : 'storytext'})
+        div = soup.find('div', {'id' : 'storytextp'})
        
        if None == div:
-            logging.debug('div id=storytext not found.  data:%s'%data)
+            logging.debug('div id=storytextp not found.  data:%s'%data)
            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)

        return self.utf8FromSoup(url,div)
@@ -0,0 +1,199 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return FanFiktionDeAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class FanFiktionDeAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["utf8",
+                       "Windows-1252"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/s/'+self.story.getMetadata('storyId') + '/1')
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','ffde')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%d.%m.%Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.fanfiktion.de'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/s/46ccbef30000616306614050"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/s/")+r"\w+(/\d+)?$"
+        
+        ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Diese Geschichte wurde als entwicklungsbeeintr' in data \
+                or 'There is no such account on our website' in data \
+                or "Noch kein registrierter Benutzer?" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self,url):
+        params = {}
+
+        if self.password:
+            params['nickname'] = self.username
+            params['passwd'] = self.password
+        else:
+            params['nickname'] = self.getConfig("username")
+            params['passwd'] = self.getConfig("password")
+        params['savelogindata'] = '1'
+        params['a'] = 'l'
+        params['submit'] = 'Login...'
+
+        loginUrl = 'https://ssl.fanfiktion.de/'
+        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['nickname']))
+        d = self._postUrl(loginUrl,params)
+    
+        if "Login erfolgreich" not in d : #Member Account
+            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['nickname']))
+            raise exceptions.FailedToLogin(url,params['nickname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        url = self.url
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+                
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+            
+        if "Uhr ist diese Geschichte nur nach einer" in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Auserhalb der Zeit von 23:00 Uhr bis 04:00 Uhr ist diese Geschichte nur nach einer erfolgreichen Altersverifikation zuganglich.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'/s/'+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        head = soup.find('div', {'style' : 'width:85%;float:left;'})
+        a = head.find('a')
+        self.story.setMetadata('authorId',a['href'].split('/')[2])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.find('select').findAll('option'):
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/s/'+self.story.getMetadata('storyId')+'/'+chapter['value']))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+        self.story.setMetadata('language','German')
+
+        #find metadata on the story page
+        self.story.setMetadata('datePublished', makeDate(head.text.split('erstellt: ')[1].split('\n')[0], self.dateformat))
+        
+        self.story.setMetadata('dateUpdated', makeDate(head.text.split('letztes Update: ')[1].split('\n')[0], self.dateformat))
+        
+        for genre in head.text.split('&nbsp;&nbsp;&nbsp;')[3].split('/')[0].split(', '):
+            self.story.addToList('genre',genre)
+            
+        if 'fertiggestellt' in head.text:
+            self.story.setMetadata('status', 'Completed')
+        else:
+            self.story.setMetadata('status', 'In Progress')
+
+        #find metadata on the author's page
+        asoup = bs.BeautifulSoup(self._fetchUrl("http://"+self.getSiteDomain()+"?a=q&a1=v&t=nickdetailsstories&lbi=stories&ar=0&nick="+self.story.getMetadata('authorId')))
+        tr=asoup.findAll('tr')
+        for i in range(1,len(tr)):
+            a = tr[i].find('a')
+            if a['href'] == '/s/'+self.story.getMetadata('storyId'):
+                break
+        self.setDescription(url,a['onmouseover'].split("', '")[1])
+        
+        td = tr[i].findAll('td')
+        self.story.addToList('category',td[1].string)
+        self.story.setMetadata('rating', td[4].string)
+        self.story.setMetadata('numWords', td[5].string)
+ 
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+        time.sleep(0.5) ## ffde has "floodlock" protection
+
+        soup = bs.BeautifulSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'storytext'})
+        for a in div.findAll('script'):
+            a.extract()
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -94,7 +94,7 @@ class FicBookNetAdapter(BaseSiteAdapter):
        
        ## Title
        a = soup.find('h1')
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        logging.debug("Title: (%s)"%self.story.getMetadata('title'))
        
        # Find authorid and URL from... author url.
@@ -201,7 +201,7 @@ class FicBookNetAdapter(BaseSiteAdapter):
                break
 				
        summary=soup.find('span', {'class' : 'urlize'})
-        self.setDescription(url,summary.text)
+        self.setDescription(url,summary)
        #self.story.setMetadata('description', summary.text)
            
    # grab the text for an individual chapter.
@@ -37,7 +37,6 @@ class FictionAlleyOrgSiteAdapter(BaseSiteAdapter):
                               # Most sites that claim to be
                               # iso-8859-1 (and some that claim to be
                               # utf8) are really windows-1252.
-        self.story.addToList("category","Harry Potter")
        self.is_adult=False
        
        # get storyId from url--url validation guarantees query correct
@@ -124,7 +124,7 @@ class FicwadComSiteAdapter(BaseSiteAdapter):

        # description
        storydiv = soup.find("div",{"id":"story"})
-        self.setDescription(url,storydiv.find("blockquote",{'class':'summary'}).p.string)
+        self.setDescription(url,storydiv.find("blockquote",{'class':'summary'}).p)
        #self.story.setMetadata('description', storydiv.find("blockquote",{'class':'summary'}).p.string)

        # most of the meta data is here:
@@ -20,12 +20,15 @@ import logging
 import re
 import urllib2
 import cookielib as cl
-import datetime
+from datetime import datetime
+import json

 from .. import BeautifulSoup as bs
 from ..htmlcleanup import stripHTML
 from .. import exceptions as exceptions

+from ..bbcodeutils.bbcodeparser import bbcodeparser
+
 from base_adapter import BaseSiteAdapter,  makeDate

 def getClass():
@@ -36,7 +39,7 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
    def __init__(self, config, url):
        BaseSiteAdapter.__init__(self, config, url)
        self.story.setMetadata('siteabbrev','fimficnet')
-        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
+        self.story.setMetadata('storyId', self.parsedUrl.path.split('/',)[2])
        self._setURL("http://"+self.getSiteDomain()+"/story/"+self.story.getMetadata('storyId')+"/")
        self.is_adult = False
        
@@ -74,6 +77,10 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
            self.opener = urllib2.build_opener(cookieproc)
        
        try:
+            apiResponse = urllib2.urlopen("http://www.fimfiction.net/api/story.php?story=%s" % (self.story.getMetadata("storyId"))).read()
+            apiData = json.loads(apiResponse)
+            
+            # Unfortunately, we still need to load the story index page to parse the characters
            data = self._fetchUrl(self.url)
        except urllib2.HTTPError, e:
            if e.code == 404:
@@ -87,9 +94,12 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
        if "/images/missing_story.png" in data:
            raise exceptions.StoryDoesNotExist(self.url)
        
+        if "Invalid story id" in apiData.values():
+            raise exceptions.StoryDoesNotExist(self.url)
+        
        if "This story has been marked as having adult content." in data:
            raise exceptions.AdultCheckRequired(self.url)
-
+        
        if self.password:
            params = {}
            params['password'] = self.password
@@ -100,98 +110,69 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
                raise exceptions.FailedToDownload("%s requires story password and fail_on_password is true."%self.url)
            else:
                raise exceptions.FailedToLogin(self.url,"Story requires individual password",passwdonly=True)
+         
+        storyMetadata = apiData["story"]    
+            
+        self.story.setMetadata("title", storyMetadata["title"])
+        self.story.setMetadata("author", storyMetadata["author"]["name"])
+        self.story.setMetadata("authorId", storyMetadata["author"]["id"])
+        self.story.setMetadata("authorUrl", "http://%s/user/%s" % (self.getSiteDomain(), storyMetadata["author"]["name"]))
        
-        soup = bs.BeautifulSoup(data).find("div", {"class":"content_box post_content_box"})
-
-        titleheader = soup.find("h2")
-        title = titleheader.find("a", href=re.compile(r'^/story/')).text
-        author = titleheader.find("a", href=re.compile(r'^/user/')).text
-        self.story.setMetadata("title", title)
-        self.story.setMetadata("author", author)
-        self.story.setMetadata("authorId", author) # The author's name will be unique
-        self.story.setMetadata("authorUrl", "http://%s/user/%s" % (self.getSiteDomain(),author))
+        chapters = [{"chapterTitle": chapter["title"], "chapterURL": chapter["link"]} for chapter in storyMetadata["chapters"]]
+        for chapter in chapters:
+            self.chapterUrls.append((chapter["chapterTitle"], chapter["chapterURL"]))
+        self.story.setMetadata("numChapters", len(self.chapterUrls))
        
-        chapterDates = []
-
-        for chapter in soup.findAll("a", {"class":"chapter_link"}):
-            chapterDates.append(chapter.span.extract().text.strip("()"))
-            self.chapterUrls.append((chapter.text.strip(), "http://"+self.getSiteDomain() + chapter['href']))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-        
-        for character in [character_icon['title'] for character_icon in soup.findAll("a", {"class":"character_icon"})]:
-            self.story.addToList("characters", character)
-        for category in [category.text for category in soup.find("div", {"class":"categories"}).findAll("a")]:
-            self.story.addToList("genre", category)
-        self.story.addToList("category", "My Little Pony")
-        
-        
-        # The very last list element in the list of chapters contains the status, rating and word count e.g.:
-        #
-        #    <li>
-        #       Incomplete | Rating:
-        #       <span style="color:#c78238;">Teen</span>
-        #       <div class="word_count"><b>5,203</b>words total</div>
-        #    </li>
-        #
-
-        status_bar = soup.findAll('li')[-1]
        # In the case of fimfiction.net, possible statuses are 'Completed', 'Incomplete', 'On Hiatus' and 'Cancelled'
        # For the sake of bringing it in line with the other adapters, 'Incomplete' and 'On Hiatus' become 'In-Progress'
        # and 'Complete' beomes 'Completed'. 'Cancelled' seems an important enough (not to mention more strictly true) 
        # status to leave unchanged.
-        status = status_bar.text.split("|")[0].strip().replace("Incomplete", "In-Progress").replace("On Hiatus", "In-Progress").replace("Complete", "Completed")
-        self.story.setMetadata('status', status)
-        self.story.setMetadata('rating', status_bar.span.text)
-        # This way is less elegant, perhaps, but more robust in face of format changes.
-        numWords = status_bar.find("div",{"class":"word_count"}).b.text
-        self.story.setMetadata('numWords', numWords)
-        
-        description_soup = soup.find("div", {"class":"description"})
-        # Sometimes the description has an expanding element
-        # This removes the ellipsis and the expand button
-        try:
-            description_soup.find('span', {"id":re.compile(r"description_more_elipses_\d+")}).extract() # Web designer can't spell 'ellipsis'
-            description_soup.find('a', {"class":"more"}).extract()
-        except:
-            pass
+        status = storyMetadata["status"].replace("Incomplete", "In-Progress").replace("On Hiatus", "In-Progress").replace("Complete", "Completed")
+        self.story.setMetadata("status", status)
+        self.story.setMetadata("rating", storyMetadata["content_rating_text"])
+            
+        for category in storyMetadata["categories"]:
+            if storyMetadata["categories"][category]:
+                self.story.addToList("genre", category) 
+
+        self.story.setMetadata("numWords", str(storyMetadata["words"]))
        
        # fimfic is the first site with an explicit cover image.
-        story_img = soup.find('img',{'class':'story_image'})
-        if self.getConfig('include_images') and story_img:
-            coverurl = story_img['src']
+        if "image" in storyMetadata.keys():
+            if "full_image" in storyMetadata:
+                coverurl = storyMetadata["full_image"]
+            else:
+                coverurl = storyMetadata["image"]
            if coverurl.startswith('//static.fimfiction.net'): # fix for img urls missing 'http:'
                coverurl = "http:"+coverurl
-            self.story.addImgUrl(self,self.url,coverurl,self._fetchUrlRaw,cover=True)
-        self.setDescription(self.url,description_soup.text)
-        #self.story.setMetadata('description', description_soup.text)
-        
-        # Unfortunately, nowhere on the page is the year mentioned.
-        # Best effort to deal with this:
-        # Use this year, if that's a date in the future, subtract one year.
-        # Their earliest story is Jun, so they'll probably change the date
-        # around then.
+            self.setCoverImage(self.url,coverurl)

-        now = datetime.datetime.now()
+        # the fimfic API gives bbcode for desc, not html.
+        # btw, bbcode honors newlines, html doesn't.  change newlines to br tags.
+        self.setDescription(self.url,
+                            bbcodeparser().parse(storyMetadata["description"]).html(doDeepCopy=False).replace('\r','').replace('\n','<br />'))
        
-        dateUpdated_soup = bs.BeautifulSoup(data).find("div", {"class":"calendar"})
-        dateUpdated_soup.find('span').extract()
-        dateUpdated = makeDate("%s%s"%(now.year,dateUpdated_soup.text), "%Y%b%d")
-        if dateUpdated > now :
-            dateUpdated = dateUpdated.replace(year=now.year-1)
-        self.story.setMetadata("dateUpdated", dateUpdated)
+        # Dates are in Unix time
+        # Take the publish date from the first chapter posted
+        rawDatePublished = storyMetadata["chapters"][0]["date_modified"]
+        self.story.setMetadata("datePublished", datetime.fromtimestamp(rawDatePublished))
+        rawDateUpdated = storyMetadata["date_modified"]
+        self.story.setMetadata("dateUpdated", datetime.fromtimestamp(rawDateUpdated))
        
-        # Get the date of creation from the first chapter
-        if len(chapterDates) > 0:
-            datePublished_text = chapterDates[0]
-            day, month = datePublished_text.split()
-            day = re.sub(r"[^\d.]+", '', day)
-            datePublished = makeDate("%s%s%s"%(now.year,month,day), "%Y%b%d")
-            if datePublished > now :
-                datePublished = datePublished.replace(year=now.year-1)
-            self.story.setMetadata("datePublished", datePublished)
-        else:
-            self.story.setMetadata("datePublished", dateUpdated)
+        soup = bs.BeautifulSoup(data).find("div", {"class":"story"})
+        # fimfic stopped putting the char name on or around the char
+        # icon now for some reason.  Pull it from the image name with
+        # some heuristics.
+        for character in [character_icon["src"] for character_icon in soup.findAll("img", {"class":"character_icon"})]:
+            # //static.fimfiction.net/images/characters/twilight_sparkle.png
+            # 5th split /, remove last four, replace _, capitolize every word(title())
+            char = character.split('/')[5][:-4].replace('_',' ').title()
+            if char == 'Oc':
+                char = "OC"
+            if char == 'Cmc':
+                char = "Cutie Mark Crusaders"
+            self.story.addToList("characters", char)
+            
            
    def getChapterText(self, url):
        logging.debug('Getting chapter text from: %s' % url)
@@ -0,0 +1,293 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return FineStoriesComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class FineStoriesComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url
+        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2].split(':')[0])
+        if 'storyInfo' in self.story.getMetadata('storyId'):
+            self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/s/storyInfo.php?id='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','fnst')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%Y-%m-%d"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'finestories.com'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/s/10537 http://"+self.getSiteDomain()+"/s/10537:4010 http://"+self.getSiteDomain()+"/library/storyInfo.php?id=10537"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain())+r"/(s|library)?/(storyInfo.php\?id=)?\d+(:\d+)?(;\d+)?$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Free Registration' in data \
+                or "Invalid Password!" in data \
+                or "Invalid User Name!" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['theusername'] = self.username
+            params['thepassword'] = self.password
+        else:
+            params['theusername'] = self.getConfig("username")
+            params['thepassword'] = self.getConfig("password")
+        params['rememberMe'] = '1'
+        params['page'] = 'http://finestories.com/'
+        params['submit'] = 'Login'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/login.php'
+        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['theusername']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "My Account" not in d : #Member Account
+            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['theusername']))
+            raise exceptions.FailedToLogin(url,params['theusername'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'/s/'+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.text)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"/a/\w+"))
+        self.story.setMetadata('authorId',a['href'].split('/')[2])
+        self.story.setMetadata('authorUrl','http://'+self.host+a['href'])
+        self.story.setMetadata('author',a.text)
+
+        # Find the chapters:
+        chapters = soup.findAll('a', href=re.compile(r'/s/'+self.story.getMetadata('storyId')+":\d+$"))
+        if len(chapters) != 0:
+            for chapter in chapters:
+                # just in case there's tags, like <i> in chapter titles.
+                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+chapter['href']))
+        else:
+            self.chapterUrls.append((self.story.getMetadata('title'),'http://'+self.host+'/s/'+self.story.getMetadata('storyId')))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # surprisingly, the detailed page does not give enough details, so go to author's page
+        
+        skip=0
+        i=0
+        while i == 0:
+            asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')+"&skip="+str(skip)))
+        
+            a = asoup.findAll('td', {'class' : 'lc2'})
+            for lc2 in a:
+                if lc2.find('a')['href'] == '/s/'+self.story.getMetadata('storyId'):
+                    i=1
+                    break
+                if a[len(a)-1] == lc2:
+                    skip=skip+10
+        
+        for cat in lc2.findAll('div', {'class' : 'typediv'}):
+            self.story.addToList('category',cat.text)
+        
+        self.story.setMetadata('numWords', lc2.findNext('td', {'class' : 'num'}).text)
+        
+        lc4 = lc2.findNext('td', {'class' : 'lc4'})
+        
+        
+        try:
+            a = lc4.find('a', href=re.compile(r"/library/show_series.php\?id=\d+"))
+            i = a.parent.text.split('(')[1].split(')')[0]
+            self.setSeries(a.text, i)
+        except:
+            pass
+        try:
+            a = lc4.find('a', href=re.compile(r"/library/universe.php\?id=\d+"))
+            self.story.addToList("category",a.text)
+        except:
+            pass
+            
+        for a in lc4.findAll('span', {'class' : 'help'}):
+            a.extract()
+            
+        self.setDescription('http://'+self.host+'/s/'+self.story.getMetadata('storyId'),lc4.text.split('[More Info')[0])
+            
+        for b in lc4.findAll('b'):
+            label = b.text
+            value = b.nextSibling
+            
+            if 'For Age' in label:
+                self.story.setMetadata('rating', value)
+                
+            if 'Tags' in label:
+                for genre in value.split(', '):
+                    self.story.addToList('genre',genre)
+                    
+            if 'Posted' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value.split('/ (')[0]), self.dateformat))
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value.split('/ (')[0]), self.dateformat))
+                
+            if 'Concluded' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value.split('/ (')[0]), self.dateformat))
+                
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value.split('/ (')[0]), self.dateformat))
+                
+        status = lc4.find('span', {'class' : 'ab'})
+        if  status != None:
+            self.story.setMetadata('status', 'In-Progress')
+            if "Last Activity" in status.text:
+                self.story.setMetadata('dateUpdated', makeDate(status.text.split('Activity: ')[1].split(')')[0], self.dateformat))
+        else:
+            self.story.setMetadata('status', 'Completed')
+
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+        
+        # some big chapters are split over several pages
+        pager = div.find('span', {'class' : 'pager'})
+        if pager != None:
+            urls=pager.findAll('a')
+            urls=urls[:len(urls)-1]
+            
+            
+            for ur in urls:
+                soup = bs.BeautifulSoup(self._fetchUrl("http://"+self.getSiteDomain()+ur['href']),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+                div1 = soup.find('div', {'id' : 'story'})
+                
+                # appending next section
+                last=div.findAll('p')
+                next=div1.find('span', {'class' : 'conTag'}).nextSibling
+            
+                last[len(last)-1]=last[len(last)-1].append(next)
+                div.append(div1)
+            
+            
+            
+        # removing all the left-over stuff    
+        for a in div.findAll('span'):
+            a.extract()
+            
+        for a in div.findAll('h1'):
+            a.extract()
+        for a in div.findAll('h2'):
+            a.extract()
+        for a in div.findAll('h3'):
+            a.extract()
+        for a in div.findAll('h4'):
+            a.extract()
+        for a in div.findAll('br'):
+            a.extract()
+        for a in div.findAll('div', {'class' : 'date'}):
+            a.extract()
+            
+        a = div.find('form')
+        if a != None:
+            b = a.nextSibling
+            while b != None:
+                a.extract()
+                a=b
+                b=b.nextSibling
+        
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -0,0 +1,297 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return GrangerEnchantedCom
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class GrangerEnchantedCom(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        self.story.setMetadata('section',self.parsedUrl.path.split('/',)[1])
+             
+        # normalized story URL.
+        if "malfoymanor" in self.parsedUrl.netloc:
+            self._setURL('http://malfoymanor.' + self.getSiteDomain() + '/themanor/viewstory.php?sid='+self.story.getMetadata('storyId'))
+            self.story.addToList("category","The Manor")
+        else:
+            self._setURL('http://' + self.getSiteDomain() + '/enchant/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','gech')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%d/%b/%Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'grangerenchanted.com'
+        
+    @classmethod
+    def getAcceptDomains(cls):
+        return ['grangerenchanted.com','malfoymanor.grangerenchanted.com']
+
+    def getSiteExampleURLs(self):
+        return "http://grangerenchanted.com/enchant/viewstory.php?sid=1234 http://malfoymanor.grangerenchanted.com/themanor/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return r"http://(malfoymanor.)?grangerenchanted.com/(enchant|themanor)?/viewstory.php\?sid=\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Registered Users Only' in data \
+                or 'There is no such account on our website' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['cookiecheck'] = '1'
+        params['submit'] = 'Submit'
+    
+        if "enchant" in self.story.getMetadata('section'):
+            loginUrl = 'http://grangerenchanted.com/enchant/user.php?action=login'
+        else:
+            loginUrl = 'http://malfoymanor.grangerenchanted.com/themanor/user.php?action=login'
+        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "Member Account" not in d : #Member Account
+            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=1"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&amp;ageconsent=ok)?&amp;warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid &amp; error in url.
+                addurl = addurl.replace("&amp;","&")
+                url = self.url+'&index=1'+addurl
+                logging.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+self.story.getMetadata('section')+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=4'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+self.story.getMetadata('section')+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            i=1
+            for a in storyas:
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        break
+                    i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story1'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -37,7 +37,6 @@ class HarryPotterFanFictionComSiteAdapter(BaseSiteAdapter):
                               # Most sites that claim to be
                               # iso-8859-1 (and some that claim to be
                               # utf8) are really windows-1252.
-        self.story.addToList("category","Harry Potter")
        self.is_adult=False
        
        # get storyId from url--url validation guarantees query is only psid=1234
@@ -0,0 +1,228 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return HLFictionNetAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class HLFictionNetAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','hlf')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%m/%d/%y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'hlfiction.net'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title and author
+        a = soup.find('div', {'id' : 'pagetitle'})
+        
+        aut = a.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',aut['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+aut['href'])
+        self.story.setMetadata('author',aut.string)
+        aut.extract()
+        
+        self.story.setMetadata('title',a.string[:(len(a.string)-3)])
+
+        # Find the chapters:
+        chapters=soup.find('select')
+        if chapters != None:
+            for chapter in chapters.findAll('option'):
+                # just in case there's tags, like <i> in chapter titles. 
+                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/viewstory.php?sid='+self.story.getMetadata('storyId')+'&chapter='+chapter['value']))
+        else:
+            self.chapterUrls.append((self.story.getMetadata('title'),url))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+        
+        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
+        
+        for list in asoup.findAll('div', {'class' : re.compile('listbox\s+')}):
+            a = list.find('a')
+            if ('viewstory.php?sid='+self.story.getMetadata('storyId')) in a['href']:
+                break
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = list.findAll('span', {'class' : 'classification'})
+        for labelspan in labels:
+            label = labelspan.string
+            value = labelspan.nextSibling
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'classification':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value[:len(value)-2])
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'categories.php\?catid=\d+'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                for char in value.string.split(', '):
+                    if not 'None' in char:
+                        self.story.addToList('characters',char)
+
+            if 'Genre' in label:
+                for genre in value.string.split(', '):
+                    if not 'None' in genre:
+                        self.story.addToList('genre',genre)
+
+            if 'Warnings' in label:
+                for warning in value.string.split(', '):
+                    if not 'None' in warning:
+                        self.story.addToList('warnings',warning)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = list.find('a', href=re.compile(r"series.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            i=1
+            for a in storyas:
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if ('viewstory.php?sid='+self.story.getMetadata('storyId')) in a['href']:
+                        self.setSeries(series_name, i)
+                        break
+                    i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -60,11 +60,6 @@ class HPFandomNetAdapterAdapter(BaseSiteAdapter): # XXX
        # Each adapter needs to have a unique site abbreviation.
        self.story.setMetadata('siteabbrev','hpfdm') # XXX

-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Harry Potter") # XXX
-
        # The date format will vary from site to site.
        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
        self.dateformat = "%Y.%m.%d" # XXX
@@ -0,0 +1,216 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return HPFanficArchiveComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class HPFanficArchiveComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/stories/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','hpffa')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%B %d, %Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.hpfanficarchive.com'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/stories/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/stories/viewstory.php?sid=")+r"\d+$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/stories/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/stories/'+chapter['href']))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) # XXX
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/stories/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            i=1
+            for a in storyas:
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        break
+                    i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+        
+
+        soup = bs.BeautifulSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -0,0 +1,280 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return IkEternalNetAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class IkEternalNetAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','ike')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%B %d, %Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.ik-eternal.net'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Registered Users Only' in data \
+                or 'There is no such account on our website' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['cookiecheck'] = '1'
+        params['submit'] = 'Submit'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
+        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "Member Account" not in d : #Member Account
+            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&warning=1"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+
+        # The actual text that is used to announce you need to be an
+        # adult varies from site to site.  Again, print data before
+        # the title search to troubleshoot.
+            
+        # Since the warning text can change by warning level, let's
+        # look for the warning pass url.  ksarchive uses
+        # &amp;warning= -- actually, so do other sites.  Must be an
+        # eFiction book.
+            
+        # viewstory.php?sid=1882&amp;warning=4
+        # viewstory.php?sid=1654&amp;ageconsent=ok&amp;warning=5
+        #print data
+        #m = re.search(r"'viewstory.php\?sid=1882(&amp;warning=4)'",data)
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&amp;ageconsent=ok)?&amp;warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid &amp; error in url.
+                addurl = addurl.replace("&amp;","&")
+                url = self.url+'&index=1'+addurl
+                logging.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data,selfClosingTags=('p')) #poor formatting of the paragraphs in the title page
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        asoup = soup.find('div', {'class': 'listbox'})
+        for a in asoup.findAll('p'):
+            a.name='br'
+        labels = asoup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -60,11 +60,6 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
        # Each adapter needs to have a unique site abbreviation.
        self.story.setMetadata('siteabbrev','ksa') # XXX

-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Star Trek") # XXX
-
        # The date format will vary from site to site.
        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
        self.dateformat = "%b/%d/%Y" # XXX
@@ -167,7 +162,7 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
        self.story.setMetadata('authorId',a['href'].split('=')[1])
        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
+        self.story.setMetadata('author',stripHTML(a))

        # Find the chapters:
        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
@@ -190,14 +185,20 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
        labels = soup.findAll('span',{'class':'label'})
        for labelspan in labels:
            value = labelspan.nextSibling
-            label = labelspan.string
+            label = stripHTML(labelspan)

            if 'Summary' in label:
                ## Everything until the next span class='label'
                svalue = ""
                while not defaultGetattr(value,'class') == 'label':
                    svalue += str(value)
-                    value = value.nextSibling
+                    # poor HTML(unclosed <p> for one) can cause run on
+                    # over the next label.
+                    if '<span class="label">' in svalue:
+                        svalue = svalue[0:svalue.find('<span class="label">')]
+                        break
+                    else:
+                        value = value.nextSibling
                self.setDescription(url,svalue)
                #self.story.setMetadata('description',stripHTML(svalue))

@@ -209,48 +210,50 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX

            if 'Categories' in label:
                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-                catstext = [cat.string for cat in cats]
+                catstext = [stripHTML(cat) for cat in cats]
                for cat in catstext:
-                    if cat.string.strip() in ('Poetry','Essays'):
-                        self.story.addToList('category',cat.string)
+                    # ran across one story with an empty <a href="browse.php?type=categories&amp;catid=1"></a>
+                    # tag in the desc once.
+                    if cat and cat.strip() in ('Poetry','Essays'):
+                        self.story.addToList('category',stripHTML(cat))

            if 'Characters' in label:
                self.story.addToList('characters','Kirk')
                self.story.addToList('characters','Spock')
                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-                charstext = [char.string for char in chars]
+                charstext = [stripHTML(char) for char in chars]
                for char in charstext:
-                    self.story.addToList('characters',char.string)
+                    self.story.addToList('characters',stripHTML(char))

            ## Not all sites use Genre, but there's no harm to
            ## leaving it in.  Check to make sure the type_id number
            ## is correct, though--it's site specific.
            if 'Genre' in label:
                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) # XXX
-                genrestext = [genre.string for genre in genres]
+                genrestext = [stripHTML(genre) for genre in genres]
                self.genre = ', '.join(genrestext)
                for genre in genrestext:
-                    self.story.addToList('genre',genre.string)
+                    self.story.addToList('genre',stripHTML(genre))

            ## In addition to Genre (which is very site specific) KSA
            ## has 'Story Type', which is much more what most sites
            ## call genre.
            if 'Story Type' in label:
                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=5')) # XXX
-                genrestext = [genre.string for genre in genres]
+                genrestext = [stripHTML(genre) for genre in genres]
                self.genre = ', '.join(genrestext)
                for genre in genrestext:
-                    self.story.addToList('genre',genre.string)
+                    self.story.addToList('genre',stripHTML(genre))

            ## Not all sites use Warnings, but there's no harm to
            ## leaving it in.  Check to make sure the type_id number
            ## is correct, though--it's site specific.
            if 'Warnings' in label:
                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
-                warningstext = [warning.string for warning in warnings]
+                warningstext = [stripHTML(warning) for warning in warnings]
                self.warning = ', '.join(warningstext)
                for warning in warningstext:
-                    self.story.addToList('warnings',warning.string)
+                    self.story.addToList('warnings',stripHTML(warning))

            if 'Completed' in label:
                if 'Yes' in value:
@@ -269,18 +272,20 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
        try:
            # Find Series name from series URL.
            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
-            series_name = a.string
+            series_name = stripHTML(a)
            series_url = 'http://'+self.host+'/'+a['href']

            # use BeautifulSoup HTML parser to make everything easier to find.
            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
            i=1
            for a in storyas:
-                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                    self.setSeries(series_name, i)
-                    break
-                i+=1
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        break
+                    i+=1
            
        except:
            # I find it hard to care if the series parsing fails
@@ -0,0 +1,247 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+
+def getClass():
+    return LibraryOfMoriaComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class LibraryOfMoriaComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/a/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','lom')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%B %d, %Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.libraryofmoria.com'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/a/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/a/viewstory.php?sid=")+r"\d+$"
+
+
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            addurl = "&ageconsent=ok&warning=3"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&amp;ageconsent=ok)?&amp;warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid &amp; error in url.
+                addurl = addurl.replace("&amp;","&")
+                url = self.url+'&index=1'+addurl
+                logging.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/a/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                catstext = [cat.string for cat in cats]
+                for cat in catstext:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                charstext = [char.string for char in chars]
+                for char in charstext:
+                    self.story.addToList('characters',char.string)
+
+            if 'Type' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warning' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=5'))
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/a/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            i=1
+            for a in storyas:
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        break
+                    i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -0,0 +1,234 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return LumosSycophantHexComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class LumosSycophantHexComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+		
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','lsph')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%m/%d/%Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'lumos.sycophanthex.com'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=19"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        # The actual text that is used to announce you need to be an
+        # adult varies from site to site.  Again, print data before
+        # the title search to troubleshoot.
+        if "Age Consent Required" in data:
+            raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        pt = soup.find('div', {'id' : 'pagetitle'})
+        a = pt.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.text)
+        
+        # Find authorid and URL from... author url.
+        a = pt.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        rating=pt.text.split('(')[1].split(')')[0]
+        self.story.setMetadata('rating', rating)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+				
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+		
+        labels = soup.findAll('span',{'class':'label'})
+		
+        value = labels[0].previousSibling
+        svalue = ""
+        while value != None:
+            val = value
+            value = value.previousSibling
+        while not defaultGetattr(val,'class') == 'label':
+            svalue += str(val)
+            val = val.nextSibling
+        self.setDescription(url,svalue)
+		
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value.split(' -')[0])
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Complete' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value.split(' -')[0]), self.dateformat))
+            
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -0,0 +1,290 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return MerlinFicDtwinsCoUk
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class MerlinFicDtwinsCoUk(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','mrfd')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%b %d, %Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'merlinfic.dtwins.co.uk'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Registered Users Only' in data \
+                or 'There is no such account on our website' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['cookiecheck'] = '1'
+        params['submit'] = 'Submit'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
+        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "Member Account" not in d : #Member Account
+            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=4"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&amp;ageconsent=ok)?&amp;warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid &amp; error in url.
+                addurl = addurl.replace("&amp;","&")
+                url = self.url+'&index=1'+addurl
+                logging.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+                    
+            if 'Pairing' in label:
+                ships = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+                for ship in ships:
+                    self.story.addToList('ships',ship.string)
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=3'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            i=1
+            for a in storyas:
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        break
+                    i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -60,11 +60,6 @@ class MidnightwhispersCaAdapter(BaseSiteAdapter): # XXX
        # Each adapter needs to have a unique site abbreviation.
        self.story.setMetadata('siteabbrev','mw') # XXX

-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Queer as Folk") # XXX
-
        # The date format will vary from site to site.
        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
        self.dateformat = "%B %d, %Y" # XXX
@@ -260,10 +255,12 @@ class MidnightwhispersCaAdapter(BaseSiteAdapter): # XXX
            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
            i=1
            for a in storyas:
-                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                    self.setSeries(series_name, i)
-                    break
-                i+=1
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        break
+                    i+=1
            
        except:
            # I find it hard to care if the series parsing fails
@@ -59,11 +59,6 @@ class MuggleNetComAdapter(BaseSiteAdapter): # XXX
        # Each adapter needs to have a unique site abbreviation.
        self.story.setMetadata('siteabbrev','mgln') # XXX

-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Harry Potter") # XXX
-
        # The date format will vary from site to site.
        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
        self.dateformat = "%m/%d/%y" # XXX
@@ -303,13 +298,15 @@ class MuggleNetComAdapter(BaseSiteAdapter): # XXX

            # use BeautifulSoup HTML parser to make everything easier to find.
            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
            i=1
            for a in storyas:
-                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                    self.setSeries(series_name, i)
-                    break
-                i+=1
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        break
+                    i+=1
            
        except:
            # I find it hard to care if the series parsing fails
@@ -0,0 +1,208 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return NationalLibraryNetAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class NationalLibraryNetAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only storyid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?storyid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','ntlb')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%m-%d-%y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        return 'national-library.net'
+        
+    @classmethod
+    def getAcceptDomains(cls):
+        return ['www.national-library.net','national-library.net']
+
+    def getSiteExampleURLs(self):
+        return "ONLY the stories archived on or after June 17, 2006 and that are hosted on the website: http://"+self.getSiteDomain()+"/viewstory.php?storyid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://")+"(www\.)?"+re.escape(self.getSiteDomain()+"/viewstory.php?storyid=")+r"\d+$"
+
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('h1')
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"authorresults.php\?author=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for p in soup.findAll('p'):
+            chapters = p.findAll('a', href=re.compile(r'viewstory.php\?storyid='+self.story.getMetadata('storyId')+"&chapnum=\d+$"))
+            if len(chapters) > 0:
+                for chapter in chapters:
+                    # just in case there's tags, like <i> in chapter titles.
+                    self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']))
+                break
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+        self.story.setMetadata('status', 'Completed')
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('b')
+        for x in range(2,len(labels)):
+            value = labels[x].nextSibling
+            label = labels[x].string
+            
+            if 'Summary' in label:
+                self.setDescription(url,value)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rating' in label:
+                self.story.setMetadata('rating', stripHTML(value.nextSibling))
+
+            if 'Word Count' in label:
+                self.story.setMetadata('numWords', value.string)
+
+            if 'Category' in label:
+                for cat in value.string.split(', '):
+                    self.story.addToList('category',cat)
+            if 'Crossover Shows' in label:
+                for cat in value.string.split(', '):
+                    if "No Show" not in cat:
+                        self.story.addToList('category',cat)
+
+            if 'Character' in label:
+                for char in value.string.split(', '):
+                    self.story.addToList('characters',char)
+                    
+            if 'Pairing' in label:
+                for char in value.string.split(', '):
+                    self.story.addToList('ships',char)
+                    
+            if 'Warnings' in label:
+                for warning in value.string.split(', '):
+                    self.story.addToList('warnings',warning)
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Series' in label:
+                self.setSeries(stripHTML(value.nextSibling), value.nextSibling.nextSibling.string[2:])
+                
+        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
+        story=asoup.find('a', href=re.compile(r'viewstory.php\?storyid='+self.story.getMetadata('storyId')))
+            
+        a=story.findNext(text=re.compile('Genre')).parent.nextSibling.string.split(', ')
+        for genre in a:
+            self.story.setMetadata('genre', genre)
+        
+        a=story.findNext(text=re.compile('Archived'))
+        self.story.setMetadata('datePublished', makeDate(stripHTML(a.parent.nextSibling), self.dateformat))
+        self.story.setMetadata('dateUpdated', makeDate(stripHTML(a.parent.nextSibling), self.dateformat))
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div')
+        
+        # bit messy since higly inconsistent
+        for p in soup.findAll('p', {'align' : 'center'}):
+            p.extract()
+        p = soup.findAll('p')
+        for x in range(0,3):
+            p[x].extract()
+        if "Chapters: " in stripHTML(p[3]):
+            p[3].extract()
+        for x in range(len(p)-2,len(p)-1):
+            p[x].extract()
+
+        for p in soup.findAll('h1'):
+            p.extract()
+        for p in soup.findAll('h3'):
+            p.extract()
+        for p in soup.findAll('a'):
+            p.extract()
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -0,0 +1,215 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return NCISFicComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class NCISFicComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only storyid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?storyid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','ncisf')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%m-%d-%y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        return 'ncisfic.com'
+        
+    @classmethod
+    def getAcceptDomains(cls):
+        return ['www.ncisfic.com','ncisfic.com']
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?storyid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://")+"(www\.)?"+re.escape(self.getSiteDomain()+"/viewstory.php?storyid=")+r"\d+$"
+
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('h1')
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"authorresults.php\?author=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for p in soup.findAll('p'):
+            chapters = p.findAll('a', href=re.compile(r'viewstory.php\?storyid='+self.story.getMetadata('storyId')+"&chapnum=\d+$"))
+            if len(chapters) > 0:
+                for chapter in chapters:
+                    # just in case there's tags, like <i> in chapter titles.
+                    self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']))
+                break
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+        self.story.setMetadata('status', 'Completed')
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('b')
+        for x in range(2,len(labels)):
+            value = labels[x].nextSibling
+            label = labels[x].string
+            
+            if 'Summary' in label:
+                self.setDescription(url,value)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rating' in label:
+                self.story.setMetadata('rating', stripHTML(value.nextSibling))
+
+            if 'Word Count' in label:
+                self.story.setMetadata('numWords', value.string)
+
+            if 'Category' in label:
+                for cat in value.string.split(', '):
+                    self.story.addToList('category',cat)
+            if 'Crossover Shows' in label:
+                for cat in value.string.split(', '):
+                    if "No Show" not in cat:
+                        self.story.addToList('category',cat)
+
+            if 'Character' in label:
+                for char in value.string.split(', '):
+                    self.story.addToList('characters',char)
+                    
+            if 'Pairing' in label:
+                for char in value.string.split(', '):
+                    self.story.addToList('ships',char)
+                    
+            if 'Warnings' in label:
+                for warning in value.string.split(', '):
+                    self.story.addToList('warnings',warning)
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Series' in label:
+                if "No Series" not in value.nextSibling.string:
+                    self.setSeries(stripHTML(value.nextSibling), value.nextSibling.nextSibling.string[2:])
+                
+        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
+        story=asoup.find('a', href=re.compile(r'viewstory.php\?storyid='+self.story.getMetadata('storyId')))
+        
+        a=story.findNext('font')
+        if 'Complete' in a.string:
+            self.story.setMetadata('status', 'Completed')
+        else:
+            self.story.setMetadata('status', 'In-Progress')
+            
+        a=story.findNext(text=re.compile('Genre')).parent.nextSibling.string.split(', ')
+        for genre in a:
+            self.story.setMetadata('genre', genre)
+        
+        a=story.findNext(text=re.compile('Archived'))
+        self.story.setMetadata('datePublished', makeDate(stripHTML(a.parent.nextSibling), self.dateformat))
+        self.story.setMetadata('dateUpdated', makeDate(stripHTML(a.parent.nextSibling), self.dateformat))
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div')
+        
+        # bit messy since higly inconsistent
+        for p in soup.findAll('p', {'align' : 'center'}):
+            p.extract()
+        p = soup.findAll('p')
+        for x in range(0,3):
+            p[x].extract()
+        if "Chapters: " in stripHTML(p[3]):
+            p[3].extract()
+        for x in range(len(p)-2,len(p)-1):
+            p[x].extract()
+
+        for p in soup.findAll('h1'):
+            p.extract()
+        for p in soup.findAll('h3'):
+            p.extract()
+        for p in soup.findAll('a'):
+            p.extract()
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -0,0 +1,200 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return NCISFictionComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class NCISFictionComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["iso-8859-1",
+                       "Windows-1252"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL("http://"+self.getSiteDomain()\
+                         +"/chapters.php?stid="+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','ncisfn')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%d/%m/%Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.ncisfiction.com'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/story.php?stid=01234 http://"+self.getSiteDomain()+"/chapters.php?stid=1234"
+
+    def getSiteURLPattern(self):
+        return "http://"+self.getSiteDomain()+r'/(chapters|story)?.php\?stid=\d+'
+       
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulStoneSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title and author
+        a = soup.find('div', {'class' : 'main_title'})
+        
+        aut = a.find('a')
+        self.story.setMetadata('authorId',aut['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+aut['href'])
+        self.story.setMetadata('author',aut.string)
+        
+        aut.extract()
+        self.story.setMetadata('title',stripHTML(a)[:len(stripHTML(a))-2])
+
+        # Find the chapters:
+        i=0
+        chapters=soup.findAll('table', {'class' : 'story_table'})
+        for chapter in chapters:
+            ch=chapter.find('a')
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(ch),'http://'+self.host+'/'+ch['href']))
+            if i == 0:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(chapter.find('td')).split('Added: ')[1], self.dateformat))
+            if i == len(chapters)-1:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(chapter.find('td')).split('Added: ')[1], self.dateformat))
+            i=i+1
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        info = soup.find('table', {'class' : 'story_info'})
+        
+        # no convenient way to calculate word count as it is logged differently for stories with and without series
+        
+        labels = info.findAll('tr')
+        for tr in labels:
+            value = tr.find('td')
+            label = tr.find('th').string
+
+            if 'Summary' in label:
+                self.setDescription(url,value)
+
+            if 'Rating' in label:
+                self.story.setMetadata('rating', value.string)
+
+            if 'Category' in label:
+                cats = value.findAll('a')
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = value.findAll('a')
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Pairing' in label:
+                ships = value.findAll('a')
+                for ship in ships:
+                    self.story.addToList('ships',ship.string)
+
+            if 'Genre' in label:
+                genres = value.findAll('a')
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = value.findAll('a')
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Status' in label:
+                if 'not completed' in value.text:
+                    self.story.setMetadata('status', 'In-Progress')
+                else:
+                    self.story.setMetadata('status', 'Completed')
+                    
+        try:
+            # Find Series name from series URL.
+            a = soup.find('div',{'class' : 'sub_header'})
+            series_name = a.find('a').string
+            i = a.text.split('#')[1]
+            self.setSeries(series_name, i)
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'class' : 'story_text'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -60,11 +60,6 @@ class NfaCommunityComAdapter(BaseSiteAdapter): # XXX
        # Each adapter needs to have a unique site abbreviation.
        self.story.setMetadata('siteabbrev','nfa') # XXX

-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","NCIS") # XXX
-
        # The date format will vary from site to site.
        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
        self.dateformat = "%m/%d/%Y" # XXX
@@ -261,13 +256,15 @@ class NfaCommunityComAdapter(BaseSiteAdapter): # XXX

            # use BeautifulSoup HTML parser to make everything easier to find.
            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
            i=1
            for a in storyas:
-                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                    self.setSeries(series_name, i)
-                    break
-                i+=1
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        break
+                    i+=1
            
        except:
            # I find it hard to care if the series parsing fails
@@ -0,0 +1,211 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return NHAMagicalWorldsUsAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class NHAMagicalWorldsUsAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+		
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','nha')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = " %m/%d/%y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'nha.magical-worlds.us'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+        url = self.url
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+            
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
+		
+        try:
+            # in case link points somewhere other than the first chapter
+            a = soup.findAll('option')[1]['value']
+            self.story.setMetadata('storyId',a.split('=',)[1])
+            url = 'http://'+self.host+'/'+a
+            soup = bs.BeautifulSoup(self._fetchUrl(url))
+        except:
+            pass
+		
+        for info in asoup.findAll('table', {'width' : '100%', 'bordercolor' : re.compile(r'#')}):
+            a = info.find('a')
+            if ('viewstory.php?sid='+self.story.getMetadata('storyId')) in a['href']:
+                self.story.setMetadata('title',a.string)
+                break
+		
+
+        # Find the chapters:
+        chapters=soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+'&chapter=\d+$'))
+        if len(chapters) == 0:
+            self.chapterUrls.append((self.story.getMetadata('title'),url))
+        else:
+            for chapter in chapters:
+                # just in case there's tags, like <i> in chapter titles.
+                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d):
+            try:
+                return d.name
+            except:
+                return ""
+				
+        cats = info.findAll('a',href=re.compile('categories.php'))
+        for cat in cats:
+            self.story.addToList('category',cat.string)
+					
+        a = info.find('a', href=re.compile(r'reviews.php\?sid='+self.story.getMetadata('storyId')))
+        val = a.nextSibling
+        svalue = ""
+        while not defaultGetattr(val) == 'br':
+            val = val.nextSibling
+        val = val.nextSibling
+        while not defaultGetattr(val) == 'br':
+            svalue += str(val)
+            val = val.nextSibling
+        self.setDescription(url,svalue)
+        
+        #does not provide convenient way to get word count
+        labels = info.findAll('i')
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = stripHTML(labelspan)
+
+            if 'Rating' in label:
+                self.story.setMetadata('rating', value.split(' -')[0])
+
+            if 'Genres' in label:
+                genres = value.string.split(', ')
+                for genre in genres:
+                    if 'None' not in genre:
+                        self.story.addToList('genre',genre.split(' -')[0])
+            
+            if 'Characters' in label:
+                chars = value.string.split(', ')
+                for char in chars:
+                    if 'None' not in char:
+                        self.story.addToList('characters',char.split(' -')[0])
+
+            if 'Warnings' in label:
+                warnings = value.string.split(', ')
+                for warning in warnings:
+                    if 'None' not in warning:
+                        self.story.addToList('warnings',warning.split(' -')[0])
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(value.split(' -')[0], self.dateformat))
+            
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(value.split(' -')[0], self.dateformat))
+
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+		
+        data = self._fetchUrl(url)
+
+        soup = bs.BeautifulSoup(data, selfClosingTags=('br','hr','span','center')) # some chapters seem to be hanging up on those tags, so it is safer to close them
+        
+        story = soup.find('div', {"id" : "story"})
+
+        if None == story:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,story)
@@ -0,0 +1,260 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return OcclumencySycophantHexComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class OcclumencySycophantHexComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+		
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','osph')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%m/%d/%Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'occlumency.sycophanthex.com'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'This story contains adult content and/or themes.' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['rememberme'] = '1'
+        params['sid'] = ''
+        params['intent'] = ''
+        params['submit'] = 'Submit'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/user.php'
+        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "Logout" not in d : #Member Account
+            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
+		
+        try:
+            # in case link points somewhere other than the first chapter
+            a = soup.findAll('option')[1]['value']
+            self.story.setMetadata('storyId',a.split('=',)[1])
+            url = 'http://'+self.host+'/'+a
+            soup = bs.BeautifulSoup(self._fetchUrl(url))
+        except:
+            pass
+		
+        for info in asoup.findAll('table', {'class' : 'border'}):
+            a = info.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+            if a != None:
+                self.story.setMetadata('title',a.text)
+                break
+		
+
+        # Find the chapters:
+        chapters=soup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+&i=1$'))
+        if len(chapters) == 0:
+            self.chapterUrls.append((self.story.getMetadata('title'),url))
+        else:
+            for chapter in chapters:
+                # just in case there's tags, like <i> in chapter titles.
+                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d):
+            try:
+                return d.name
+            except:
+                return ""
+				
+        cats = info.findAll('a',href=re.compile('categories.php'))
+        for cat in cats:
+            self.story.addToList('category',cat.string)
+					
+			
+        a = info.find('a', href=re.compile(r'reviews.php\?sid='+self.story.getMetadata('storyId')))
+        val = a.nextSibling
+        svalue = ""
+        while not defaultGetattr(val) == 'br':
+            val = val.nextSibling
+        val = val.nextSibling
+        while not defaultGetattr(val) == 'table':
+            svalue += str(val)
+            val = val.nextSibling
+        self.setDescription(url,svalue)
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = info.findAll('b')
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = stripHTML(labelspan)
+
+            if 'Rating' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word Count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Genres' in label:
+                genres = value.string.split(', ')
+                for genre in genres:
+                    if genre != 'none':
+                        self.story.addToList('genre',genre)
+						
+            if 'Characters' in label:
+                chars = value.string.split(', ')
+                for char in chars:
+                    if char != 'none':
+                        self.story.addToList('characters',char)
+
+            if 'Warnings' in label:
+                warnings = value.string.split(', ')
+                for warning in warnings:
+                    if warning != ' none':
+                        self.story.addToList('warnings',warning)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+		
+        data = self._fetchUrl(url)
+        data = data.replace('<div align="left"', '<div align="left">')
+
+        soup = bs.BeautifulSoup(data, selfClosingTags=('br','hr'))
+        
+        story = soup.find('div', {"align" : "left"})
+
+        if None == story:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,story)
@@ -0,0 +1,266 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return OneDirectionFanfictionComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class OneDirectionFanfictionComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','odf')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%m/%d/%y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'onedirectionfanfiction.com'
+        
+    @classmethod
+    def getAcceptDomains(cls):
+        return ['www.onedirectionfanfiction.com','onedirectionfanfiction.com']
+
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://")+"(www\.)?"+re.escape(self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+        
+        ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Registered Users Only' in data \
+                or 'There is no such account on our website' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['cookiecheck'] = '1'
+        params['submit'] = 'Submit'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
+        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "Member Account" not in d : #Member Account
+            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+    
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=4"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+            
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+        
+        # The actual text that is used to announce you need to be an
+        # adult varies from site to site.  Again, print data before
+        # the title search to troubleshoot.
+        if "Age Consent Required" in data:
+            raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=6'))
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -0,0 +1,237 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2, urllib, cookielib
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return PhoenixSongNetAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class PhoenixSongNetAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[3])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/fanfiction/story/' +self.story.getMetadata('storyId')+'/')
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','phs')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%B %d %Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.phoenixsong.net'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/fanfiction/story/1234/"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/fanfiction/story/")+r"\d+/?$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Please login to continue.' in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+		
+        if self.password:
+            params['txtusername'] = self.username
+            params['txtpassword'] = self.password
+        else:
+            params['txtusername'] = self.getConfig("username")
+            params['txtpassword'] = self.getConfig("password")
+        #params['remember'] = '1'
+        params['login'] = 'Login'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/users/processlogin.php'
+        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['txtusername']))
+        d = self._fetchUrl(loginUrl, params)
+    
+        if 'Please login to continue.' in d : #Member Account
+            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['txtusername']))
+            raise exceptions.FailedToLogin(url,params['txtusername'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+			
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        b = soup.find('div', {'id' : 'nav25'})
+        a = b.find('a', href=re.compile(r'fanfiction/story/'+self.story.getMetadata('storyId')+"/$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.  /fanfiction/stories.php?psid=125
+        a = b.find('a', href=re.compile(r"/fanfiction/stories.php\?psid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        chapters = soup.find('select')
+        if chapters == None:
+            self.chapterUrls.append((self.story.getMetadata('title'),url))
+            for b in soup.findAll('b'):
+                if b.text == "Updated":
+                    date = b.nextSibling.string.split(': ')[1].split(',')
+                    self.story.setMetadata('datePublished', makeDate(date[0]+date[1], self.dateformat))
+                    self.story.setMetadata('dateUpdated', makeDate(date[0]+date[1], self.dateformat))
+        else:
+            i = 0
+            chapters = chapters.findAll('option')
+            for chapter in chapters:
+                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+chapter['value']))
+                if i == 0:
+                    self.story.setMetadata('storyId',chapter['value'].split('/')[3])
+                    head = bs.BeautifulSoup(self._fetchUrl('http://'+self.host+chapter['value'])).findAll('b')
+                    for b in head:
+                        if b.text == "Updated":
+                            date = b.nextSibling.string.split(': ')[1].split(',')
+                            self.story.setMetadata('datePublished', makeDate(date[0]+date[1], self.dateformat))
+                            
+                if  i == (len(chapters)-1):
+                    head = bs.BeautifulSoup(self._fetchUrl('http://'+self.host+chapter['value'])).findAll('b')
+                    for b in head:
+                        if b.text == "Updated":
+                            date = b.nextSibling.string.split(': ')[1].split(',')
+                            self.story.setMetadata('dateUpdated', makeDate(date[0]+date[1], self.dateformat))
+                i = i+1
+				
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))        	
+				
+        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
+        
+        info = asoup.find('a', href=re.compile(r'fanfiction/story/'+self.story.getMetadata('storyId')+"/$"))
+        while info != None:
+            info = info.findNext('div')
+            b = info.find('b')
+            val = b.nextSibling
+            
+            if 'Rating' in b.string:
+                self.story.setMetadata('rating', val.string.split(': ')[1])
+				
+            if 'Words' in b.string:
+                self.story.setMetadata('numWords', val.string.split(': ')[1])
+				
+            if 'Setting' in b.string:
+                self.story.addToList('category', val.string.split(': ')[1])
+				
+            if 'Status' in b.string:
+                if 'Completed' in val:
+                    val = 'Completed'
+                else:
+                    val = 'In-Progress'
+                self.story.setMetadata('status', val)
+            
+            if 'Summary' in b.string:
+                b.extract()
+                info.find('br').extract()
+                self.setDescription(url,info)
+                break
+	
+	
+	# grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+
+        chapter=bs.BeautifulSoup('<div class="story"></div>')
+        for p in soup.findAll('p'):
+            if "This is for problems with the formatting or the layout of the chapter." in stripHTML(p):
+                break
+            chapter.append(p)
+			
+        for a in chapter.findAll('div'):
+            a.extract()
+        for a in chapter.findAll('table'):
+            a.extract()
+        for a in chapter.findAll('script'):
+            a.extract()
+        for a in chapter.findAll('form'):
+            a.extract()
+        for a in chapter.findAll('textarea'):
+            a.extract()
+
+
+        if None == chapter:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,chapter)
@@ -0,0 +1,247 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return PonyFictionArchiveNetAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class PonyFictionArchiveNetAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logging.info(self.parsedUrl.netloc)
+        # normalized story URL.
+        if "explicit" in self.parsedUrl.netloc:
+            self._setURL('http://explicit.' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+            self.dateformat = "%d/%b/%y"
+        else:
+            self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+            self.dateformat = "%d %b %Y"
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','pffa')
+
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'ponyfictionarchive.net'
+        
+    @classmethod
+    def getAcceptDomains(cls):
+        return ['www.ponyfictionarchive.net','ponyfictionarchive.net','explicit.ponyfictionarchive.net']
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://")+"(www\.|explicit\.)?"+re.escape(self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&warning=9"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&amp;ageconsent=ok)?&amp;warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid &amp; error in url.
+                addurl = addurl.replace("&amp;","&")
+                url = self.url+'&index=1'+addurl
+                logging.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        genres = soup.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+        for genre in genres:
+            self.story.addToList('genre',genre.string)
+           
+        warnings = soup.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=3'))
+        for warning in warnings:
+            self.story.addToList('warnings',warning.string)
+        
+        status = soup.find('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
+        self.story.setMetadata('status',status.string)
+        
+        section = soup.findAll('span', {'class' : 'General'})[1]            
+        
+        self.story.setMetadata('rating', section.previousSibling.previousSibling.string)
+
+        value = section.nextSibling
+        svalue = ""
+        while not defaultGetattr(value,'class') == 'label':
+            svalue += str(value)
+            value = value.nextSibling
+        self.setDescription(url,svalue)
+        
+        # <span class="label">Rated:</span> NC-17<br /> etc           
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            i=1
+            for a in storyas:
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        break
+                    i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -62,11 +62,6 @@ class PortkeyOrgAdapter(BaseSiteAdapter): # XXX
        # Each adapter needs to have a unique site abbreviation.
        self.story.setMetadata('siteabbrev','prtky') # XXX

-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Harry Potter") # XXX
-
        # The date format will vary from site to site.
        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
        self.dateformat = "%d/%m/%y" # XXX
@@ -37,7 +37,6 @@ class PotionsAndSnitchesNetSiteAdapter(BaseSiteAdapter):
                               # Most sites that claim to be
                               # iso-8859-1 (and some that claim to be
                               # utf8) are really windows-1252.
-        self.story.addToList("category","Harry Potter")
        
        # get storyId from url--url validation guarantees query is only sid=1234
        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
@@ -142,7 +141,7 @@ class PotionsAndSnitchesNetSiteAdapter(BaseSiteAdapter):
                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
                charstext = [char.string for char in chars]
                for char in charstext:
-                    if char == "!Snape and Harry (required)":
+                    if "Snape and Harry (required)" in char:
                        self.story.addToList('characters',"Snape")
                        self.story.addToList('characters',"Harry")
                    else:
@@ -0,0 +1,250 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return PretenderCenterComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class PretenderCenterComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/missingpieces/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','ptdc')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%d/%m/%y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'pretendercentre.com'
+        
+    @classmethod
+    def getAcceptDomains(cls):
+        return ['www.pretendercentre.com','pretendercentre.com']
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/missingpieces/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://")+"(www\.)?"+re.escape(self.getSiteDomain()+"/missingpieces/viewstory.php?sid=")+r"\d+$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=4"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&amp;ageconsent=ok)?&amp;warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid &amp; error in url.
+                addurl = addurl.replace("&amp;","&")
+                url = self.url+'&index=1'+addurl
+                logging.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.text)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/missingpieces/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/missingpieces/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+                
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) # XXX
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/missingpieces/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            i=1
+            for a in storyas:
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        break
+                    i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.findAll('div', {'id' : 'story'})[1]
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -0,0 +1,215 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return PrisonBreakFicNetAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class PrisonBreakFicNetAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','pbf')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%B %d, %Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.prisonbreakfic.net'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+                
+# summary, rated, word count, categories, characters, genre, warnings, completed, published, updated, seires
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Pairing' in label:
+                ships = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=3')) # XXX
+                for ship in ships:
+                    self.story.addToList('ships',ship.string.replace(" and ","/"))
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) # XXX
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -0,0 +1,229 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return SamDeanArchiveNuAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class SamDeanArchiveNuAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','sda')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%m/%d/%y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'samdean.archive.nu'
+
+    @classmethod
+    def getAcceptDomains(cls):
+        return ['www.samdean.archive.nu','samdean.archive.nu']
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://")+r"(www\.)?"+re.escape(self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title and author
+        a = soup.find('div', {'id' : 'pagetitle'})
+        
+        aut = a.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',aut['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+aut['href'])
+        self.story.setMetadata('author',aut.string)
+        aut.extract()
+        
+        self.story.setMetadata('title',a.string[:(len(a.string)-3)])
+
+        # Find the chapters:
+        chapters=soup.find('select')
+        if chapters != None:
+            for chapter in chapters.findAll('option'):
+                # just in case there's tags, like <i> in chapter titles. http://samdean.archive.nu/viewstory.php?sid=4317&chapter=2
+                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/viewstory.php?sid='+self.story.getMetadata('storyId')+'&chapter='+chapter['value']))
+        else:
+            self.chapterUrls.append((self.story.getMetadata('title'),url))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+        
+        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
+        
+        for list in asoup.findAll('div', {'class' : re.compile('listbox\s+')}):
+            a = list.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+            if a != None:
+                break
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = list.findAll('span', {'class' : 'classification'})
+        for labelspan in labels:
+            label = labelspan.string
+            value = labelspan.nextSibling
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'classification':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value[:len(value)-2])
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'categories.php\?catid=\d+'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                for char in value.string.split(', '):
+                    if not 'None' in char:
+                        self.story.addToList('characters',char)
+
+            if 'Genre' in label:
+                for genre in value.string.split(', '):
+                    if not 'None' in genre:
+                        self.story.addToList('genre',genre)
+
+            if 'Warnings' in label:
+                for warning in value.string.split(', '):
+                    if not 'None' in warning:
+                        self.story.addToList('warnings',warning)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = list.find('a', href=re.compile(r"series.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -0,0 +1,244 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return ScarvesAndCoffeeNetAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class ScarvesAndCoffeeNetAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','scacf')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%m/%d/%Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.scarvesandcoffee.net'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Registered Users Only' in data \
+                or 'There is no such account on our website' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=20"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&amp;ageconsent=ok)?&amp;warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid &amp; error in url.
+                addurl = addurl.replace("&amp;","&")
+                url = self.url+'&index=1'+addurl
+                logging.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('genre',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            i=1
+            for a in storyas:
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        break
+                    i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -0,0 +1,255 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return SG1HeliopolisComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class SG1HeliopolisComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        self.story.setMetadata('section',self.parsedUrl.path.split('/',)[1])
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/'+self.story.getMetadata('section')+'/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','sghp')
+
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.  Can't use extracategories, could be Atlantis or SG-1
+        if 'atlantis' in self.story.getMetadata('section'):
+            self.story.addToList("category","Stargate: Atlantis")
+        else:
+            self.story.addToList("category","Stargate: SG-1")
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%Y.%m.%d"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'sg1-heliopolis.com'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/archive/viewstory.php?sid=1234 http://"+self.getSiteDomain()+"/adult/viewstory.php?sid=1234 http://"+self.getSiteDomain()+"/atlantis/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return r"http://sg1-heliopolis.com/(archive|adult|atlantis)?/viewstory.php\?sid=\d+$"
+
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&warning=4"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&amp;ageconsent=ok)?&amp;warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid &amp; error in url.
+                addurl = addurl.replace("&amp;","&")
+                url = self.url+'&index=1'+addurl
+                logging.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+self.story.getMetadata('section')+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+self.story.getMetadata('section')+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            i=1
+            for a in storyas:
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        break
+                    i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -0,0 +1,248 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return SinfulDesireOrgAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class SinfulDesireOrgAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/archive/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','snds')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%m/%d/%Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.sinful-desire.org'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/archive/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/archive/viewstory.php?sid=")+r"\d+$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&warning=5"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&amp;ageconsent=ok)?&amp;warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid &amp; error in url.
+                addurl = addurl.replace("&amp;","&")
+                url = self.url+'&index=1'+addurl
+                logging.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/archive/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/archive/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=3'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/archive/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            i=1
+            for a in storyas:
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        break
+                    i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -56,11 +56,6 @@ class SiyeCoUkAdapter(BaseSiteAdapter): # XXX
        # Each adapter needs to have a unique site abbreviation.
        self.story.setMetadata('siteabbrev','siye') # XXX

-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Harry Potter") # XXX
-
        # The date format will vary from site to site.
        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
        self.dateformat = "%Y.%m.%d" # XXX
@@ -174,7 +169,16 @@ class SiyeCoUkAdapter(BaseSiteAdapter): # XXX
            
        # eFiction sites don't help us out a lot with their meta data
        # formating, so it's a little ugly.
-        moremeta = stripHTML(titlea.parent.parent.parent.find('div',{'class':'desc'}))
+                
+        # SIYE formats stories in the author list differently when their part of a series.
+        # Look for non-series...
+        divdesc = titlea.parent.parent.find('div',{'class':'desc'})
+        if not divdesc:
+            # ... now look for series.
+            divdesc = titlea.parent.parent.findNextSibling('tr').find('div',{'class':'desc'})
+
+        moremeta = stripHTML(divdesc)
+        #print("moremeta:%s"%moremeta)
        for part in moremeta.replace(' - ','\n').split('\n'):
            #print("part:%s"%part)
            try:
@@ -0,0 +1,236 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+
+def getClass():
+    return SquidgeOrgPejaAdapter
+
+## XXX IMPORTANT NOTE!!  This adapter is for squidge.org/peja ONLY!
+## There are lots of other sites and stuff under squidge.org that
+## we're not supporting.  If/When we ever want to support more
+## sections of squidge.org, FFDL will need to be changed more
+## fundamentally to find different adapters under the same domain.
+##
+## For now, I've only implemented the part for ini section names so
+## if/when more adapters under squidge.org come along, existing ini
+## files will still work correctly.
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class SquidgeOrgPejaAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["utf8",
+                       "Windows-1252"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/peja/cgi-bin/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','wwomb')
+        
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%m/%d/%y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.squidge.org'
+    
+    @classmethod # must be @staticmethod, don't remove it.
+    def getConfigSection(cls):
+        # The config section name.  Only override if != site domain.
+        return cls.getSiteDomain()+'/peja'
+    
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/peja/cgi-bin/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/")+"~?"+re.escape("peja/cgi-bin/viewstory.php?sid=")+r"\d+$"        
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        url = self.url
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        data = self._fetchUrl(url)
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        # Find authorid and URL from... author url.
+        author = soup.find('div', {'id':"pagetitle"}).find('a')
+        self.story.setMetadata('authorId',author['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/peja/cgi-bin/'+author['href'])
+        self.story.setMetadata('author',author.string)
+		
+        authorSoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        title = authorSoup.find('a',{'href':'viewstory.php?sid='+self.story.getMetadata('storyId')})
+        self.story.setMetadata('title',title.string)
+        titleblock=title.parent.parent
+        
+        chapterselect=soup.find('select',{'name':'chapter'})
+        if chapterselect:
+            for ch in chapterselect.findAll('option'):
+                self.chapterUrls.append((stripHTML(ch),'http://'+self.host+'/peja/cgi-bin/viewstory.php?sid='+self.story.getMetadata('storyId')+'&chapter='+ch['value']))
+        else:
+            self.chapterUrls.append((title,url))
+		
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="classification">Rated:</span> NC-17<br /> etc
+        labels = titleblock.findAll('span',{'class':'classification'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'classification':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                if value.endswith("["):
+                    value = value[:-1]
+                self.story.setMetadata('rating', value)
+
+            if 'Characters' in label:
+                for char in value.split(','):
+                    self.story.addToList('characters',char.strip())
+
+            if 'Genre' in label:
+                for genre in value.split(','):
+                    if genre.strip() != "None":
+                        self.story.addToList('genre',genre.strip())
+
+            if 'Warnings' in label:
+                for warning in value.split(','):
+                    if warning.strip() != 'None':
+                        self.story.addToList('warnings',warning.strip())
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Fandoms' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'categories.php'))
+                catstext = [cat.string for cat in cats]
+                for cat in catstext:
+                    self.story.addToList('category',cat.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            # http://www.squidge.org/peja/cgi-bin/series.php?seriesid=254
+            a = titleblock.find('a', href=re.compile(r"series.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/peja/cgi-bin/'+a['href']
+    
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+                
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+	
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        chaptext = soup.find('div',{'id':"story"}).find('span')
+
+        if None == chaptext:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,chaptext)
@@ -0,0 +1,226 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return StargateAtlantisOrgAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class StargateAtlantisOrgAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/fanfics/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','stat')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%B %d %Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'stargate-atlantis.org'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/fanfics/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/fanfics/viewstory.php?sid=")+r"\d+$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title and author
+        a = soup.find('div', {'id' : 'pagetitle'})
+        
+        aut = a.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',aut['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/fanfics/'+aut['href'])
+        self.story.setMetadata('author',aut.string)
+        aut.extract()
+        
+        self.story.setMetadata('title',a.string[:(len(a.string)-3)])
+
+        # Find the chapters:
+        chapters=soup.findAll('div', {'class' : 'content'})
+        if len(chapters) > 1:
+            for chapter in chapters:
+                # just in case there's tags, like <i> in chapter titles. 
+                link = chapter.find('a')
+                self.chapterUrls.append((stripHTML(link),'http://'+self.host+'/fanfics/'+link['href']))
+        else:
+            self.chapterUrls.append((self.story.getMetadata('title'),url))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+        
+        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
+        
+        for list in asoup.findAll('div', {'class' : re.compile('listbox\s+')}):
+            a = list.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+            if a != None:
+                break
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = list.findAll('span', {'class' : 'classification'})
+        for labelspan in labels:
+            label = labelspan.string
+            value = labelspan.nextSibling
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'tail' and value != None:
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value[:len(value)-2])
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'categories.php\?catid=\d+'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                for char in value.string.split(', '):
+                    if not 'None' in char:
+                        self.story.addToList('characters',char)
+
+            if 'Genre' in label:
+                for genre in value.string.split(', '):
+                    if not 'None' in genre:
+                        self.story.addToList('genre',genre)
+
+            if 'Warnings' in label:
+                for warning in value.string.split(', '):
+                    if not 'None' in warning:
+                        self.story.addToList('warnings',warning)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value.split(' ::')[0]), self.dateformat))
+            
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = list.find('a', href=re.compile(r"series.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/fanfics/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+                                     
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -0,0 +1,147 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return StoriesOfArdaComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class StoriesOfArdaComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/chapterlistview.asp?SID='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','soa')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%m/%d/%Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.storiesofarda.com'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/chapterlistview.asp?SID=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/chapterlistview.asp?SID=")+r"\d+$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title and author
+        a = soup.find('th', {'colspan' : '3'})
+        
+        aut = a.find('a')
+        self.story.setMetadata('authorId',aut['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+aut['href'])
+        self.story.setMetadata('author',aut.string)
+        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
+        
+        a.find('em').extract()
+        self.story.setMetadata('title',a.text)
+
+        # Find the chapters: chapterview.asp?sid=7000&cid=30919
+        chapters=soup.findAll('a', href=re.compile(r'chapterview.asp\?sid='+self.story.getMetadata('storyId')+"&cid=\d+$"))
+        if len(chapters)==1:
+            self.chapterUrls.append((self.story.getMetadata('title'),'http://'+self.host+'/'+chapters[0]['href']))
+        else:
+            for chapter in chapters:
+                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+        
+        summary = soup.find('td', {'colspan' : '3'})
+        self.setDescription(url,summary)
+        
+        # no convenient way to get word count
+
+        for td in asoup.findAll('td', {'colspan' : '3'}):
+            if td.find('a', href=re.compile('chapterlistview.asp\?SID='+self.story.getMetadata('storyId'))) != None:
+                break     
+        td=td.nextSibling.nextSibling
+        self.story.setMetadata('dateUpdated', makeDate(stripHTML(td).split(': ')[1], self.dateformat))
+        tr=td.parent.nextSibling.nextSibling.nextSibling.nextSibling
+        td=tr.findAll('td')
+        self.story.setMetadata('rating', td[0].string.split(': ')[1])
+        self.story.setMetadata('status', td[2].string.split(': ')[1])
+        self.story.setMetadata('datePublished', makeDate(stripHTML(td[4]).split(': ')[1], self.dateformat))
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('table', {'width' : '90%'}).find('td')
+        div.name='div'
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -0,0 +1,271 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return SVUFictionComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class SVUFictionComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','svuf')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%b %d, %Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'svufiction.com'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Registered Users Only' in data \
+                or 'There is no such account on our website' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['cookiecheck'] = '1'
+        params['submit'] = 'Submit'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
+        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "Member Account" not in d : #Member Account
+            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+    
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=6"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+                
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+                
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&amp;ageconsent=ok)?&amp;warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid &amp; error in url.
+                addurl = addurl.replace("&amp;","&")
+                url = self.url+'&index=1'+addurl
+                logging.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        pt = soup.find('div', {'class' : 'title'})
+        a = pt.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        if a.text != "":
+            self.story.setMetadata('title',a.text)
+        else:
+            self.story.setMetadata('title','Banner')
+        
+        # Find authorid and URL from... author url.
+        a = pt.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        rating=pt.text.split('[')[1].split(']')[0]
+        self.story.setMetadata('rating', rating)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        bottom=soup.find('div', {'class' : 'bottom'}).text
+        self.story.setMetadata('numWords', bottom.split('Word count: ')[1].split(' Read:')[0])
+        self.story.setMetadata('datePublished', makeDate(bottom.split('Published: ')[1].split(' Updated:')[0], self.dateformat))
+        self.story.setMetadata('dateUpdated', makeDate(bottom.split('Updated: ')[1], self.dateformat))
+
+        
+        content=soup.find('div', {'class' : 'content'})
+        value=content.find('h1').nextSibling
+        svalue = ""
+        while not defaultGetattr(value,'class') == 'smaller':
+            svalue += str(value)
+            value = value.nextSibling
+        self.setDescription(url,svalue)
+        
+        status = content.text.split('Completed: ')[1]
+        if 'Yes' in status:
+            self.story.setMetadata('status', 'Completed')
+        else:
+            self.story.setMetadata('status', 'In-Progress')
+        
+        cats = content.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+        for cat in cats:
+            self.story.addToList('category',cat.string)
+            
+        chars = content.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+        for char in chars:
+            self.story.addToList('characters',char.string)
+            
+        genres = content.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+        for genre in genres:
+            self.story.addToList('genre',genre.string)
+            
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            i=1
+            for a in storyas:
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        break
+                    i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -73,8 +73,8 @@ class TestSiteAdapter(BaseSiteAdapter):
            self.story.setMetadata(u'title',"Test Story Title "+self.story.getMetadata('storyId'))
            self.story.setMetadata('author','Test Author aa')
        self.story.setMetadata('storyUrl',self.url)
-        self.story.setMetadata('description',u'Description '+self.crazystring+u''' Done
-
+        self.setDescription(self.url,u'Description '+self.crazystring+u''' Done
+<p>
 Some more longer description.  "I suck at summaries!"  "Better than it sounds!"  "My first fic"
 ''')
        self.story.setMetadata('datePublished',makeDate("1975-03-15","%Y-%m-%d"))
@@ -90,22 +90,32 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
        else:
            self.story.setMetadata('status','Completed')

-        langs = {
-            0:"English",
-            1:"Russian",
-            2:"French",
-            3:"German",
-            }
+        # greater than 10, no language or series.
        if idnum < 10:
+            langs = {
+                0:"English",
+                1:"Russian",
+                2:"French",
+                3:"German",
+                }
            self.story.setMetadata('language',langs[idnum%len(langs)])
-        # greater than 10, no language.
-            
-        self.setSeries('The Great Test',idnum)
+            self.setSeries('The Great Test',idnum)
        
        self.story.setMetadata('rating','Tweenie')
        
-        self.story.setMetadata('authorId','98765')
-        self.story.setMetadata('authorUrl','http://author/url')
+
+        if self.story.getMetadata('storyId') == '673':
+            self.story.addToList('author','Author From List')
+            self.story.addToList('author','Author From List 2')
+        
+            self.story.addToList('authorId','98765')
+            self.story.addToList('authorId','98765-2')
+        
+            self.story.addToList('authorUrl','http://author/url')
+            self.story.addToList('authorUrl','http://author/url-2')
+        else:
+            self.story.setMetadata('authorId','98765')
+            self.story.setMetadata('authorUrl','http://author/url')

        self.story.addToList('warnings','Swearing')
        self.story.addToList('warnings','Violence')
@@ -115,35 +125,60 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
        self.story.addToList('category','Crossover')
        self.story.addToList('category',u'Puella Magi Madoka Magica/魔法少女まどか★マギカ')
        self.story.addToList('category',u'Magical Girl Lyrical Nanoha')
+
        self.story.addToList('genre','Fantasy')
+        self.story.addToList('genre','Comedy')
        self.story.addToList('genre','SF')
        self.story.addToList('genre','Noir')
        
+        self.story.addToList('characters','Bob Smith')
+        self.story.addToList('characters','George Johnson')
+        self.story.addToList('characters','Fred Smythe')
+        
+        self.story.addToList('listX','xVal1')
+        self.story.addToList('listX','xVal2')
+        self.story.addToList('listX','xVal3')
+        self.story.addToList('listX','xVal4')
+        
+        self.story.addToList('listY','yVal1')
+        self.story.addToList('listY','yVal2')
+        self.story.addToList('listY','yVal3')
+        self.story.addToList('listY','yVal4')
+        
+        self.story.addToList('listZ','zVal1')
+        self.story.addToList('listZ','zVal2')
+        self.story.addToList('listZ','zVal3')
+        self.story.addToList('listZ','zVal4')
+        
+        self.story.setMetadata('metaA','98765')
+        self.story.setMetadata('metaB','01245')
+        self.story.setMetadata('metaC','The mighty metaC!')
+
        self.chapterUrls = [(u'Prologue '+self.crazystring,self.url+"&chapter=1"),
                            ('Chapter 1, Xenos on Cinnabar',self.url+"&chapter=2"),
                            ('Chapter 2, Sinmay on Kintikin',self.url+"&chapter=3"),
                            ('Chapter 3, Over Cinnabar',self.url+"&chapter=4"),
                            ('Chapter 4',self.url+"&chapter=5"),
                            ('Chapter 5',self.url+"&chapter=6"),
-                            ('Chapter 6',self.url+"&chapter=6"),
-                            # ('Chapter 7',self.url+"&chapter=6"),
-                            # ('Chapter 8',self.url+"&chapter=6"),
-                            # ('Chapter 9',self.url+"&chapter=6"),
-                            # ('Chapter 0',self.url+"&chapter=6"),
-                            # ('Chapter a',self.url+"&chapter=6"),
-                            # ('Chapter b',self.url+"&chapter=6"),
-                            # ('Chapter c',self.url+"&chapter=6"),
-                            # ('Chapter d',self.url+"&chapter=6"),
-                            # ('Chapter e',self.url+"&chapter=6"),
-                            # ('Chapter f',self.url+"&chapter=6"),
-                            # ('Chapter g',self.url+"&chapter=6"),
-                            # ('Chapter h',self.url+"&chapter=6"),
-                            # ('Chapter i',self.url+"&chapter=6"),
-                            # ('Chapter j',self.url+"&chapter=6"),
-                            # ('Chapter k',self.url+"&chapter=6"),
-                            # ('Chapter l',self.url+"&chapter=6"),
-                            # ('Chapter m',self.url+"&chapter=6"),
-                            # ('Chapter n',self.url+"&chapter=6"),
+                            #('Chapter 6',self.url+"&chapter=7"),
+                            #('Chapter 7',self.url+"&chapter=8"),
+                            #('Chapter 8',self.url+"&chapter=9"),
+                            #('Chapter 9',self.url+"&chapter=0"),
+                            #('Chapter 0',self.url+"&chapter=a"),
+                            #('Chapter a',self.url+"&chapter=b"),
+                            #('Chapter b',self.url+"&chapter=c"),
+                            #('Chapter c',self.url+"&chapter=d"),
+                            #('Chapter d',self.url+"&chapter=e"),
+                            #('Chapter e',self.url+"&chapter=f"),
+                            #('Chapter f',self.url+"&chapter=g"),
+                            #('Chapter g',self.url+"&chapter=h"),
+                            #('Chapter h',self.url+"&chapter=i"),
+                            #('Chapter i',self.url+"&chapter=j"),
+                            #('Chapter j',self.url+"&chapter=k"),
+                            #('Chapter k',self.url+"&chapter=l"),
+                            #('Chapter l',self.url+"&chapter=m"),
+                            #('Chapter m',self.url+"&chapter=n"),
+                            #('Chapter n',self.url+"&chapter=o"),
                            ]
        self.story.setMetadata('numChapters',len(self.chapterUrls))
                            
@@ -161,7 +196,7 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
            text=u'''
 <div>
 <h3>Prologue</h3>
-<p>This is a fake adapter for testing purposes.  Different storyId's will give different errors:</p>
+<p>This is a fake adapter for testing purposes.  Different sid's will give different errors:</p>
 <p>http://test1.com?sid=664 - Crazy string title</p>
 <p>http://test1.com?sid=665 - raises AdultCheckRequired</p>
 <p>http://test1.com?sid=666 - raises StoryDoesNotExist</p>
@@ -171,19 +206,20 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
 <p>http://test1.com?sid=670 - Succeeds, but sleeps 2sec on each chapter</p>
 <p>http://test1.com?sid=671 - Succeeds, but sleeps 2sec metadata only</p>
 <p>http://test1.com?sid=672 - Succeeds, quick meta, sleeps 2sec chapters only</p>
-<p>And other storyId will succeed with the same output.</p>
+<p>http://test1.com?sid=673 - Succeeds, multiple authors</p>
+<p>Odd sid's will be In-Progress, evens complete.  sid&lt;10 will be assigned one of four languages and included in a series.</p>
 </div>
 '''
        else:
            text=u'''
 <div>
 <h3>Chapter title from site</h3>
-<p><center>Centered text</center></p>
-<p>Lorem '''+self.crazystring+''' <i>italics</i>, <b>bold</b>, <u>underline</u> consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p>
+<p>Lorem '''+self.crazystring+u''' <i>italics</i>, <b>bold</b>, <u>underline</u> consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p>
 br breaks<br><br>
-
-<a href="http://code.google.com/p/fanficdownloader/wiki/FanFictionDownLoaderPluginWithReadingList" title="Tilt-a-Whirl by Jim &amp; Sarah, on Flickr"><img src="http://i.imgur.com/bo8eD.png"></a><br/>
+Puella Magi Madoka Magica/魔法少女まどか★マギカ
+<!-- a href="http://code.google.com/p/fanficdownloader/wiki/FanFictionDownLoaderPluginWithReadingList" title="Tilt-a-Whirl by Jim &amp; Sarah, on Flickr"><img src="http://i.imgur.com/bo8eD.png"></a --><br/>
 br breaks<br><br>
+Don't&#8212e;ver&#8212d;o&#8212;that&#8212a;gain, &#27861; &#xE9;
 <hr>
 horizontal rules
 <hr size=1 noshade>
@@ -0,0 +1,211 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return TheAlphaGateComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class TheAlphaGateComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','tag')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%d %b %Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.thealphagate.com'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -0,0 +1,203 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return TheHexFilesNetAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class TheHexFilesNetAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','thf')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%Y.%m.%d"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'thehexfiles.net'
+
+    @classmethod
+    def getAcceptDomains(cls):
+        return ['www.thehexfiles.net','thehexfiles.net']
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://")+"(www\.)?"+re.escape(self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
+		
+        try:
+            # in case link points somewhere other than the first chapter
+            a = soup.findAll('option')[1]['value']
+            self.story.setMetadata('storyId',a.split('=',)[1])
+            url = 'http://'+self.host+'/'+a
+            soup = bs.BeautifulSoup(self._fetchUrl(url))
+        except:
+            pass
+		
+        for info in asoup.findAll('table', {'cellspacing' : '4'}):
+            a = info.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+            if a != None:
+                self.story.setMetadata('title',a.string)
+                break
+		
+
+        # Find the chapters:
+        chapters=soup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+&i=1$'))
+        if len(chapters) == 0:
+            self.chapterUrls.append((self.story.getMetadata('title'),url))
+        else:
+            for chapter in chapters:
+                # just in case there's tags, like <i> in chapter titles.
+                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+				
+        cats = info.findAll('a',href=re.compile('categories.php'))
+        for cat in cats:
+            self.story.addToList('category',cat.string)
+			
+        words = info.find(text=re.compile('Words:')).split('|')[1].split(': ')[1]
+        self.story.setMetadata('numWords', words)
+		
+        comp = info.find('span', {'class' : 'completed'}).string.split(': ')[1]
+        if 'Yes' in comp:
+            self.story.setMetadata('status', 'Completed')
+        else:
+            self.story.setMetadata('status', 'In-Progress')
+						
+        summary = info.find('td', {'class' : 'summary'})
+        self.setDescription(url,summary)
+		
+        rating=stripHTML(info.find('td', {'align' : 'left'})).split('(')[1].split(')')[0]
+        self.story.setMetadata('rating', rating)
+
+        labels = info.findAll('td', {'width' : '10%'})
+        values = info.findAll('td', {'width' : '40%'})
+        for i in range(0,len(labels)):
+            value = stripHTML(values[i])
+            label = stripHTML(labels[i])
+
+            if 'Genres' in label:
+                genres = value.split(', ')
+                for genre in genres:
+                    if genre != 'none':
+                        self.story.addToList('genre',genre)
+						
+
+            if 'Warnings' in label:
+                warnings = value.split(', ')
+                for warning in warnings:
+                    if warning != 'none':
+                        self.story.addToList('warnings',warning)
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr','img')) # otherwise soup eats the br/hr tags.
+
+        if None == soup:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        # Ugh.  chapter html doesn't haven't anything useful around it to demarcate.
+        for a in soup.findAll('table'):
+            a.extract()        
+
+        for a in soup.findAll('head'):
+            a.extract()
+
+        html = soup.find('html')
+        html.name='div'
+            
+        return self.utf8FromSoup(url,soup)
@@ -0,0 +1,305 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+# By virtue of being recent and requiring both is_adult and user/pass,
+# adapter_fanficcastletvnet.py is the best choice for learning to
+# write adapters--especially for sites that use the eFiction system.
+# Most sites that have ".../viewstory.php?sid=123" in the story URL
+# are eFiction.
+
+# For non-eFiction sites, it can be considerably more complex, but
+# this is still a good starting point.
+
+# In general an 'adapter' needs to do these five things:
+
+# - 'Register' correctly with the downloader
+# - Site Login (if needed)
+# - 'Are you adult?' check (if needed--some do one, some the other, some both)
+# - Grab the chapter list
+# - Grab the story meta-data (some (non-eFiction) adapters have to get it from the author page)
+# - Grab the chapter texts
+
+# Search for XXX comments--that's where things are most likely to need changing.
+
+# This function is called by the downloader in all adapter_*.py files
+# in this dir to register the adapter class.  So it needs to be
+# updated to reflect the class below it.  That, plus getSiteDomain()
+# take care of 'Registering'.
+def getClass():
+    return TheHookupZoneNetAdapter # XXX
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class TheHookupZoneNetAdapter(BaseSiteAdapter): # XXX
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
+        self._setURL('http://' + self.getSiteDomain() + '/CriminalMinds/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','thupz') # XXX
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%b %d, %Y" # XXX
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'thehookupzone.net' # XXX
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/CriminalMinds/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/CriminalMinds/viewstory.php?sid=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Registered Users Only' in data \
+                or 'There is no such account on our website' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['cookiecheck'] = '1'
+        params['submit'] = 'Submit'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/CriminalMinds/user.php?action=login'
+        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "Member Account" not in d : #Member Account
+            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=4" # XXX
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+
+        # The actual text that is used to announce you need to be an
+        # adult varies from site to site.  Again, print data before
+        # the title search to troubleshoot.
+        if "Age Consent Required" in data: # XXX 
+            raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/CriminalMinds/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                catstext = [cat.string for cat in cats]
+                for cat in catstext:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                charstext = [char.string for char in chars]
+                for char in charstext:
+                    self.story.addToList('characters',char.string)
+
+            ## Not all sites use Genre, but there's no harm to
+            ## leaving it in.  Check to make sure the type_id number
+            ## is correct, though--it's site specific.
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
+                genrestext = [genre.string for genre in genres]
+                self.genre = ', '.join(genrestext)
+                for genre in genrestext:
+                    self.story.addToList('genre',genre.string)
+
+            ## Not all sites use Warnings, but there's no harm to
+            ## leaving it in.  Check to make sure the type_id number
+            ## is correct, though--it's site specific.
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
+                warningstext = [warning.string for warning in warnings]
+                self.warning = ', '.join(warningstext)
+                for warning in warningstext:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/CriminalMinds/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -0,0 +1,271 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return TheMasqueNetAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class TheMasqueNetAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        if self.parsedUrl.path.split('/',)[1] == 'wiktt':
+            self.story.addToList("category","Harry Potter")
+            self.story.setMetadata('section','/wiktt/efiction/')
+            self.dateformat = "%m/%d/%Y"
+        else:
+            self.story.addToList("category","Originals")
+            self.story.setMetadata('section','/efiction/')
+            self.dateformat = "%b %d, %Y"
+            
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + self.story.getMetadata('section') + 'viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','msq')
+        
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'themasque.net'
+        
+    def getSiteExampleURLs(self):
+        return "http://themasque.net/wiktt/efiction/viewstory.php?sid=1234 http://themasque.net/efiction/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain())+"(/wiktt)?/efiction"+re.escape("/viewstory.php?sid=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Registered Users Only' in data \
+                or 'There is no such account on our website' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['cookiecheck'] = '1'
+        params['submit'] = 'Submit'
+    
+        loginUrl = 'http://' + self.getSiteDomain()  + self.story.getMetadata('section') + 'user.php?action=login'
+        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "Member Account" not in d : #Member Account
+            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=4"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&amp;ageconsent=ok)?&amp;warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid &amp; error in url.
+                addurl = addurl.replace("&amp;","&")
+                url = self.url+'&index=1'+addurl
+                logging.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host + self.story.getMetadata('section') + chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+                
+# summary, rated, word count, categories, characters, genre, warnings, completed, published, updated, seires
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.text
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -0,0 +1,239 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return ThePetulantPoetessComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class ThePetulantPoetessComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId') +'&i=1')
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','tpp')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%Y/%m/%d"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.thepetulantpoetess.com'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'You must be a member to read this story.' in data \
+                or "The Ministry of Magic does not have a record of that password. " in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['cookiecheck'] = '1'
+        params['submit'] = 'Submit'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
+        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "My Account Page" not in d : #Member Account
+            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        chapters=soup.find('select', {'name' : 'sid'})
+        if chapters == None:
+            self.chapterUrls.append((self.story.getMetadata('title'),url))
+        else:
+            for chapter in chapters.findAll('option', value=re.compile(r"viewstory.php\?sid=\d+&i=1")):
+                # just in case there's tags, like <i> in chapter titles.
+                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['value']))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+        
+        # make sure that the story id is from the first chapter
+        self.story.setMetadata('storyId',self.chapterUrls[0][1].split('=')[1].split('&')[0])
+        
+        #locate the story on author's page
+        index = 1
+        found = 0
+        while found == 0:
+            asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')+"&page="+str(index)))
+        
+            for info in asoup.findAll('td', {'class' : 'highlightcolor1'}):
+                a = info.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+                if a != None:
+                    self.story.setMetadata('title',a.text)                
+                    found = 1
+                    break
+            index=index+1
+            
+        # extract metadata
+        b=info.find('b')
+        b.find('a').extract()
+        self.story.setMetadata('rating', b.text.split('[')[1].split(']')[0])
+        
+        info = info.findNext('td', {'colspan' : '2'}) 
+        for label in info.findAll('b'):
+            value = label.nextSibling
+            
+            if "Category" in label.text:
+                for cat in info.findAll('a'):
+                    self.story.addToList('category',cat.string)
+            
+            if "Characters" in label.text:
+                for char in value.split(', '):
+                    self.story.addToList('characters',char)
+                    
+            if "Genres" in label.text:
+                for genre in value.split(', '):
+                    if "General" not in genre:
+                        self.story.addToList('genre',genre)
+                    
+            if "Warnings" in label.text:
+                for warning in value.split(', '):
+                    if "none" not in warning.lower():
+                        self.story.addToList('warnings',warning)
+                        
+                        
+        info = info.findNext('td', {'class' : 'tblborder'}) 
+        info.find('b').extract()
+        self.setDescription(url,info)
+        
+        info = info.findNext('td', {'class' : 'highlightcolor2'}) 
+        for label in info.findAll('b'):
+            value = label.nextSibling
+            
+            if "Completed" in label.text:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+                    
+            if 'Published' in label.text:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label.text:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+                
+            if 'Word Count' in label.text:
+                self.story.setMetadata('numWords', value)
+            
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.findAll('table')[2].findAll('td')[1]
+        for a in div.findAll('div'):
+            a.extract()
+        div.name='div'
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -60,11 +60,6 @@ class TheQuidditchPitchOrgAdapter(BaseSiteAdapter): # XXX
        # Each adapter needs to have a unique site abbreviation.
        self.story.setMetadata('siteabbrev','tqdpch') # XXX

-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Harry Potter") # XXX
-
        # The date format will vary from site to site.
        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
        self.dateformat = "%m/%d/%Y" # XXX
@@ -133,7 +133,7 @@ class TheWritersCoffeeShopComSiteAdapter(BaseSiteAdapter):

        ## Title
        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -165,7 +165,13 @@ class TheWritersCoffeeShopComSiteAdapter(BaseSiteAdapter):
                svalue = ""
                while not defaultGetattr(value,'class') == 'label':
                    svalue += str(value)
-                    value = value.nextSibling
+                    # poor HTML(unclosed <p> for one) can cause run on
+                    # over the next label.
+                    if '<span class="label">' in svalue:
+                        svalue = svalue[0:svalue.find('<span class="label">')]
+                        break
+                    else:
+                        value = value.nextSibling
                self.setDescription(url,svalue)

            if 'Rated' in label:
@@ -151,12 +151,33 @@ class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter):
        self.story.setMetadata('authorId',a['href'].split('/')[1].split('-')[1])
        self.story.setMetadata('authorUrl','http://'+self.host+a['href'])
        self.story.setMetadata('author',stripHTML(a))
+        authorurl = 'http://'+self.host+a['href']

+        ainfo = soup.find('a', href='/StoryInfo-%s-1'%self.story.getMetadata('storyId'))
+        if ainfo != None: # indicates multiple authors/contributors.
+            try:
+                # going to pull part of the meta data from author list page.
+                infourl = 'http://'+self.host+ainfo['href']
+                logging.debug("**StoryInfo** URL: "+infourl)
+                infodata = self._fetchUrl(infourl)
+                infosoup = bs.BeautifulSoup(infodata)
+
+                for a in infosoup.findAll('a',href=re.compile(r"^/Author-\d+")):
+                    self.story.addToList('authorId',a['href'].split('/')[1].split('-')[1])
+                    self.story.addToList('authorUrl','http://'+self.host+a['href'].replace("/Author-","/AuthorStories-"))
+                    self.story.addToList('author',stripHTML(a))                
+                                            
+            except urllib2.HTTPError, e:
+                if e.code == 404:
+                    raise exceptions.StoryDoesNotExist(url)
+                else:
+                    raise e
+    
        try:
-            # going to pull part of the meta data from author list page.
-            logging.debug("**AUTHOR** URL: "+self.story.getMetadata('authorUrl'))
-            authordata = self._fetchUrl(self.story.getMetadata('authorUrl'))
-            descurl=self.story.getMetadata('authorUrl')
+            # going to pull part of the meta data from *primary* author list page.
+            logging.debug("**AUTHOR** URL: "+authorurl)
+            authordata = self._fetchUrl(authorurl)
+            descurl=authorurl
            authorsoup = bs.BeautifulSoup(authordata)
            # author can have several pages, scan until we find it.
            while( not authorsoup.find('a', href=re.compile(r"^/Story-"+self.story.getMetadata('storyId'))) ):
@@ -209,7 +230,7 @@ class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter):
                if not BtVSNonX:
                    BtVS = False # Don't add BtVS if Non-Crossover, unless it's a BtVS/AtS Non-Crossover

-        print("BtVS: %s BtVSNonX: %s"%(BtVS,BtVSNonX))
+        #print("BtVS: %s BtVSNonX: %s"%(BtVS,BtVSNonX))
        if BtVS:
            self.story.addToList('category','Buffy: The Vampire Slayer')
            
@@ -0,0 +1,185 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return TwilightArchivesComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class TwilightArchivesComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL. http://www.twilightarchives.com/read/9353
+        self._setURL('http://' + self.getSiteDomain() + '/read/'+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','twa')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%d %b %y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.twilightarchives.com'
+       
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/read/1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://" + self.getSiteDomain()+"/read/")+r"\d+(/d+)?$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('h1')
+        self.story.setMetadata('title',a.string)
+        
+        # Find the chapters:
+        chapters=soup.find('ol', {'class' : 'chapters'})
+        if chapters != None:
+            for chapter in chapters.findAll('a', href=re.compile(r'/read/'+self.story.getMetadata('storyId')+"/\d+$")):
+                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+chapter['href']))
+        else:
+            self.chapterUrls.append((self.story.getMetadata('title'),url))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+                
+# rated, genre, warnings, seires
+
+        summary = soup.find('p', {'class' : 'images'})
+        self.setDescription(url,summary)
+
+        for c in soup.findAll('h2', {'class' : 'title'}):
+            div = c.nextSibling.nextSibling
+            
+            if 'Information' in c.text:
+                for dt in div.findAll('dt'):
+                    dd=dt.nextSibling.nextSibling
+                    
+                    if 'Author' in dt.text:
+                        a=dd.find('a')
+                        self.story.setMetadata('authorId',a['href'].split('/')[2])
+                        self.story.setMetadata('authorUrl','http://'+self.host+a['href'])
+                        self.story.setMetadata('author',a.text)
+                        
+                    if 'Words' in dt.text:
+                        self.story.setMetadata('numWords', dd.text)
+                        
+                    if 'Published' in dt.text:
+                        self.story.setMetadata('datePublished', makeDate(stripHTML(dd.text), self.dateformat))
+            
+                    if 'Updated' in dt.text:
+                        self.story.setMetadata('dateUpdated', makeDate(stripHTML(dd.text), self.dateformat))
+                    
+                    if 'Status' in dt.text:
+                        if 'Complete' in dd.text:
+                            self.story.setMetadata('status', 'Completed')
+                        else:
+                            self.story.setMetadata('status', 'In-Progress')
+
+            if 'Categories' in c.text:
+                for a in div.findAll('a'):
+                    self.story.addToList('category',a.text)
+                    
+            if 'Characters' in c.text:
+                for a in div.findAll('a'):
+                    self.story.addToList('category',a.text)
+
+            if 'Series' in c.text:
+                a=div.find('a')
+                series_name = a.text
+                series_url = 'http://'+self.host+a['href']
+                
+                seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+                storyas = seriessoup.find('tbody').findAll('a', href=re.compile(r'^/read/\d+$'))
+                i=1
+                for a in storyas:
+                    if a['href'] == ('/read/'+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        break
+                    i+=1
+                    
+        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
+        a=asoup.find('tbody').find('a', href=re.compile(r'^/read/'+self.story.getMetadata('storyId')))
+        self.story.setMetadata('rating',a.parent.nextSibling.nextSibling.text)
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'class' : 'size images medium'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -37,7 +37,6 @@ class TwilightedNetSiteAdapter(BaseSiteAdapter):
                               # Most sites that claim to be
                               # iso-8859-1 (and some that claim to be
                               # utf8) are really windows-1252.
-        self.story.addToList("category","Twilight")
        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
        self.password = ""
        
@@ -37,7 +37,6 @@ class TwiwriteNetSiteAdapter(BaseSiteAdapter):
                               # Most sites that claim to be
                               # iso-8859-1 (and some that claim to be
                               # utf8) are really windows-1252.
-        self.story.addToList("category","Twilight")
        self.is_adult = False
        self.username = "NoneGiven" # if left empty, twiwrite.net doesn't return any message at all.
        self.password = ""
@@ -0,0 +1,228 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return WalkingThePlankOrgAdapter
+
+class WalkingThePlankOrgAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/archive/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','wtp')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%b %d, %Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.walkingtheplank.org'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/archive/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/archive/viewstory.php?sid=")+r"\d+$"
+
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&warning=4"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+
+        # The actual text that is used to announce you need to be an
+        # adult varies from site to site.  Again, print data before
+        # the title search to troubleshoot.
+    
+        if "By clicking this link, you acknowledge" in data:
+            raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/archive/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+				
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                catstext = [cat.string for cat in cats]
+                for cat in catstext:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                charstext = [char.string for char in chars]
+                for char in charstext:
+                    self.story.addToList('characters',char.string)
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/archive/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -0,0 +1,299 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return WizardTalesNetAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class WizardTalesNetAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','wzt')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%m/%d/%Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.wizardtales.net'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Registered Users Only' in data \
+                or 'There is no such account on our website' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['cookiecheck'] = '1'
+        params['submit'] = 'Submit'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
+        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "Member Account" not in d : #Member Account
+            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=4"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&amp;ageconsent=ok)?&amp;warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid &amp; error in url.
+                addurl = addurl.replace("&amp;","&")
+                url = self.url+'&index=1'+addurl
+                logging.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        pt = soup.find('div', {'id' : 'pagetitle'})
+        a = pt.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.text)
+        
+        # Find authorid and URL from... author url.
+        a = pt.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        rating=pt.text.split('[')[1].split(']')[0]
+        self.story.setMetadata('rating', rating)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        content=soup.find('div',{'class' : 'content'})
+        
+        for genre in content.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')):
+            self.story.addToList('genre',genre.string)
+
+        for warning in content.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')):
+            self.story.addToList('warnings',warning.string)
+        
+        labels = content.findAll('b')
+        
+        value = labels[0].previousSibling
+        svalue = ""
+        while value != None:
+            val = value
+            value = value.previousSibling
+        while "Categories" not in val:
+            svalue += str(val)
+            val = val.nextSibling
+        self.setDescription(url,svalue)
+        
+        
+        
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', stripHTML(value).split(': ')[1].split(';')[0])
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value).split(': ')[1].split(';')[0], self.dateformat))
+            
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value).split(': ')[1], self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            i=1
+            for a in storyas:
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        break
+                    i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.findAll('div', {'id' : 'story'})[1]
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -0,0 +1,216 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return WolverineAndRogueComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class WolverineAndRogueComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/wrfa/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','wrfa')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%m/%d/%Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.wolverineandrogue.com'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/wrfa/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/wrfa/viewstory.php?sid=")+r"\d+$"
+
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        pt = soup.find('div', {'id' : 'pagetitle'})
+        a = pt.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.text)
+        
+        # Find authorid and URL from... author url.
+        a = pt.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/wrfa/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        rating=pt.text.split('(')[1].split(')')[0]
+        self.story.setMetadata('rating', rating)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/wrfa/'+chapter['href']))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        content=soup.find('div',{'class' : 'content'})
+        labels = soup.findAll('span',{'class':'label'})
+        
+        value = labels[0].previousSibling
+        svalue = ""
+        while value != None:
+            val = value
+            value = value.previousSibling
+        while "Categories" not in str(val):
+            svalue += str(val)
+            val = val.nextSibling
+        self.setDescription(url,svalue)
+        
+        
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value.split(' -')[0])
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Complete' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(value.split(' -')[0], self.dateformat))
+            
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/wrfa/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            i=1
+            for a in storyas:
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        break
+                    i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -0,0 +1,222 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+
+def getClass():
+    return WraithBaitComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class WraithBaitComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','wb')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%d  %b %Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.wraithbait.com'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=12"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+				
+        if "for adults only" in data:
+            raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        pt = soup.find('div', {'id' : 'pagetitle'})
+        a = pt.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        alist = pt.findAll('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        for a in alist:
+            self.story.addToList('authorId',a['href'].split('=')[1])
+            self.story.addToList('authorUrl','http://'+self.host+'/'+a['href'])
+            self.story.addToList('author',a.string)
+
+        rating=pt.text.split('[')[1].split(']')[0]
+        self.story.setMetadata('rating', rating)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+		
+        info = soup.find('div', {'class' : 'small'})
+		
+        word=info.find(text=re.compile("Word count:")).split(':')
+        self.story.setMetadata('numWords', word[1])
+		
+        cats = info.findAll('a',href=re.compile(r'browse.php\?type=categories&id=\d'))
+        for cat in cats:
+            if "General" != cat.string:
+                self.story.addToList('category',cat.string)
+				
+        chars = info.findAll('a',href=re.compile(r'browse.php\?type=characters&charid=\d'))
+        for char in chars:
+            self.story.addToList('characters',char.string)
+			
+        completed=info.find(text=re.compile("Completed: Yes"))
+        if completed != None:
+            self.story.setMetadata('status', 'Completed')
+        else:
+            self.story.setMetadata('status', 'In-Progress')
+			
+        date=soup.find('div',{'class' : 'bottom'})
+        pd=date.find(text=re.compile("Published:")).string.split(': ')
+        self.story.setMetadata('datePublished', makeDate(stripHTML(pd[1].split(' U')[0]), self.dateformat))
+        self.story.setMetadata('dateUpdated', makeDate(stripHTML(pd[2]), self.dateformat))
+		
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        pub=0
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Genres' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+			
+        info.extract()
+        summary = soup.find('div', {'class' : 'content'})
+        self.setDescription(url,summary)
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulSoup(self._fetchUrl(url))
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
+
@@ -0,0 +1,283 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return YourFanfictionComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class YourFanfictionComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        # yourfanfiction.com blocks the default user-agent.  However,
+        # when asked, they said it was just general anti-spam, not
+        # targeted as us and offered to 'whitelist our IP'.  Clearly,
+        # that wouldn't work, but it does let me do this in good
+        # conscience:
+        self.opener.addheaders = [('User-agent', 'FFDL/1.6')]
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','yff')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%d %b %Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.yourfanfiction.com'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=4"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        # The actual text that is used to announce you need to be an
+        # adult varies from site to site.  Again, print data before
+        # the title search to troubleshoot.
+            
+        # Since the warning text can change by warning level, let's
+        # look for the warning pass url.  ksarchive uses
+        # &amp;warning= -- actually, so do other sites.  Must be an
+        # eFiction book.
+            
+        # viewstory.php?sid=1882&amp;warning=4
+        # viewstory.php?sid=1654&amp;ageconsent=ok&amp;warning=5
+        #print data
+        #m = re.search(r"'viewstory.php\?sid=1882(&amp;warning=4)'",data)
+        m = re.search(r"'viewstory.php\?sid=\d+((&amp;ageconsent=ok)?&amp;warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid &amp; error in url.
+                # explicitly put ageconsent because google appengine regexp doesn't include it for some reason.
+                addurl = addurl.replace("&amp;","&")+'&ageconsent=ok'
+                url = self.url+'&index=1'+addurl
+                logging.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # because for some reason, this works while simple 'print data' errors on ascii conversion.
+        # loopdata = data
+        # chklen=5000
+        # while len(loopdata) > 0:
+        #     if len(loopdata) < 5000:
+        #         chklen = len(loopdata)
+        #     logging.info("loopdata: %s" % loopdata[:chklen])
+        #     loopdata = loopdata[chklen:]
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while value and not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                # sometimes poorly formated desc (<p> w/o </p>) leads
+                # to all labels being included.
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=5'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+                    
+            if 'Tags' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=7'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=6'))
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            i=1
+            for a in storyas:
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        break
+                    i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -67,12 +67,8 @@ class BaseSiteAdapter(Configurable):
    def validateURL(self):
        return re.match(self.getSiteURLPattern(), self.url)

-    def __init__(self, config, url):
-        self.config = config
-        Configurable.__init__(self, config)
-        self.setSectionOrder(self.getSiteDomain())
-        # self.addConfigSection(self.getSiteDomain())
-        # self.addConfigSection("overrides")
+    def __init__(self, configuration, url):
+        Configurable.__init__(self, configuration)
        
        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
        self.password = ""
@@ -81,14 +77,17 @@ class BaseSiteAdapter(Configurable):
        self.opener = u2.build_opener(u2.HTTPCookieProcessor(),GZipProcessor())
        self.storyDone = False
        self.metadataDone = False
-        self.story = Story()
-        self.story.setMetadata('site',self.getSiteDomain())
+        self.story = Story(configuration)
+        self.story.setMetadata('site',self.getConfigSection())
        self.story.setMetadata('dateCreated',datetime.datetime.now())
        self.chapterUrls = [] # tuples of (chapter title,chapter url)
        self.chapterFirst = None
        self.chapterLast = None
        self.oldchapters = None
        self.oldimgs = None
+        self.oldcover = None # (data of existing cover html, data of existing cover image)
+        self.calibrebookmark = None
+        self.logfile = None
        ## order of preference for decoding.
        self.decode = ["utf8",
                       "Windows-1252"] # 1252 is a superset of
@@ -134,6 +133,7 @@ class BaseSiteAdapter(Configurable):
                        code=detected['encoding']
                    else:
                        continue
+                logging.debug("try code:"+code)
                return data.decode(code)
            except:
                logging.debug("code failed:"+code)
@@ -159,9 +159,9 @@ class BaseSiteAdapter(Configurable):

    def _fetchUrlRaw(self, url, parameters=None):
        if parameters != None:
-            return self.opener.open(url,urllib.urlencode(parameters)).read()
+            return self.opener.open(url.replace(' ','%20'),urllib.urlencode(parameters)).read()
        else:
-            return self.opener.open(url).read()
+            return self.opener.open(url.replace(' ','%20')).read()
    
    # parameters is a dict()
    def _fetchUrl(self, url, parameters=None):
@@ -213,18 +213,32 @@ class BaseSiteAdapter(Configurable):
            if self.getConfig('include_images') and \
                    not self.story.cover and \
                    self.getConfig('default_cover_image'):
-                self.story.addImgUrl(self,
-                                     None,
+                self.story.addImgUrl(None,
                                     #self.getConfig('default_cover_image'),
                                     self.story.formatFileName(self.getConfig('default_cover_image'),
                                                               self.getConfig('allow_unsafe_filename')),
                                     self._fetchUrlRaw,
                                     cover=True)
+
+            # no new cover, set old cover, if there is one.
+            if not self.story.cover and self.oldcover:
+                self.story.oldcover = self.oldcover
+                
+            # cheesy way to carry calibre bookmark file forward across update.
+            if self.calibrebookmark:
+                self.story.calibrebookmark = self.calibrebookmark
+            if self.logfile:
+                self.story.logfile = self.logfile
+                
        return self.story

    def getStoryMetadataOnly(self):
        if not self.metadataDone:
            self.extractChapterUrlsAndMetadata()
+            
+            if not self.story.getMetadataRaw('dateUpdated'):
+                self.story.setMetadata('dateUpdated',self.story.getMetadataRaw('datePublished'))
+
            self.metadataDone = True
        return self.story

@@ -235,6 +249,11 @@ class BaseSiteAdapter(Configurable):
        "Needs to be overriden in each adapter class."
        return 'no such domain'
    
+    @classmethod
+    def getConfigSection(cls):
+        "Only needs to be overriden if != site domain."
+        return cls.getSiteDomain()
+    
    ## URL pattern validation is done *after* picking an adaptor based
    ## on domain instead of *as* the adaptor selector so we can offer
    ## the user example(s) for that particular site.
@@ -262,18 +281,22 @@ class BaseSiteAdapter(Configurable):
    # Just for series, in case we choose to change how it's stored or represented later.
    def setSeries(self,name,num):
        if self.getConfig('collect_series'):
-            self.story.setMetadata('series','%s [%s]'%(name, num))
+            self.story.setMetadata('series','%s [%s]'%(name, int(num)))

    def setDescription(self,url,svalue):
        #print("\n\nsvalue:\n%s\n"%svalue)
        if self.getConfig('keep_summary_html'):
-            if isinstance(svalue,str) or isinstance(svalue,unicode):
+            if isinstance(svalue,basestring):
                svalue = bs.BeautifulSoup(svalue)
            self.story.setMetadata('description',self.utf8FromSoup(url,svalue))
        else:
            self.story.setMetadata('description',stripHTML(svalue))
        #print("\n\ndescription:\n"+self.story.getMetadata('description')+"\n\n")

+    def setCoverImage(self,storyurl,imgurl):
+        if self.getConfig('include_images'):
+            self.story.addImgUrl(storyurl,imgurl,self._fetchUrlRaw,cover=True)
+
    # This gives us a unicode object, not just a string containing bytes.
    # (I gave soup a unicode string, you'd think it could give it back...)
    # Now also does a bunch of other common processing for us.
@@ -289,7 +312,7 @@ class BaseSiteAdapter(Configurable):
                # some pre-existing epubs have img tags that had src stripped off.
                if img.has_key('src'):
                    img['longdesc']=img['src']
-                    img['src']=self.story.addImgUrl(self,url,img['src'],fetch,
+                    img['src']=self.story.addImgUrl(url,img['src'],fetch,
                                                    coverexclusion=self.getConfig('cover_exclusion_regexp'))

        for attr in soup._getAttrMap().keys():
@@ -0,0 +1,325 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+#
+# Author:        Pau Sanchez (contact@pausanchez.com)
+# Version:       v1.0
+# Last Modified: 2010/09/15
+# 
+# For the latest version check out:
+#   http://www.codigomanso.com/en/projects
+# 
+# My blog:
+#   http://www.codigomanso.com/en/  - English Version
+#   http://www.codigomanso.com/es/  - Spanish Version
+#
+
+import sys
+import os
+import re
+import urllib
+
+class bbcode2html:
+  '''
+  This class gets a parsed BBCode and transforms it to valid HTML
+
+  Useful functions of this class:
+    html
+    convertToHTML
+
+  Example:
+    > parser = bbcodeparser ()
+    > parser.parse ('[b]bold[/b]')
+    > bbcode2html (parser).html()
+    <b>bold</b>
+
+    # This is faster for huge strings but changes the parser object internally
+    > bbcode2html (parser).html(doDeepCopy = False)
+    <b>bold</b>
+  '''
+  def __init__ (self, parser):
+    self._parser = parser
+    return
+
+  def html (self, allowClassAttr = False, doDeepCopy = True, parser = None):
+    '''
+    Convert current parsed code to HTML
+
+    Example:
+      code = bbcodeparser ('[b]bold[/b]')
+      code.html() -> '<b>bold</b>'
+    '''
+    if parser is None:
+      parser = self._parser
+
+    tokens = parser
+    if instanceof (parser, bbcodeparser):
+      tokens = parser.getTokens()
+
+    return bbcode2html.convertToHTML (tokens, allowClassAttr = allowClassAttr, doDeepCopy = doDeepCopy)
+
+  @staticmethod
+  def htmlString (string):
+    toReplace = {
+      u'<' : '&lt;',
+      u'>' : '&gt;',
+      u'"' : "&quot;",
+      u'&' : "&amp;"
+    }
+    for entity in toReplace:
+      string = string.replace(entity, toReplace[entity])
+    return string
+
+  @staticmethod
+  def getValidTags ():
+    simpleTags = ['b', 'u', 'i', 'sup', 'sub', 'ul', 'ol', 'li', 'table', 'tr', 'th', 'td', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6']
+    validTags = {
+      'p'         : { 'color' : 'color', 'size' : 'size', 'font' : 'font' },
+      'color'     : { 'color' : 'color' },
+      'size'      : { 'size' : 'size' },
+      'font'      : { 'font' : 'font' },
+      'img'       : { 'alt' : 'alt', 'title' : 'title', 'width' : 'width' , 'height' : 'height', 'img' : 'img'},
+      'url'       : { 'href' : 'href', 'url' : 'href', 'link'  : 'href', 'title' : 'title' },
+      's'         : { },
+      'code'      : { },
+      'quote'     : { },
+      'list'      : { 'list' : 'type' },
+      'email'     : { 'email': 'href'},
+      'google'    : { 'google':  'google'},
+      'wikipedia' : { 'wikipedia' : 'wikipedia', 'language' : 'language', 'lang' : 'lang'}
+    }
+
+    for tag in simpleTags:
+      validTags[tag] = { }
+    return validTags
+
+  @staticmethod
+  def convertToHTML (tokens, allowClassAttr = False, validTags = None, doDeepCopy = True):
+    '''
+    Convert internally parsed BBCode to XHTML
+
+    @doDeepCopy 
+      True:  it does a deep copy of tokens so this list will remain unchanged
+      False: tokens will be modified internally, but the output will be produced like 5x faster
+             it's a good idea to use False only when this is the last operation
+    '''
+    # do a deep copy
+    if doDeepCopy:
+      import copy
+      tokens = copy.deepcopy (tokens)
+
+    # filter invalid tags and attributes
+    if validTags is None:
+      validTags = bbcode2html.getValidTags()    
+
+    bbcode2html._filterInvalidTagsAndAttributes (tokens, validTags, allowClassAttr)
+    
+    # Start to convert
+    index       = 0
+    tokenLength = len (tokens)
+
+    # use a list for the output (an order of magnitude faster than using string concatenation)
+    htmlList = []
+    lastListOpener = []
+
+    while index < tokenLength:
+
+      if isinstance (tokens [index], basestring):
+        htmlList.append (bbcode2html.htmlString (tokens [index]))
+        index += 1
+        continue
+
+      token     = tokens[index] 
+      tag       = token['tag']  # opening or closing simple tag. e.g: 'b', '/b', '/u', ...
+      tagName   = (tag[1:] if tag[0] == '/' else tag)
+      tagOpener = (u'/' if tag[0] == '/' else u'')
+      tokenArgs = (token['args'] if 'args' in token else {})
+
+      # opening or closing simple tag COLOR / SIZE
+      if (tagName in ['p', 'color', 'size', 'font']): 
+        style  = ''
+        style += ((u' color: ' + tokenArgs['color'] + u';') if ('color' in tokenArgs) else '')
+        style += ((u' font-size: ' + tokenArgs['size'] + u'pt;') if ('size' in tokenArgs) else '')
+        style += ((u' font-family: ' + tokenArgs['font'] + u';') if ('font' in tokenArgs) else '')
+        style  = style.strip()
+
+        pArgs = {}
+        if style != '':
+          pArgs ['style'] = style
+
+        if 'class' in tokenArgs:
+          pArgs ['class'] = tokenArgs['class']
+
+        if ('args' not in token) and (tagName != 'p'):
+          if (tagOpener == '/'): # if closing tag, close it
+            htmlList.append (u'</span>')
+          index += 1
+          continue
+
+        if tagName != 'p':
+          tag = tagOpener + u'span'
+
+        htmlList.append (bbcode2html.xml (tag, pArgs))
+
+      # IMG tag
+      elif tag == 'img' and (index+2 < tokenLength):
+        if 'img' in tokenArgs:
+          # has the form of <width>x<height> ?
+          sizeMatch = re.match (u'^\s*(\d+)[xX](\d+)\s*$', tokenArgs['img'])
+          if sizeMatch is not None:
+            tokenArgs['width']  = sizeMatch.group(1)
+            tokenArgs['height'] = sizeMatch.group(2)
+          # then assume is the alternative text
+          else:
+            tokenArgs['alt'] = tokenArgs['img']
+          del tokenArgs['img']
+
+        # add the source of the image
+        tokenArgs ['src'] = tokens[index+1]
+
+        # [img]http://www.whatever.com/pic.jpg[/img]
+        htmlList.append (
+          bbcode2html.xml ('img', tokenArgs, soloTag=True)
+        )
+        index += 2 # skip next token and closing tag
+
+      # URL tag
+      elif tag == 'url':
+        if ('args' not in token) and (index+2 < tokenLength):
+          # [url]http://www.google.com[/url]
+          htmlList.append (bbcode2html.xml ('a', { 'href' : tokens[index+1] }))
+        else:
+          # [url=http://www.google.com]Google[/url]
+          # [url link=http://www.google.com title="This is Google"]Google[/url]
+          htmlList.append (bbcode2html.xml ('a', tokenArgs))
+  
+      # URL closing tag (sometimes needed)
+      elif (tag == '/url') or (tag == '/email'):
+        htmlList.append (u'</a>')
+
+      # Email tag
+      elif tag == 'email':
+        if ('args' not in token) and (index+2 < tokenLength):
+          # [email]asdf@asdf.com]
+          htmlList.append (bbcode2html.xml ('a', { 'href' : u'mailto:' + tokens[index+1].strip() }))
+        else:
+          # [email=asdf@asfd.com]john smith[/email]
+          if 'href' in tokenArgs:
+            tokenArgs['href'] = u'mailto:' + tokenArgs['href']
+          htmlList.append (bbcode2html.xml ('a', tokenArgs))
+  
+      elif tagName == 'list':
+        if tagOpener == '/':
+          htmlList.append (bbcode2html.xml (u'/' + lastListOpener.pop()))
+        else:
+          if ('type' not in tokenArgs):
+            htmlList.append (bbcode2html.xml (tagOpener + u'ul', tokenArgs))
+            lastListOpener.append ('ul')
+          else:
+            htmlList.append (bbcode2html.xml (tagOpener + u'ol', tokenArgs))
+            lastListOpener.append ('ol')
+
+      elif tagName == '*':
+        htmlList.append (bbcode2html.xml (tagOpener + u'li', tokenArgs))
+
+      elif (tagName == 's'):
+        tokenArgs['style'] =  'text-decoration: line-through;'
+        htmlList.append (bbcode2html.xml (tagOpener + u'span', tokenArgs))
+
+      elif (tagName == 'code'):
+        htmlList.append (bbcode2html.xml (tagOpener + u'pre', tokenArgs))
+
+      elif (tagName == 'quote'):
+        htmlList.append (bbcode2html.xml (tagOpener + u'blockquote', tokenArgs))
+
+      elif (tagName == 'google'):
+        htmlList.append (
+          bbcode2html.xml (
+            tagOpener + u'a', 
+            {'href' : 'http://www.google.com/search?q=' + urllib.quote_plus (tokens[index+1])},
+            tokens[index+1]
+          )
+        )
+        index += 2
+
+      elif (tagName == 'wikipedia'):
+        subdomain = 'www'
+        for arg  in ['lang', 'language', 'wikipedia']:
+          if arg in tokenArgs:
+            subdomain = tokenArgs[arg] 
+
+        htmlList.append (
+          bbcode2html.xml (
+            tagOpener + u'a', 
+            {'href' : 'http://' + subdomain + '.wikipedia.org/wiki/' + tokens[index+1].replace (' ', '_')},
+            tokens[index+1]
+          )
+        )
+        index += 2
+
+      elif (tagName in validTags):
+        htmlList.append (
+          bbcode2html.xml (tag, tokenArgs)
+        )
+
+      else:
+        # ignore this tag
+        pass 
+
+      index += 1
+
+    return ''.join (htmlList)
+
+  @staticmethod
+  def _filterInvalidTagsAndAttributes (tokens, validTags, allowClassAttr):
+    '''
+    Helper function to filter out invalid attributes from the tokens list
+    '''
+    # add 'class' attribute as valid (mapping 'class' itself)
+    if allowClassAttr:
+      for attr in validTags:
+        validTags[attr]['class'] = 'class'
+
+    # remove invalid attributes from tokens
+    for tindex in range(0, len(tokens)):
+      if isinstance (tokens[tindex], dict) and ('args' in tokens[tindex]) and (tokens[tindex]['tag'] in validTags):
+        validList = validTags[tokens[tindex]['tag']]
+
+        filteredArgs = {}
+        for arg in tokens[tindex]['args']:
+          if arg in validList:
+            # rename the argument
+            filteredArgs[validList[arg]] = tokens[tindex]['args'][arg]
+          else:
+            pass # do not include this arg in the filteredArgs
+
+        tokens[tindex]['args'] = filteredArgs
+
+    return
+
+  @staticmethod
+  def xml (tag, attrs = {}, text = None, soloTag = False):
+    '''
+    Helper function to produce valid XML output
+    '''
+    xml = u'<' + tag.lower()
+
+    # make sure we sort attributes alphabetically (for deterministic output)
+    # Faster but non-deterministic:
+    #  for (key, value) in attrs.iteritems():
+    #    xml += u' ' + key + u'="' + value + u'"'
+    for key in sorted (attrs.keys()):
+      xml += u' ' + key + u'="' + attrs[key] + u'"'
+      
+    # close tag
+    if text is None:
+      if soloTag:
+        xml += u' />'
+      else:
+        xml += u'>'
+    else:
+      xml += u'>' + text + u'</' + tag.lower() + '>'
+
+    return xml
+
+
+
@@ -0,0 +1,77 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+#
+# Author:        Pau Sanchez (contact@pausanchez.com)
+# Version:       v1.0
+# Last Modified: 2010/09/15
+# 
+# For the latest version check out:
+#   http://www.codigomanso.com/en/projects
+# 
+# My blog:
+#   http://www.codigomanso.com/en/  - English Version
+#   http://www.codigomanso.com/es/  - Spanish Version
+#
+
+import sys
+import os
+import re
+import hashlib
+
+class bbcodebuilder:
+  '''
+  This class helps to build BBCode programmatically.
+
+  The function names are used as the tag name, then the first parameter
+  is the string that goes inside the tags and any extra parameter is 
+  appended as a parameter to the tag
+
+  Examples:
+    > bbcode = bbcodebuilder()  # create a instance!
+
+    > print bbcode.b ('bold')
+    [b]bold[/b]
+
+    > print bbcode.color ('this goes in red', 'red')
+    [color=red]this goes in red[/color]
+
+    > print bbcode.url ('Google', 'http://www.google.com')
+    [url=http://www.google.com]Google[/url]
+    
+    > print bbcode.alist('item 1', 'item 2')
+    [list=a]
+      [*]item 1
+      [*]item 2
+    [/list]
+
+
+  This solution is based on the recipe found on:
+    http://code.activestate.com/recipes/576831-simple-bbcode-support/
+  '''
+
+  def __getattr__(self, name):
+    '''
+    This is a generic getter that returns a function which gets the first parameter
+    as the string that goes between the tags, and extra parameters as tag parameters.
+
+    The name of the attribute is used as the tag name
+    '''
+    class bbcodebuilder_helper:
+      def __init__(self, name):
+        self._name = name
+
+      def __call__(self, string, *args):
+        return u'[{0}{1}]{2}[/{0}]'.format(self._name, (u'=' + u','.join(map(str, args))) if args else u'', string)
+    
+    return bbcodebuilder_helper (name)
+
+  def list(self, *items):
+    return u'[list]' + u''.join(map(lambda item: u"\n  [*]" + item, items)) + u"\n[/list]"
+
+  def nlist(self, *items):
+    return u'[list=1]' + u''.join(map(lambda item: u"\n  [*]" + item, items)) + u"\n[/list]"
+
+  def alist(self, *items):
+    return u'[list=a]' + u''.join(map(lambda item: u"\n  [*]" + item, items)) + u"\n[/list]"
+
+
@@ -0,0 +1,255 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+#
+# Author:        Pau Sanchez (contact@pausanchez.com)
+# Version:       v1.0
+# Last Modified: 2010/09/15
+# 
+# For the latest version check out:
+#   http://www.codigomanso.com/en/projects
+# 
+# My blog:
+#   http://www.codigomanso.com/en/  - English Version
+#   http://www.codigomanso.com/es/  - Spanish Version
+#
+
+import sys
+import os
+import re
+import hashlib
+
+class bbcodeparser:
+  '''
+  This class parses BBCode into a internal structure to allow later processing and
+  conversion to HTML.
+
+  The parser tries to fix invalid code (like unclosed tags)
+
+  Useful URLs:
+    http://en.wikipedia.org/wiki/BBCode
+    http://www.bbcode.org/reference.php
+
+  Example:
+    > bbcode = bbcodeparser ()
+    > bbcode.parse ('[b]text in bold[/b]').html()
+    <b>text in bold</b>
+
+    # dump HTML
+    > bbcode.parse ('[p][color=red]text in red').html()
+    <p><span style="color:red;">text in red</span></p>
+
+    # dump fixed BBCode
+    > bbcode.parse ('[p][color=red]text in red').bbcode()
+    [p][color=red]text in red[/color][/p]
+
+    > bbcode.parse ('This [b][i]code[/b] will be fixed[/invalid]').bbcode()
+    This [b][i]code[/i][/b] will be fixed
+
+    # dump fixed bbcode
+    > str (bbcodeparse ('This [b][i]code[/b] will be fixed[/invalid]'))
+    This [b][i]code[/i][/b] will be fixed
+  '''
+  _bbcode = ''
+  _tokens = []
+
+  def __init__ (self, bbcode = '', fixInvalidCode = True):
+    ''' Initialize and parse bbcode string (if any is given)
+    '''
+    self.parse (bbcode, fixInvalidCode)
+    return
+
+  def __str__ (self):
+    return self.bbcode()
+
+  def parse (self, bbcode = None, fixInvalidCode = True):
+    '''
+    It will parse and return the token list, trying to fix tags if
+    fixInvalidCode is True
+
+    It will return the current object to allow chaining
+
+    Example:
+      code = bbcode()
+      code.parse ('<b>bold</b>', True) -> 
+      code.parse ('<b>bold<i>italics</b>', True) -> internally will add the missing '</i>'
+    '''
+    if bbcode is not None:
+      self._bbcode = bbcode
+      self._tokens = self.tokenize (bbcode)
+      if fixInvalidCode:
+        self._tokens = self.fixWrongTags (self._tokens)
+
+    return self
+
+  # return ALL tokens
+  def getTokens (self):
+    return self._tokens
+
+  def bbcode (self):
+    '''
+    Dump BBCode again. This is useful for dumping valid BBCode
+    '''
+    bbcode = []
+    for token in self._tokens:
+      if token is None:
+        continue
+
+      if isinstance (token, basestring):
+        bbcode.append (token.replace (u'[', u'\[').replace (u']', u'\]'))
+        continue
+      
+      tag       = token['tag']  # opening or closing simple tag. e.g: 'b', '/b', '/u', ...
+      tagOpener = (u'/' if tag[0] == u'/' else u'')
+
+      if (tagOpener == '/') or ('args' not in token):
+        bbcode.append (u'[' + tag + u']')
+      else:
+        # process args
+        argstr = ''
+
+        # the arg with the same name as the tag repersents the '=whatever'
+        if tag in token['args']:
+          if re.match ('\s|"', token['args'][tag]) is None:
+            argstr = u'=' + token['args'][tag]
+          else:
+            argstr = u'="' + token['args'][tag].replace (u'"', u'\"') + u'"'
+
+        for (k,v) in token['args'].iteritems():
+          if k == tag: # already processed
+            continue
+          argstr += ' ' + k + u'="' + v.replace (u'"', u'\"') + u'"'
+          
+        bbcode.append (u'[' + tag + argstr + ']')
+
+    return u''.join (bbcode)
+
+  def html (self, allowClassAttr = False, doDeepCopy = True):
+    '''
+    Convert current parsed code to HTML
+
+    @allowClassAttr
+      Is something like [b class="asdf"] allowed?
+
+    @doDeepCopy 
+      True:  it does a deep copy of tokens so this list will remain unchanged
+      False: tokens will be modified internally, but the output will be produced like 5x faster
+             it's a good idea to use False when the string parsed is huge and this is the
+             last operation on the string
+
+    Example:
+      code = bbcode ('[b]bold[/b]')
+      code.html() -> '<b>bold</b>'
+    '''
+    from bbcode2html import bbcode2html
+    return bbcode2html.convertToHTML (self._tokens, allowClassAttr = allowClassAttr, doDeepCopy = doDeepCopy)
+
+
+  @staticmethod
+  def fixWrongTags (inTokenList):
+    ''' Add missing tokens that have not been closed properly and try to fix some scenarios
+    '''
+    opened       = []
+    outTokenList = []
+    for token in inTokenList:
+      # normal string... do nothing
+      if isinstance(token, basestring):
+        outTokenList.append (token)
+      else:
+        # if starts with '/' is closing a tag
+        if token['tag'][0] == '/':
+          while (len (opened) > 0) and (opened[-1] != token['tag'][1:]):
+            outTokenList.append ({'tag' : '/' + opened[-1] })
+            del opened[-1]
+
+          if len(opened):
+            del opened[-1]
+            outTokenList.append (token)
+
+        # opening tag
+        else:
+          # if I open the same tag I opened before, close it, and open it again
+          if (len(opened) > 0) and (token['tag'] == opened[-1]):
+            outTokenList.append ({'tag' : '/' + opened[-1] })
+          else:
+            opened.append (token['tag'])
+          outTokenList.append (token)
+
+    # close all elements that have not been closed
+    while len(opened):
+      outTokenList.append ({'tag' : '/' + opened[-1] })
+      del opened[-1]
+
+    return outTokenList
+
+  @staticmethod
+  def tokenize(code):
+    '''
+    Tokenize BBCode tags and parameters
+
+    Return the token list using a internal format. See the example:
+      [
+        { 'tag' : 'p', 'args' : { 'font' : 'arial' } },
+        'This is ',
+        { 'tag' : 'url', 'args' : {'url' : 'http://www.google.com'} },
+        'a link to google',
+        { 'tag' : '/url' },
+        { 'tag' : '/p' }
+      ]
+    '''
+    re_tags    = re.compile (r'(\[[^]]+\])', re.DOTALL | re.UNICODE)
+    re_tagName = re.compile (r'\[([^]=\s]+)([^]]*)\]',  re.DOTALL | re.UNICODE)
+    #re_tagArgs = re.compile (r'\s*([^=]*)=(("([^"]+)")|([^\s]+))',  re.DOTALL | re.UNICODE)
+    re_tagArgs = re.compile (r'\s*([\w]*)=(("([^"]+)")|([^\s]+))',  re.DOTALL | re.UNICODE)
+
+    # get a unique name and replace escaped braces 
+    unique = hashlib.md5(code).hexdigest()
+    code   = code.replace ('\[', unique+'_OPEN_BRACE')
+    code   = code.replace ('\]', unique+'_CLOSE_BRACE')
+
+    splitted = re_tags.split(code)
+
+    outTokenList = []
+    for token in splitted:
+      if len(token) == 0:
+        continue
+      
+      if token[0] == '[':
+        match   = re_tagName.match (token)
+        if match:
+          tagName = match.group(1)
+          tagArgs = match.group(2)
+          
+          tagToken = { 'tag' : tagName.lower() }
+  
+          # parse arguments (if any)
+          if len(tagArgs) > 0:
+            allArgs = re_tagArgs.findall(tagArgs)
+  
+            tagArgs = {}
+            for arg in allArgs:
+              # if the argument has no name, use the tagName itself
+              argName  = (arg[0] if arg[0] != '' else tagName)
+              argValue = (arg[3] if (arg[1][0] == '"') else arg[4])
+  
+              tagArgs[argName.lower()] = argValue.replace ('\"', '"')
+  
+            tagToken['args'] = tagArgs
+  
+          outTokenList.append (tagToken)
+
+        # no match, append the text as it is
+        else:
+          outTokenList.append (token)
+      # append the text as it is
+      else:
+        outTokenList.append (token)
+
+    # restore escaped braces back (once code is parsed)
+    restoredTokenList = []
+    for token in outTokenList:
+      if isinstance (token, basestring):
+        token = token.replace (unique+'_OPEN_BRACE', '[').replace (unique+'_CLOSE_BRACE', ']')
+      restoredTokenList.append (token)
+
+    return restoredTokenList
+
@@ -0,0 +1,81 @@
+AUTHOR
+  Pau Sanchez
+  http://www.codigomanso.com/
+
+VERSION:
+  bbcodeutils v1.0
+  
+LICENSE
+  This code is licensed under Creative Commons Attribution 3.0
+  http://creativecommons.org/licenses/by/3.0/
+
+  You can use this python module or any part of the code you want as long as you add
+  my name as a contributor to your project.
+  
+DESCRIPTION
+  This module can be used to produce HTML from BBCode, to generate BBCode or to fix invalid BBCode.
+
+  The classes are:
+    - bbcodeparser
+    - bbcodebuilder
+    - bbcode2html
+  
+  You can use bbcodeparser to parse BBCode and to produce output in any format you want.
+
+  Open the python file to find more information and examples of use of each class. It can
+  be a good idea to check the test.py for examples
+
+  To run the unit tests:
+    > python test.py
+  
+  To run the performance test:
+    > python test.py BBCodeTests.performanceTest  
+
+
+EXAMPLES OF BBCode:
+
+  [b] -> bold
+  [u] -> underline
+  [i] -> italic
+
+  [center] -> center the text inside
+  [color=XXX] -> change color of text
+  [size=XXX] -> change size of text
+
+  Lists:
+  [ul] -> unordered list
+  [ol] -> ordered list
+  [li] -> list item
+
+  [list] -> start unordered list
+  [*]    -> list item
+  [list=1] -> start a list of numbers
+  [list=a] -> start a list of alphabetic characters
+
+  Advanced:
+  [url] -> link to url
+  [url=http://link/url/]text[/url]  
+  [url link=http://link/url/ title="This is the title"]text[/url]  
+
+  [img]http://to/image[/img]
+  [img=230x330]http://to/image[/img]
+  [img="Alt text here"]http://to/image[/img]
+  [img="Alt text here" width=320 height=240]http://to/image[/img]
+
+  [email]asdf@asdf.com[/email]
+  [email=john@asdf.com]John Smith[/email]
+
+  [google]search this[/google]
+  [wikipedia]Tom Hanks[/wikipedia]
+  [wikipedia lang=es]Tom Hanks[/wikipedia]
+
+  Tables:
+  [table]
+  [tr]
+  [th]
+  [td]
+
+  Advanced:
+  [google]
+  [wikipedia]
+
@@ -0,0 +1,420 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+#
+# Author:        Pau Sanchez (contact@pausanchez.com)
+# Version:       v1.0
+# Last Modified: 2010/09/15
+# 
+# For the latest version check out:
+#   http://www.codigomanso.com/en/projects
+# 
+# My blog:
+#   http://www.codigomanso.com/en/  - English Version
+#   http://www.codigomanso.com/es/  - Spanish Version
+#
+
+from bbcodeparser import bbcodeparser
+from bbcodebuilder import bbcodebuilder
+
+import random
+import unittest
+
+class BBCodeTests(unittest.TestCase):
+  def setUp (self):
+    self.bbcode = bbcodeparser()
+    return
+
+  def testConstructor (self):
+    self.assertEqual (bbcodeparser ('whatever').html(), 'whatever')
+    self.assertEqual (bbcodeparser ('[b]bold[/b]').html(), '<b>bold</b>')
+    self.assertEqual (str (bbcodeparser ('[b]bold[/b]')), '[b]bold[/b]')
+    return
+
+  def testBold (self):
+    self.assertEqual (self.bbcode.parse ('whatever').html(), 'whatever')
+    self.assertEqual (self.bbcode.parse ('[b]bold[/b]').html(), '<b>bold</b>')
+    self.assertEqual (self.bbcode.parse ('[B]bold[/b]').html(), '<b>bold</b>')
+    self.assertEqual (self.bbcode.parse ('this is [B]bold[/B]').html(), 'this is <b>bold</b>')
+    return
+
+  def testItalic (self):
+    self.assertEqual (self.bbcode.parse ('[i]italic[/i]').html(), '<i>italic</i>')
+    return
+
+  def testUnderline (self):
+    self.assertEqual (self.bbcode.parse ('[u]italic[/u]').html(), '<u>italic</u>')
+    return
+
+  def testURLs (self):
+    self.assertEqual (
+      self.bbcode.parse ('[url]http://www.google.com[/url]').html(),
+      '<a href="http://www.google.com">http://www.google.com</a>'
+    )
+    self.assertEqual (
+      self.bbcode.parse ('[url=http://www.google.com]Google[/url]').html(),
+      '<a href="http://www.google.com">Google</a>'
+    )
+    self.assertEqual (
+      self.bbcode.parse ('[url="http://www.google.com"]Google[/url]').html(),
+      '<a href="http://www.google.com">Google</a>'
+    )
+    self.assertEqual (
+      self.bbcode.parse ('[url="http://www.google.com" title="Search Engine"]Google[/url]').html(),
+      '<a href="http://www.google.com" title="Search Engine">Google</a>'
+    )
+    self.assertEqual (
+      self.bbcode.parse ('[url link="http://www.google.com"]Google[/url]').html(),
+      '<a href="http://www.google.com">Google</a>'
+    )
+    return
+
+  def testPTag (self):
+    self.assertEqual (
+      self.bbcode.parse ('[p color=#0000ff]blue[/p]').html(),
+      u'<p style="color: #0000ff;">blue</p>'
+    )
+    self.assertEqual (
+      self.bbcode.parse ('[p size=12]12pt font[/p]').html(),
+      u'<p style="font-size: 12pt;">12pt font</p>'
+    )
+
+    self.assertEqual (
+      self.bbcode.parse ('[p font=arial]arial[/p]').html(),
+      u'<p style="font-family: arial;">arial</p>'
+    )
+
+    self.assertEqual (
+      self.bbcode.parse ('[p font=arial color=blue size=14]blue 14pt arial').html(),
+      u'<p style="color: blue; font-size: 14pt; font-family: arial;">blue 14pt arial</p>' 
+    )
+
+    self.assertEqual (
+      self.bbcode.parse ('[p class=whatever]text[/p]').html(),
+      u'<p>text</p>' 
+    )
+    return
+
+  def testColorTag (self):
+    self.assertEqual (
+      self.bbcode.parse ('[color=#0000ff]blue[/color]').html(),
+      u'<span style="color: #0000ff;">blue</span>'
+    )
+    return
+
+  def testSizeTag (self):
+    self.assertEqual (
+      self.bbcode.parse ('[size=12]12pt font[/size]').html(),
+      u'<span style="font-size: 12pt;">12pt font</span>'
+    )
+    return
+
+  def testEmail(self):
+    self.assertEqual (
+      self.bbcode.parse ('[email]asdf@asdf.com[/email]').html(),
+      u'<a href="mailto:asdf@asdf.com">asdf@asdf.com</a>'
+    )
+
+    self.assertEqual (
+      self.bbcode.parse ('[email=john@smith.com]John Smith[/email]').html(),
+      u'<a href="mailto:john@smith.com">John Smith</a>'
+    )
+    return
+
+  def testImgTag (self):
+    self.assertEqual (
+      self.bbcode.parse ('[img]http://www.codigomanso.com/image.jpg[/img]').html(),
+      u'<img src="http://www.codigomanso.com/image.jpg" />'
+    )
+
+    self.assertEqual (
+      self.bbcode.parse ('[img="This is the ALT of the image"]http://www.codigomanso.com/image.jpg[/img]').html(),
+      u'<img alt="This is the ALT of the image" src="http://www.codigomanso.com/image.jpg" />'
+    )
+
+    self.assertEqual (
+      self.bbcode.parse ('[img=320x200]http://www.codigomanso.com/image.jpg[/img]').html(),
+      u'<img height="200" src="http://www.codigomanso.com/image.jpg" width="320" />'
+    )
+
+    self.assertEqual (
+      self.bbcode.parse ('[img=320x200 title="Image Test"]http://www.codigomanso.com/image.jpg[/img]').html(),
+      u'<img height="200" src="http://www.codigomanso.com/image.jpg" title="Image Test" width="320" />'
+    )
+
+    self.assertEqual (
+      self.bbcode.parse ('[img="whatever" width=320 height="212" title="Image Test"]http://www.codigomanso.com/image.jpg[/img]').html(),
+      u'<img alt="whatever" height="212" src="http://www.codigomanso.com/image.jpg" title="Image Test" width="320" />'
+    )
+    return
+
+  def testGoogleURL (self):
+    self.assertEqual (
+      self.bbcode.parse ('[google]asdf[/google]').html(),
+      u'<a href="http://www.google.com/search?q=asdf">asdf</a>'
+    )
+    self.assertEqual (
+      self.bbcode.parse ('[google]Tom Hanks[/google]').html(),
+      u'<a href="http://www.google.com/search?q=Tom+Hanks">Tom Hanks</a>'
+    )
+    return
+    
+  def testWikipediaURL (self):
+    self.assertEqual (
+      self.bbcode.parse ('[wikipedia]Tom Hanks[/wikipedia]').html(),
+      u'<a href="http://www.wikipedia.org/wiki/Tom_Hanks">Tom Hanks</a>'
+    )
+
+    self.assertEqual (
+      self.bbcode.parse ('[wikipedia language=en]Tom Hanks[/wikipedia]').html(),
+      u'<a href="http://en.wikipedia.org/wiki/Tom_Hanks">Tom Hanks</a>'
+    )
+    
+    self.assertEqual (
+      self.bbcode.parse ('[wikipedia lang=es]Tom Hanks[/wikipedia]').html(),
+      u'<a href="http://es.wikipedia.org/wiki/Tom_Hanks">Tom Hanks</a>'
+    )
+    
+    self.assertEqual (
+      self.bbcode.parse ('[wikipedia=es]Tom Hanks[/wikipedia]').html(),
+      u'<a href="http://es.wikipedia.org/wiki/Tom_Hanks">Tom Hanks</a>'
+    )
+    return
+    
+  def testListTags (self):
+    self.assertEqual (
+      self.bbcode.parse ('[ul][li]item 1[/li][li]item 2[/li][/ul]').html(),
+      u'<ul><li>item 1</li><li>item 2</li></ul>'
+    )
+    
+    self.assertEqual (
+      self.bbcode.parse ('[ol][li]item 1[/li][li]item 2[/li][/ol]').html(),
+      u'<ol><li>item 1</li><li>item 2</li></ol>'
+    )
+
+    self.assertEqual (
+      self.bbcode.parse ('[list][li]item 1[/li][li]item 2[/li][/list]').html(),
+      u'<ul><li>item 1</li><li>item 2</li></ul>'
+    )
+
+    self.assertEqual (
+      self.bbcode.parse ('[list][*]item 1[*]item 2[/list]').html(),
+      u'<ul><li>item 1</li><li>item 2</li></ul>'
+    )
+
+    self.assertEqual (
+      self.bbcode.parse ('[list=1][li]item 1[/li][li]item 2[/li][/list]').html(),
+      u'<ol type="1"><li>item 1</li><li>item 2</li></ol>'
+    )
+    return
+
+  def testInvalidCode (self):
+    self.assertEqual (self.bbcode.parse ('[invalid]valid text[/invalid]').html(), 'valid text')
+    self.assertEqual (
+      self.bbcode.parse ('[b]bold and [i]italics[/b]').html(),
+      '<b>bold and <i>italics</i></b>'
+    )
+    self.assertEqual (
+      self.bbcode.parse ('[/b]invalid[/b][/p]').html(),
+      'invalid'
+    )
+    self.assertEqual (
+      self.bbcode.parse ('[p][b]bold').html(),
+      '<p><b>bold</b></p>'
+    )
+    self.assertEqual (
+      self.bbcode.parse ('[p][b]a <b>').html(),
+      '<p><b>a &lt;b&gt;</b></p>'
+    )
+
+    self.assertEqual (
+      self.bbcode.parse ('[ol][li]item 1[li]item 2[/li][/ol]').html(),
+      u'<ol><li>item 1</li><li>item 2</li></ol>'
+    )
+
+    self.assertEqual (
+      self.bbcode.parse ('[b]\[b\] stands for [b]bold[/b]').html(),
+      u'<b>[b] stands for </b><b>bold</b>'
+    )
+    return
+
+  def testEscapedBrackets (self):
+    self.assertEqual (
+      self.bbcode.parse ('\[b\]not bold\[/b\]').html(),
+      u'[b]not bold[/b]'
+    )
+
+    self.assertEqual (
+      self.bbcode.parse ('[b]\[b\] stands for bold[/b]').html(),
+      u'<b>[b] stands for bold</b>'
+    )
+
+    self.assertEqual (
+      self.bbcode.parse ('\[b\][b]stands for bold[/b]').html(),
+      u'[b]<b>stands for bold</b>'
+    )
+
+    self.assertEqual (
+      self.bbcode.parse ('\[b\][b]stands for bold[/b] just like <b> in HTML').html(),
+      u'[b]<b>stands for bold</b> just like &lt;b&gt; in HTML'
+    )
+
+  def testBigExample (self):
+    inputText = """check this out
+
+    [h1 class=circle]heading[/h1]
+
+    [p size=14 color=blue font="verdana, Times New Roman"]This is [b] bold [/b] and this [i]italic[/i] and this is [color=red]red[/color] and this is [color="red"]also red[/color].
+    [/p]
+
+    fix [b][i]bold [font=verdana][size=12]and[/size][/font] italic[/b]
+    [img]http://www.codigomanso.com/b.jpg[/img]
+    [url]http://www.codigomanso.com/[/url]
+    [url=http://www.codigomanso.com/]Codigo Manso[/url]
+    [uRl link=http://www.codigomanso.com title="Codigo Manso Blog"]Codigo Manso[/url]
+
+    [ul]
+     [Li]item 1[/Li]
+     [li]item 2[/LI]
+    [/UL]
+
+    [list=1 ]
+     [*]item 1
+     [*]item 2
+    [/list]
+
+    [table class="big"]
+      [tr]
+        [th]big[/th]
+      [/tr]
+    [/table]
+    [invalid class="extra"]whatever[/invalid]"""
+
+    out = self.bbcode.parse (inputText).html(allowClassAttr = True)
+    self.assertEquals (out, '''check this out
+
+    <h1 class="circle">heading</h1>
+
+    <p style="color: blue; font-size: 14pt; font-family: verdana, Times New Roman;">This is <b> bold </b> and this <i>italic</i> and this is <span style="color: red;">red</span> and this is <span style="color: red;">also red</span>.
+    </p>
+
+    fix <b><i>bold <span style="font-family: verdana;"><span style="font-size: 12pt;">and</span></span> italic</i></b>
+    <img src="http://www.codigomanso.com/b.jpg" />
+    <a href="http://www.codigomanso.com/">http://www.codigomanso.com/</a>
+    <a href="http://www.codigomanso.com/">Codigo Manso</a>
+    <a href="http://www.codigomanso.com" title="Codigo Manso Blog">Codigo Manso</a>
+
+    <ul>
+     <li>item 1</li>
+     <li>item 2</li>
+    </ul>
+
+    <ol type="1">
+     <li>item 1
+     </li><li>item 2
+    </li></ol>
+
+    <table class="big">
+      <tr>
+        <th>big</th>
+      </tr>
+    </table>
+    whatever''')
+
+
+  def testBBCodeDumper (self):
+    self.assertEquals (
+      self.bbcode.parse ('[b]bold[/b]').bbcode(),
+      '[b]bold[/b]'
+    )
+
+    self.assertEquals (
+      self.bbcode.parse ('[color=red]text in red[/color]').bbcode(),
+      '[color=red]text in red[/color]'
+    )
+    self.assertEquals (
+      self.bbcode.parse ('[p][color=red]text in red').bbcode(),
+      '[p][color=red]text in red[/color][/p]'
+    )
+
+    self.assertEquals (
+      self.bbcode.parse ('This [b][i]code[/b] will be fixed[/invalid]').bbcode(),
+      'This [b][i]code[/i][/b] will be fixed'
+    )
+
+    self.assertEquals (
+      self.bbcode.parse ('\[[url]http://www.codigomanso.com/en[/url]\]').bbcode(),
+      "\[[url]http://www.codigomanso.com/en[/url]\]"
+    )
+
+  def performanceTest(self):
+    '''
+    This test checks the performance of parse and html operations
+
+    To run this test type:
+      > python test.py BBCodeTests.performanceTest
+    '''
+    inputText = """check this out
+
+    [h1 class=circle]heading[/h1]
+
+    [p size=14 color=blue font="verdana, Times New Roman"]This is [b] bold [/b] and this [i]italic[/i] and this is [color=red]red[/color] and this is [color="red"]also red[/color].
+    [/p]
+
+    fix [b][i]bold [font=verdana][size=12]and[/size][/font] italic[/b]
+    [img]http://www.codigomanso.com/b.jpg[/img]
+    [url]http://www.codigomanso.com/[/url]
+    [url=http://www.codigomanso.com/]Codigo Manso[/url]
+    [uRl link=http://www.codigomanso.com title="Codigo Manso Blog"]Codigo Manso[/url]
+
+    [ul]
+     [Li]item 1[/Li]
+     [li]item 2[/LI]
+    [/UL]
+
+    [list=1 ]
+     [*]item 1
+     [*]item 2
+    [/list]
+
+    [table class="big"]
+      [tr]
+        [th]big[/th]
+      [/tr]
+    [/table]
+    [invalid class="extra"]whatever[/invalid]"""
+
+    import time
+    start = time.time()
+
+    for i in range(0, 12):
+      inputText += inputText
+
+    print "len(inputText) = %.2f MB  (took %.2f seconds)" % (len(inputText)/(1024.0*1024.0), time.time() - start)
+    
+    bbcode = bbcodeparser()
+    start = time.time()
+    bbcode.parse (inputText)
+    total = (time.time() - start)
+    print "time (bbcode.parse()) = %f" % total
+    print "  >> %.2f chars/second" % (len(inputText) / total)
+
+    start = time.time()
+    bbcode.html(doDeepCopy = False)
+    total = (time.time() - start)
+    print "time (bbcode.html()) = %f" % total
+    print "  >> %.2f chars/second" % (len(inputText) / total)
+    return
+    
+  def testCodeBuilder (self):
+    bbcode = bbcodebuilder ()
+    self.assertEquals (bbcode.b ('bold'), u'[b]bold[/b]')
+    self.assertEquals (bbcode.color ('this goes in red', 'red'), u'[color=red]this goes in red[/color]')
+    self.assertEquals (bbcode.url ('Google', 'http://www.google.com'), u'[url=http://www.google.com]Google[/url]')
+    self.assertEquals (bbcode.alist('item 1', 'item 2'), u"[list=a]\n  [*]item 1\n  [*]item 2\n[/list]")
+    return
+
+if __name__ == '__main__':
+    unittest.main()
+
+
+
@@ -21,10 +21,6 @@ import ConfigParser
 # inherit from Configurable.  The config file(s) uses ini format:
 # [sections] with key:value settings.
 #
-# writer does [defaults], [www.whofic.com], [epub], [www.whofic.com:epub], [overrides]
-#
-# Until a write is created, the adapter only has [defaults], [www.whofic.com], [overrides]
-#
 # [defaults]
 # titlepage_entries: category,genre, status
 # [www.whofic.com]
@@ -36,29 +32,76 @@ import ConfigParser
 # [overrides]
 # titlepage_entries: category

+class Configuration(ConfigParser.SafeConfigParser):

-class Configurable(object):
-
-    def __init__(self, config):
-        self.config = config
-        self.sectionslist = ['defaults']
-
-    def setSectionOrder(self,site,fileform=None):
+    def __init__(self, site, fileform):
+        ConfigParser.SafeConfigParser.__init__(self)
        self.sectionslist = ['defaults']
        self.addConfigSection(site)
        if fileform:
            self.addConfigSection(fileform)
            self.addConfigSection(site+":"+fileform)
        self.addConfigSection("overrides")
+        
+        self.validEntries = [
+            'category',
+            'genre',
+            'language',
+            'characters',
+            'ships',
+            'series',
+            'status',
+            'datePublished',
+            'dateUpdated',
+            'dateCreated',
+            'rating',
+            'warnings',
+            'numChapters',
+            'numWords',
+            'site',
+            'storyId',
+            'authorId',
+            'extratags',
+            'title',
+            'storyUrl',
+            'description',
+            'author',
+            'authorUrl',
+            'formatname',
+            'formatext',
+            'siteabbrev',
+            'version',
+            # internal stuff.
+            'langcode',
+            'output_css',
+            'authorHTML'
+            ]

    def addConfigSection(self,section):
        self.sectionslist.insert(0,section)

-    def getConfig(self, key):
-        val = ""
+    def isValidMetaEntry(self, key):
+        return key in self.getValidMetaList()
+
+    def getValidMetaList(self):
+        return self.validEntries + self.getConfigList("extra_valid_entries")
+
+    def hasConfig(self, key):
        for section in self.sectionslist:
            try:
-                val = self.config.get(section,key)
+                self.get(section,key)
+                #print("found %s in section [%s]"%(key,section))
+                return True
+            except:
+                pass
+
+        return False
+        
+    def getConfig(self, key, default=""):
+        val = default
+        for section in self.sectionslist:
+            try:
+                val = self.get(section,key)
                if val and val.lower() == "false":
                    val = False
                #print "getConfig(%s)=[%s]%s" % (key,section,val)
@@ -73,5 +116,26 @@ class Configurable(object):
        vlist = self.getConfig(key).split(',')
        vlist = [ v.strip() for v in vlist ]
        #print "vlist("+key+"):"+str(vlist)
-        return vlist
+        return vlist        
+
+# extended by adapter, writer and story for ease of calling configuration.
+class Configurable(object):
+
+    def __init__(self, configuration):
+        self.configuration = configuration
+
+    def isValidMetaEntry(self, key):
+        return self.configuration.isValidMetaEntry(key)
+
+    def getValidMetaList(self):
+        return self.configuration.getValidMetaList()
+    
+    def hasConfig(self, key):
+        return self.configuration.hasConfig(key)
+        
+    def getConfig(self, key, default=""):
+        return self.configuration.getConfig(key)
+
+    def getConfigList(self, key):
+        return self.configuration.getConfigList(key)
        
@@ -40,6 +40,51 @@ def get_update_data(inputio,
    ## Save the path to the .opf file--hrefs inside it are relative to it.
    relpath = get_path_part(rootfilename)
            
+    oldcover = None
+    calibrebookmark = None
+    logfile = None
+    # Looking for pre-existing cover.
+    for item in contentdom.getElementsByTagName("reference"):
+        if item.getAttribute("type") == "cover":
+            # there is a cover (x)html file, save the soup for it.
+            href=relpath+item.getAttribute("href")
+            oldcoverhtmlhref = href
+            oldcoverhtmldata = epub.read(href)
+            oldcoverhtmltype = "application/xhtml+xml"
+            for item in contentdom.getElementsByTagName("item"):
+                if( relpath+item.getAttribute("href") == oldcoverhtmlhref ):
+                    oldcoverhtmltype = item.getAttribute("media-type")
+                    break
+            soup = bs.BeautifulSoup(oldcoverhtmldata.decode("utf-8"))
+            src = None
+            # first img or image tag.
+            imgs = soup.findAll('img')
+            if imgs:
+                src = get_path_part(href)+imgs[0]['src']
+            else:
+                imgs = soup.findAll('image')
+                if imgs:
+                    src=get_path_part(href)+imgs[0]['xlink:href']
+
+            if not src:
+                continue
+            try:
+                # remove all .. and the path part above it, if present.
+                # Mostly for epubs edited by Sigil.
+                src = re.sub(r"([^/]+/\.\./)","",src)
+                print("epubutils: found pre-existing cover image:%s"%src)
+                oldcoverimghref = src
+                oldcoverimgdata = epub.read(src)
+                for item in contentdom.getElementsByTagName("item"):
+                    if( relpath+item.getAttribute("href") == oldcoverimghref ):
+                        oldcoverimgtype = item.getAttribute("media-type")
+                        break
+                oldcover = (oldcoverhtmlhref,oldcoverhtmltype,oldcoverhtmldata,oldcoverimghref,oldcoverimgtype,oldcoverimgdata)
+            except Exception as e:
+                print("Cover Image %s not found"%src)
+                print("Exception: %s"%(unicode(e)))
+                traceback.print_exc()
+
    filecount = 0
    soups = [] # list of xhmtl blocks
    images = {} # dict() longdesc->data
@@ -52,6 +97,11 @@ def get_update_data(inputio,
            if( item.getAttribute("media-type") == "application/xhtml+xml" ):
                href=relpath+item.getAttribute("href")
                #print("---- item href:%s path part: %s"%(href,get_path_part(href)))
+                if re.match(r'.*/log_page\.x?html',href):
+                    try:
+                        logfile = epub.read(href).decode("utf-8")
+                    except:
+                        pass # corner case I bumped into while testing.
                if re.match(r'.*/(file|chapter)\d+\.x?html',href):
                    if getsoups:
                        soup = bs.BeautifulSoup(epub.read(href).decode("utf-8"))
@@ -61,7 +111,7 @@ def get_update_data(inputio,
                            try:
                                newsrc=get_path_part(href)+img['src']
                                # remove all .. and the path part above it, if present.
-                                # Most for epubs edited by Sigil.
+                                # Mostly for epubs edited by Sigil.
                                newsrc = re.sub(r"([^/]+/\.\./)","",newsrc)
                                longdesc=img['longdesc']
                                data = epub.read(newsrc)
@@ -85,12 +135,54 @@ def get_update_data(inputio,
                        
                    filecount+=1

+    try:
+        calibrebookmark = epub.read("META-INF/calibre_bookmarks.txt")
+    except:
+        pass
+                    
    for k in images.keys():
        print("\tlongdesc:%s\n\tData len:%s\n"%(k,len(images[k])))
-    return (source,filecount,soups,images)
+    return (source,filecount,soups,images,oldcover,calibrebookmark,logfile)

 def get_path_part(n):
    relpath = os.path.dirname(n)
    if( len(relpath) > 0 ):
        relpath=relpath+"/"
    return relpath
+
+def get_story_url_from_html(inputio,_is_good_url=None):
+
+    #print("get_story_url_from_html called")
+    epub = ZipFile(inputio, 'r')
+
+    ## Find the .opf file.
+    container = epub.read("META-INF/container.xml")
+    containerdom = parseString(container)
+    rootfilenodelist = containerdom.getElementsByTagName("rootfile")
+    rootfilename = rootfilenodelist[0].getAttribute("full-path")
+
+    contentdom = parseString(epub.read(rootfilename))
+    #firstmetadom = contentdom.getElementsByTagName("metadata")[0]
+
+    ## Save the path to the .opf file--hrefs inside it are relative to it.
+    relpath = get_path_part(rootfilename)
+            
+    # spin through the manifest--only place there are item tags.
+    for item in contentdom.getElementsByTagName("item"):
+        # First, count the 'chapter' files.  FFDL uses file0000.xhtml,
+        # but can also update epubs downloaded from Twisting the
+        # Hellmouth, which uses chapter0.html.
+        #print("---- item:%s"%item)
+        if( item.getAttribute("media-type") == "application/xhtml+xml" ):
+            filehref=relpath+item.getAttribute("href")
+            soup = bs.BeautifulSoup(epub.read(filehref).decode("utf-8"))
+            for link in soup.findAll('a',href=re.compile(r'^http.*')):
+                ahref=link['href']
+                #print("href:(%s)"%ahref)
+                # hack for bad ficsaver ffnet URLs.
+                m = re.match(r"^http://www.fanfiction.net/s(?P<id>\d+)//$",ahref)
+                if m != None:
+                    ahref="http://www.fanfiction.net/s/%s/1/"%m.group('id')
+                if _is_good_url == None or _is_good_url(ahref):
+                    return ahref
+    return None
@@ -63,6 +63,7 @@ class UnknownSite(Exception):
    def __init__(self,url,supported_sites_list):
        self.url=url
        self.supported_sites_list=supported_sites_list
+        self.supported_sites_list.sort()

    def __str__(self):
        return "Unknown Site(%s).  Supported sites: (%s)" % (self.url, ", ".join(self.supported_sites_list))
@@ -0,0 +1,110 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import re
+import urlparse
+import urllib2 as u2
+
+from BeautifulSoup import BeautifulSoup 
+from gziphttp import GZipProcessor
+
+import adapters
+from configurable import Configuration
+
+def get_urls_from_page(url,configuration=None):
+
+    normalized = set() # normalized url
+    retlist = [] # orig urls.
+    if not configuration:
+        configuration = Configuration("test1.com","EPUB")
+
+    data = None
+    
+    # special stuff to log into archiveofourown.org, if possible.
+    # Unlike most that show the links to 'adult' stories, but protect
+    # them, AO3 doesn't even show them if not logged in.  Only works
+    # with saved user/pass--not going to prompt for list.
+    if 'archiveofourown.org' in url:
+        ao3adapter = adapters.getAdapter(configuration,"http://www.archiveofourown.org/works/0")
+        if ao3adapter.getConfig("username"):
+            if ao3adapter.getConfig("is_adult"):
+                addurl = "?view_adult=true"
+            else:
+                addurl=""
+            # just to get an authenticity_token.
+            data = ao3adapter._fetchUrl(url+addurl)
+            # login the session.
+            ao3adapter.performLogin(url,data)
+            # get the list page with logged in session.
+            data = ao3adapter._fetchUrl(url)
+
+    if not data:
+        opener = u2.build_opener(u2.HTTPCookieProcessor(),GZipProcessor())
+        data = opener.open(url).read()
+
+    soup = BeautifulSoup(data)
+    
+    for a in soup.findAll('a'):
+        if a.has_key('href'):
+            href = form_url(url,a['href'])
+            # this (should) catch normal story links, some javascript
+            # 'are you old enough' links, and 'Report This' links.
+            # The 'normalized' set prevents duplicates.
+            if 'story.php' in a['href']:
+                #print("trying:%s"%a['href'])
+                m = re.search(r"(?P<sid>(view)?story\.php\?(sid|psid|no|story|stid)=\d+)",a['href'])
+                if m != None:
+                    href = form_url(url,m.group('sid'))
+                    
+            try:
+                href = href.replace('&index=1','')
+                adapter = adapters.getAdapter(configuration,href)
+                if adapter.story.getMetadata('storyUrl') not in normalized:
+                    normalized.add(adapter.story.getMetadata('storyUrl'))
+                    retlist.append(href)
+            except:
+                pass
+
+    return retlist
+
+def form_url(parenturl,url):
+     url = url.strip() # ran across an image with a space in the
+                       # src. Browser handled it, so we'd better, too.
+ 
+     if "//" in url or parenturl == None:
+         returl = url
+     else:
+         parsedUrl = urlparse.urlparse(parenturl)
+         if url.startswith("/") :
+             returl = urlparse.urlunparse(
+                 (parsedUrl.scheme,
+                  parsedUrl.netloc,
+                  url,
+                  '','',''))
+         else:
+             toppath=""
+             if parsedUrl.path.endswith("/"):
+                 toppath = parsedUrl.path
+             else:
+                 toppath = parsedUrl.path[:parsedUrl.path.rindex('/')]
+             returl = urlparse.urlunparse(
+                 (parsedUrl.scheme,
+                  parsedUrl.netloc,
+                  toppath + '/' + url,
+                  '','',''))
+     return returl
+       
@@ -103,9 +103,10 @@ def onlywhite(line):
            return c is ' '
    return line

-def optwrap(text):
+def optwrap(text,wrap_width=BODY_WIDTH):
    """Wrap all paragraphs in the provided text."""
-    if not BODY_WIDTH:
+    
+    if not wrap_width:
        return text
    
    assert wrap, "Requires Python 2.3."
@@ -114,7 +115,7 @@ def optwrap(text):
    for para in text.split("\n"):
        if len(para) > 0:
            if para[0] is not ' ' and para[0] is not '-' and para[0] is not '*':
-                for line in wrap(para, BODY_WIDTH):
+                for line in wrap(para, wrap_width):
                    result += line + "\n"
                result += "\n"
                newlines = 2
@@ -423,8 +424,8 @@ def html2text_file(html, out=wrapwrite, baseurl=''):
    h.feed("")
    return h.close()

-def html2text(html, baseurl=''):
-    return optwrap(html2text_file(html, None, baseurl))
+def html2text(html, baseurl='', wrap_width=BODY_WIDTH):
+    return optwrap(html2text_file(html, None, baseurl),wrap_width)

 if __name__ == "__main__":
    baseurl = ''
@@ -19,12 +19,21 @@ import re

 def _unirepl(match):
    "Return the unicode string for a decimal number"
-    if match.group(1)=='x':
+    if match.group(1).startswith('x'):
        radix=16
+        s = match.group(1)[1:]
    else:
        radix=10
-    value = int(match.group(2), radix)
-    return "%s%s"%(unichr(value),match.group(3))
+        s = match.group(1)
+    try:
+        value = int(s, radix)
+        retval = "%s%s"%(unichr(value),match.group(2))
+    except:
+        # This way, at least if there's more of entities out there
+        # that fail, it doesn't blow the entire download.
+        print "Numeric entity translation failed, skipping: &#x%s%s"%(match.group(1),match.group(2))
+        retval = ""
+    return retval

 def _replaceNumberEntities(data):
    # The same brokenish entity parsing in SGMLParser that inserts ';'
@@ -32,7 +41,9 @@ def _replaceNumberEntities(data):
    # entities, including part of the next word if it's a-z.
    # "Don't&#8212ever&#8212do&#8212that&#8212again," becomes
    # "Don't&#8212e;ver&#8212d;o&#8212;that&#8212a;gain,"
-    p = re.compile(r'&#(x?)([0-9a-fA-F]{,4})([0-9a-fA-F]*);')
+    # Also need to allow for 5 digit decimal entities &#27861;
+    # Last expression didn't allow for 2 digit hex correctly: &#xE9;
+    p = re.compile(r'&#(x[0-9a-fA-F]{,4}|[0-9]{,5})([0-9a-fA-F]*?);')
    return p.sub(_unirepl, data)

 def _replaceNotEntities(data):
@@ -19,8 +19,11 @@ import os, re
 import urlparse
 import string
 from math import floor
+from functools import partial

+import exceptions
 from htmlcleanup import conditionalRemoveEntities, removeAllEntities
+from configurable import Configurable

 # Create convert_image method depending on which graphics lib we can
 # load.  Preferred: calibre, PIL, none
@@ -185,10 +188,12 @@ langs = {
    "Devanagari":"hi",
    }

-class Story:
+class Story(Configurable):
    
-    def __init__(self):
+    def __init__(self, configuration):
+        Configurable.__init__(self, configuration)
        try:
+            ## calibre plugin will set externally to match PI version.
            self.metadata = {'version':os.environ['CURRENT_VERSION_ID']}
        except:
            self.metadata = {'version':'4.4'}
@@ -196,88 +201,227 @@ class Story:
        self.chapters = [] # chapters will be tuples of (title,html)
        self.imgurls = []
        self.imgtuples = []
-        self.listables = {} # some items (extratags, category, warnings & genres) are also kept as lists.
-        self.cover=None
+        
+        self.cover=None # *href* of new cover image--need to create html.
+        self.oldcover=None # (oldcoverhtmlhref,oldcoverhtmltype,oldcoverhtmldata,oldcoverimghref,oldcoverimgtype,oldcoverimgdata)
+        self.calibrebookmark=None # cheesy way to carry calibre bookmark file forward across update.
+        self.logfile=None # cheesy way to carry log file forward across update.

-    def setMetadata(self, key, value):
+        ## Look for config parameter, split and add each to metadata field.
+        for (config,metadata) in [("extratags","extratags"),
+                                  ("extracategories","category"),
+                                  ("extragenres","genre"),
+                                  ("extracharacters","characters"),
+                                  ("extraships","ships"),
+                                  ("extrawarnings","warnings")]:
+            for val in self.getConfigList(config):
+                self.addToList(metadata,val)
+
+        self.setReplace(self.getConfig('replace_metadata'))
+        
+    def setMetadata(self, key, value, condremoveentities=True):
        ## still keeps &lt; &lt; and &amp;
-        self.metadata[key]=conditionalRemoveEntities(value)
+        if condremoveentities:
+            self.metadata[key]=conditionalRemoveEntities(value)
+        else:
+            self.metadata[key]=value
        if key == "language":
            try:
                self.metadata['langcode'] = langs[self.metadata[key]]
            except:
                self.metadata['langcode'] = 'en'
+        if key == 'dateUpdated':
+            # Last Update tags for Bill.
+            self.addToList('lastupdate',value.strftime("Last Update Year/Month: %Y/%m"))
+            self.addToList('lastupdate',value.strftime("Last Update: %Y/%m/%d"))

-    def getMetadataRaw(self,key):
-        if self.metadata.has_key(key):
-            return self.metadata[key]

-    def doReplacments(self,value):
-        for (p,v) in self.replacements:
-            if (isinstance(value,str) or isinstance(value,unicode)) and re.match(p,value):
-                value = re.sub(p,v,value)                
+    ## Two or three part lines.  Two part effect everything.
+    ## Three part effect only those key(s) lists.
+    ## pattern=>replacement
+    ## metakey,metakey=>pattern=>replacement
+    def setReplace(self,replace):
+        for line in replace.splitlines():
+            if "=>" in line:
+                parts = map( lambda x: x.strip(), line.split("=>") )
+                if len(parts) > 2:
+                    parts[0] = map( lambda x: x.strip(), parts[0].split(",") )
+                    self.replacements.append(parts)
+                else:
+                    self.replacements.append([None]+parts)
+    
+    def doReplacments(self,value,key):
+        for (keys,p,v) in self.replacements:
+            if (keys == None or key in keys) \
+                    and isinstance(value,basestring) \
+                    and re.search(p,value):
+                #pv=value
+                value = re.sub(p,v,value)
+                #print("change:%s => %s === %s => %s "%(p,v,pv,value))
        return value
        
-    def getMetadata(self, key, removeallentities=False):
+    def getMetadataRaw(self,key):
+        if self.isValidMetaEntry(key) and self.metadata.has_key(key):
+            return self.metadata[key]
+
+    def getMetadata(self, key,
+                    removeallentities=False,
+                    doreplacements=True):
        value = None
-        if self.getLists().has_key(key):
-            value = ', '.join(self.getList(key))
-        if self.metadata.has_key(key):
+        if not self.isValidMetaEntry(key):
+            return value
+
+        if self.isList(key):
+            return u', '.join(self.getList(key, removeallentities, doreplacements=True))
+        elif self.metadata.has_key(key):
            value = self.metadata[key]
            if value:
                if key == "numWords":
                    value = commaGroups(value)
-                if key == "dateCreated":
-                    value = value.strftime("%Y-%m-%d %H:%M:%S")
-                if key == "datePublished" or key == "dateUpdated":
-                 value = value.strftime("%Y-%m-%d")
+                if key == "numChapters":
+                    value = commaGroups("%d"%value)
+                if key in ("dateCreated","datePublished","dateUpdated"):
+                    value = value.strftime(self.getConfig(key+"_format","%Y-%m-%d"))

-        value=self.doReplacments(value)
-        if removeallentities and value != None:
-            return removeAllEntities(value)
-        else:
-            return value
-
-    def getAllMetadata(self, removeallentities=False):
+            if doreplacements:
+                value=self.doReplacments(value,key)
+            if removeallentities and value != None:
+                return removeAllEntities(value)
+            else:
+                return value
+        
+    def getAllMetadata(self,
+                       removeallentities=False,
+                       doreplacements=True,
+                       keeplists=False):
        '''
-        All single value *and* list value metadata as strings.
+        All single value *and* list value metadata as strings (unless keeplists=True, then keep lists).
        '''
        allmetadata = {}
-        for k in self.metadata.keys():
-            allmetadata[k] = self.getMetadata(k, removeallentities)
-        for l in self.listables.keys():
-            allmetadata[l] = self.getMetadata(l, removeallentities)
-
-        return allmetadata
        
+        # special handling for authors/authorUrls
+        authlinkhtml="<a class='authorlink' href='%s'>%s</a>"
+        if self.isList('author'): # more than one author, assume multiple authorUrl too.
+            htmllist=[]
+            for i, v in enumerate(self.getList('author')):
+                aurl = self.getList('authorUrl')[i]                    
+                auth = v
+                # make sure doreplacements & removeallentities are honored.
+                if doreplacements:
+                    aurl=self.doReplacments(aurl,'authorUrl')
+                    auth=self.doReplacments(auth,'author')
+                if removeallentities:
+                    aurl=removeAllEntities(aurl)
+                    auth=removeAllEntities(auth)
+                
+                htmllist.append(authlinkhtml%(aurl,auth))
+            self.setMetadata('authorHTML',', '.join(htmllist))
+        else:
+           self.setMetadata('authorHTML',authlinkhtml%(self.getMetadata('authorUrl', removeallentities, doreplacements),
+                                                       self.getMetadata('author', removeallentities, doreplacements)))
+
+        for k in self.getValidMetaList():
+            if self.isList(k) and keeplists:
+                allmetadata[k] = self.getList(k, removeallentities, doreplacements)
+            else:
+                allmetadata[k] = self.getMetadata(k, removeallentities, doreplacements)
+                
+        return allmetadata
+
+    # just for less clutter in adapters.
+    def extendList(self,listname,l):
+        for v in l:
+            self.addToList(listname,v.strip())
+    
    def addToList(self,listname,value):
        if value==None:
            return
        value = conditionalRemoveEntities(value)
-        if not self.listables.has_key(listname):
-            self.listables[listname]=[]
+        if not self.isList(listname) or not listname in self.metadata:
+            # Calling addToList to a non-list meta will overwrite it.
+            self.metadata[listname]=[]
        # prevent duplicates.
-        if not value in self.listables[listname]:
-            self.listables[listname].append(value)
+        if not value in self.metadata[listname]:
+            self.metadata[listname].append(value)

-    def getList(self,listname):
-        if not self.listables.has_key(listname):
-            return []
-        return filter( lambda x : x!=None and x!='' ,
-                       map(self.doReplacments,self.listables[listname]) )
-
-    def getLists(self):
-        lsts = {}
-        for ln in self.listables.keys():
-            lsts[ln] = self.getList(ln)
-        return lsts
+    def isList(self,listname):
+        'Everything set with an include_in_* is considered a list.'
+        return self.hasConfig("include_in_"+listname) or \
+            ( self.isValidMetaEntry(listname) and self.metadata.has_key(listname) \
+                  and isinstance(self.metadata[listname],list) )
    
+    def getList(self,listname,
+                removeallentities=False,
+                doreplacements=True,
+                includelist=[]):
+        #print("getList(%s,%s)"%(listname,includelist))
+        retlist = []
+        
+        if not self.isValidMetaEntry(listname):
+            return retlist
+        
+        # includelist prevents infinite recursion of include_in_'s
+        if self.hasConfig("include_in_"+listname) and listname not in includelist:
+            for k in self.getConfigList("include_in_"+listname):
+                retlist.extend(self.getList(k,removeallentities,doreplacements,includelist=includelist+[listname]))
+        else:
+        
+            if not self.isList(listname):
+                retlist = [self.getMetadata(listname,removeallentities, doreplacements)]
+            else:
+                retlist = self.getMetadataRaw(listname)
+
+        if retlist:
+            if doreplacements:
+                retlist = filter( lambda x : x!=None and x!='' ,
+                                  map(partial(self.doReplacments,key=listname),retlist) )
+            if removeallentities:
+                retlist = filter( lambda x : x!=None and x!='' ,
+                                  map(removeAllEntities,retlist) )
+
+        if retlist:
+            # remove dups and sort.
+            return sorted(list(set(retlist)))
+        else:
+            return []
+
+    def getSubjectTags(self, removeallentities=False):
+        # set to avoid duplicates subject tags.
+        subjectset = set()
+        
+        tags_list = self.getConfigList("include_subject_tags") + self.getConfigList("extra_subject_tags")
+            
+        # metadata all go into dc:subject tags, but only if they are configured.
+        for (name,value) in self.getAllMetadata(removeallentities=removeallentities,keeplists=True).iteritems():
+            if name in tags_list:
+                if isinstance(value,list):
+                    for tag in value:
+                        subjectset.add(tag)
+                else:
+                    subjectset.add(value)
+
+        if None in subjectset:
+            subjectset.remove(None)
+
+        return list(subjectset | set(self.getConfigList("extratags")))
+            
    def addChapter(self, title, html):
+        if self.getConfig('strip_chapter_numbers') and \
+                self.getConfig('chapter_title_strip_pattern'):
+            title = re.sub(self.getConfig('chapter_title_strip_pattern'),"",title)
        self.chapters.append( (title,html) )

    def getChapters(self):
        "Chapters will be tuples of (title,html)"
-        return self.chapters
+        retval = []
+        if self.getConfig('add_chapter_numbers') and \
+                self.getConfig('chapter_title_add_pattern'):
+            for index, (title,html) in enumerate(self.chapters):
+                retval.append( (string.Template(self.getConfig('chapter_title_add_pattern')).substitute({'index':index+1,'title':title}),html) ) 
+        else:
+            retval = self.chapters
+            
+        return retval

    def formatFileName(self,template,allowunsafefilename=True):
        values = origvalues = self.getAllMetadata()
@@ -295,7 +439,12 @@ class Story:

    # pass fetch in from adapter in case we need the cookies collected
    # as well as it's a base_story class method.
-    def addImgUrl(self,configurable,parenturl,url,fetch,cover=False,coverexclusion=None):
+    def addImgUrl(self,parenturl,url,fetch,cover=False,coverexclusion=None):
+
+        # otherwise it saves the image in the epub even though it
+        # isn't used anywhere.
+        if cover and self.getConfig('never_make_cover'):
+            return
        
        url = url.strip() # ran across an image with a space in the
                          # src. Browser handled it, so we'd better, too.
@@ -331,18 +480,21 @@ class Story:
        prefix='ffdl'
        if imgurl not in self.imgurls:
            parsedUrl = urlparse.urlparse(imgurl)
-            sizes = [ int(x) for x in configurable.getConfigList('image_max_size') ]
+            try:
+                sizes = [ int(x) for x in self.getConfigList('image_max_size') ]
+            except Exception, e:
+                raise exceptions.FailedToDownload("Failed to parse image_max_size from personal.ini:%s\nException: %s"%(self.getConfigList('image_max_size'),e))
            try:
                (data,ext,mime) = convert_image(imgurl,
                                                fetch(imgurl),
                                                sizes,
-                                                configurable.getConfig('grayscale_images'))
+                                                self.getConfig('grayscale_images'))
            except Exception, e:
                print("Failed to load or convert image, skipping:\n%s\nException: %s"%(imgurl,e))
                return "failedtoload"
            
            # explicit cover, make the first image.
-            if cover and not configurable.getConfig('never_make_cover'):
+            if cover and not self.getConfig('never_make_cover'):
                if len(self.imgtuples) > 0 and 'cover' in self.imgtuples[0]['newsrc']:
                    # remove existing cover, if there is one.
                    del self.imgurls[0]
@@ -359,8 +511,8 @@ class Story:
                #          NOT never_make_cover AND
                #          either no coverexclusion OR coverexclusion doesn't match
                if self.cover == None and \
-                        configurable.getConfig('make_firstimage_cover') and \
-                        not configurable.getConfig('never_make_cover') and \
+                        self.getConfig('make_firstimage_cover') and \
+                        not self.getConfig('never_make_cover') and \
                        (not coverexclusion or not re.search(coverexclusion,imgurl)):
                    newsrc = "images/cover.%s"%ext
                    self.cover=newsrc
@@ -389,13 +541,8 @@ class Story:
        return retlist
    
    def __str__(self):
-        return "Metadata: " +str(self.metadata) + "\nListables: " +str(self.listables) #+ "\nChapters: "+str(self.chapters)
+        return "Metadata: " +str(self.metadata) 

-    def setReplace(self,replace):
-        for line in replace.splitlines():
-            if "=>" in line:
-                self.replacements.append(map( lambda x: x.strip(), line.split("=>") ))
-    
 def commaGroups(s):
    groups = []
    while s and s[-1].isdigit():
@@ -36,47 +36,12 @@ class BaseStoryWriter(Configurable):
    def getFormatExt():
        return '.bse'

-    def __init__(self, config, adapter):
-        Configurable.__init__(self, config)
-        self.setSectionOrder(adapter.getSiteDomain(),self.getFormatName())
-        # self.addConfigSection(adapter.getSiteDomain())
-        # self.addConfigSection(self.getFormatName())
-        # self.addConfigSection(adapter.getSiteDomain()+":"+self.getFormatName())
-        # self.addConfigSection("overrides")
+    def __init__(self, configuration, adapter):
+        Configurable.__init__(self, configuration)
        
        self.adapter = adapter
        self.story = adapter.getStoryMetadataOnly() # only cache the metadata initially.
        
-        self.story.setReplace(self.getConfig('replace_metadata'))
-        
-        self.validEntries = [
-            'category',
-            'genre',
-            'language',
-            'characters',
-            'series',
-            'status',
-            'datePublished',
-            'dateUpdated',
-            'dateCreated',
-            'rating',
-            'warnings',
-            'numChapters',
-            'numWords',
-            'site',
-            'storyId',
-            'authorId',
-            'extratags',
-            'title',
-            'storyUrl',
-            'description',
-            'author',
-            'authorUrl',
-            'formatname',
-            'formatext',
-            'siteabbrev',
-            'version']
-
        # fall back labels.
        self.titleLabels = {
            'category':'Category',
@@ -85,6 +50,7 @@ class BaseStoryWriter(Configurable):
            'status':'Status',
            'series':'Series',
            'characters':'Characters',
+            'ships':'Relationships',
            'datePublished':'Published',
            'dateUpdated':'Updated',
            'dateCreated':'Packaged',
@@ -108,12 +74,9 @@ class BaseStoryWriter(Configurable):
            }
        self.story.setMetadata('formatname',self.getFormatName())
        self.story.setMetadata('formatext',self.getFormatExt())
-        
-        for tag in self.getConfigList("extratags"):
-            self.story.addToList("extratags",tag)

-    def getMetadata(self,key):
-        return stripHTML(self.story.getMetadata(key))
+    def getMetadata(self,key, removeallentities=False):
+        return stripHTML(self.story.getMetadata(key, removeallentities))

    def getOutputFileName(self):
        if self.getConfig('zip_output'):
@@ -138,35 +101,42 @@ class BaseStoryWriter(Configurable):
        names as Story.metadata, but ENTRY should use label and value.
        """
        if self.getConfig("include_titlepage"):
-            self._write(out,START.substitute(self.story.metadata))
+            self._write(out,START.substitute(self.story.getAllMetadata()))

            if WIDE_ENTRY==None:
                WIDE_ENTRY=ENTRY

-            titleEntriesList = self.getConfigList("titlepage_entries")
+            titleEntriesList = self.getConfigList("titlepage_entries") + self.getConfigList("extra_titlepage_entries")
            wideTitleEntriesList = self.getConfigList("wide_titlepage_entries")

            for entry in titleEntriesList:
-                if entry in self.validEntries:
+                if self.isValidMetaEntry(entry):
                    if self.story.getMetadata(entry):
                        if entry in wideTitleEntriesList:
                            TEMPLATE=WIDE_ENTRY
                        else:
                            TEMPLATE=ENTRY
-                        if self.getConfigList(entry):
+                            
+                        if self.hasConfig(entry+"_label"):
                            label=self.getConfig(entry+"_label")
-                        else:
+                        elif entry in self.titleLabels:
+                            logging.debug("Using fallback label for %s_label"%entry)
                            label=self.titleLabels[entry]
+                        else:
+                            label="%s"%entry.title()
+                            logging.debug("No known label for %s, fallback to '%s'"%(entry,label))

                        # If the label for the title entry is empty, use the
-                        # 'no title' option if there is one.  
+                        # 'no title' option if there is one.
                        if label == "" and NO_TITLE_ENTRY:
                           TEMPLATE= NO_TITLE_ENTRY
                           
                        self._write(out,TEMPLATE.substitute({'label':label,
                                                             'value':self.story.getMetadata(entry)}))
+                else:
+                    self._write(out, entry)

-            self._write(out,END.substitute(self.story.metadata))
+            self._write(out,END.substitute(self.story.getAllMetadata()))

    def writeTOCPage(self, out, START, ENTRY, END):
        """
@@ -176,13 +146,13 @@ class BaseStoryWriter(Configurable):
        """
        # Only do TOC if there's more than one chapter and it's configured.
        if len(self.story.getChapters()) > 1 and self.getConfig("include_tocpage") and not self.metaonly :
-            self._write(out,START.substitute(self.story.metadata))
+            self._write(out,START.substitute(self.story.getAllMetadata()))

            for index, (title,html) in enumerate(self.story.getChapters()):
                if html:
                    self._write(out,ENTRY.substitute({'chapter':title, 'index':"%04d"%(index+1)}))

-            self._write(out,END.substitute(self.story.metadata))
+            self._write(out,END.substitute(self.story.getAllMetadata()))

    # if no outstream is given, write to file.
    def writeStory(self,outstream=None, metaonly=False, outfilename=None, forceOverwrite=False):
@@ -193,13 +163,15 @@ class BaseStoryWriter(Configurable):

        # minor cheat, tucking css into metadata.
        if self.getConfig("output_css"):
-            self.story.metadata["output_css"] = self.getConfig("output_css")
+            self.story.setMetadata("output_css",
+                                   self.getConfig("output_css"),
+                                   condremoveentities=False)
        else:
-            self.story.metadata["output_css"] = ''
+            self.story.setMetadata("output_css",'')
            
        if not outstream:
            close=True
-            logging.debug("Save directly to file: %s" % outfilename)
+            logging.info("Save directly to file: %s" % outfilename)
            if self.getConfig('make_directories'):
                path=""
                dirs = os.path.dirname(outfilename).split('/')
@@ -252,29 +224,6 @@ class BaseStoryWriter(Configurable):
        if close:
            outstream.close()

-    def getTags(self):
-        # set to avoid duplicates subject tags.
-        subjectset = set()
-        
-        if self.story.getMetadataRaw('dateUpdated'):
-            # Last Update tags for Bill.
-            self.story.addToList('lastupdate',self.story.getMetadataRaw('dateUpdated').strftime("Last Update Year/Month: %Y/%m"))
-            self.story.addToList('lastupdate',self.story.getMetadataRaw('dateUpdated').strftime("Last Update: %Y/%m/%d"))
-        
-        for entry in self.validEntries:
-            if entry in self.getConfigList("include_subject_tags") and \
-                    entry not in self.story.getLists() and \
-                    self.story.getMetadata(entry):
-                subjectset.add(self.getMetadata(entry))
-                
-        # listables all go into dc:subject tags, but only if they are configured.
-        for (name,lst) in self.story.getLists().iteritems():
-            if name in self.getConfigList("include_subject_tags"):
-                for tag in lst:
-                    subjectset.add(tag)
-
-        return list(subjectset)
-            
    def writeStoryImpl(self, out):
        "Must be overriden by sub classes."
        pass
@@ -52,7 +52,7 @@ class EpubWriter(BaseStoryWriter):
 <link href="stylesheet.css" type="text/css" charset="UTF-8" rel="stylesheet"/>
 </head>
 <body>
-<h3><a href="${storyUrl}">${title}</a> by <a href="${authorUrl}">${author}</a></h3>
+<h3><a href="${storyUrl}">${title}</a> by ${authorHTML}</h3>
 <div>
 ''')

@@ -79,7 +79,7 @@ ${value}<br />
 <link href="stylesheet.css" type="text/css" charset="UTF-8" rel="stylesheet"/>
 </head>
 <body>
-<h3><a href="${storyUrl}">${title}</a> by <a href="${authorUrl}">${author}</a></h3>
+<h3><a href="${storyUrl}">${title}</a> by ${authorHTML}</h3>
 <table class="full">
 ''')

@@ -140,6 +140,103 @@ ${value}<br />
 </html>
 ''')

+        self.EPUB_LOG_PAGE_START = string.Template('''<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head>
+<title>Update Log</title>
+<link href="stylesheet.css" type="text/css" charset="UTF-8" rel="stylesheet"/>
+</head>
+<body>
+<h3>Update Log</h3>
+''')
+
+        self.EPUB_LOG_ENTRY = string.Template('''
+<b>${label}:</b> <span id="${id}">${value}</span>
+''')
+                          
+        self.EPUB_LOG_PAGE_END = string.Template('''
+</body>
+</html>
+''')
+
+    def writeLogPage(self, out):
+        """
+       XXX 
+
+        
+        Write the log page, but only include entries that there's
+        metadata for.  START, ENTRY and END are expected to already by
+        string.Template().  START and END are expected to use the same
+        names as Story.metadata, but ENTRY should use id, label and value.
+        """
+        if self.getConfig("include_logpage"):
+
+            # if there's a self.story.logfile, there's an existing log
+            # to add to.
+            if self.story.logfile:
+                print("existing logfile found, appending")
+                print("existing data:%s"%self._getLastLogData(self.story.logfile))
+                replace_string = "</body>" # "</h3>"
+                self._write(out,self.story.logfile.replace(replace_string,self._makeLogEntry(self._getLastLogData(self.story.logfile))+replace_string))
+            else:
+                # otherwise, write a new one.
+                self._write(out,self.EPUB_LOG_PAGE_START.substitute(self.story.getAllMetadata()))
+                self._write(out,self._makeLogEntry())
+                self._write(out,self.EPUB_LOG_PAGE_END.substitute(self.story.getAllMetadata()))
+
+    # self parsing instead of Soup because it should be simple and not
+    # worth the overhead.
+    def _getLastLogData(self,logfile):
+        """
+        Make a dict() of the most recent(last) log entry for each piece of metadata.
+        Switch rindex to index to search from top instead of bottom.
+        """
+        values = {}
+        for entry in self.getConfigList("logpage_entries") + self.getConfigList("extra_logpage_entries"):
+            try:
+                # <span id="dateUpdated">1975-04-15</span>
+                span = '<span id="%s">'%entry
+                idx = logfile.rindex(span)+len(span)
+                values[entry] = logfile[idx:logfile.index('</span>',idx)]
+            except Exception, e:
+                #print("e:%s"%e)
+                pass
+
+        return values
+        
+    def _makeLogEntry(self, oldvalues={}):
+        retval = "<p class='log_entry'>"
+
+        for entry in self.getConfigList("logpage_entries") + self.getConfigList("extra_logpage_entries"):
+            if self.isValidMetaEntry(entry):
+                val = self.story.getMetadata(entry)
+                if val and ( entry not in oldvalues or val != oldvalues[entry] ):
+                    if self.hasConfig(entry+"_label"):
+                        label=self.getConfig(entry+"_label")
+                    elif entry in self.titleLabels:
+                        logging.debug("Using fallback label for %s_label"%entry)
+                        label=self.titleLabels[entry]
+                    else:
+                        label="%s"%entry.title()
+                        logging.debug("No known label for %s, fallback to '%s'"%(entry,label))
+
+                    retval = retval + self.EPUB_LOG_ENTRY.substitute({'id':entry,
+                                                                      'label':label,
+                                                                      'value':val})
+            else:
+                # could be useful for introducing extra text, but
+                # mostly it makes it easy to tell when you get the
+                # keyword wrong.
+                retval = retval + entry
+                
+        retval = retval + "</p><hr />"
+        
+        if self.getConfig('replace_hr'):
+            retval = retval.replace("<hr />","<div class='center'>* * *</div>")
+            
+        return retval
+                
    def writeStoryImpl(self, out):

        ## Python 2.5 ZipFile is rather more primative than later
@@ -185,7 +282,7 @@ ${value}<br />

        uniqueid= 'fanficdownloader-uid:%s-u%s-s%s' % (
            self.getMetadata('site'),
-            self.getMetadata('authorId'),
+            self.story.getList('authorId')[0],
            self.getMetadata('storyId'))
        
        contentdom = getDOMImplementation().createDocument(None, "package", None)
@@ -206,9 +303,15 @@ ${value}<br />
            metadata.appendChild(newTag(contentdom,"dc:title",text=self.getMetadata('title')))

        if self.getMetadata('author'):
-            metadata.appendChild(newTag(contentdom,"dc:creator",
-                                        attrs={"opf:role":"aut"},
-                                        text=self.getMetadata('author')))
+            if self.story.isList('author'):
+                for auth in self.story.getList('author'):
+                    metadata.appendChild(newTag(contentdom,"dc:creator",
+                                                attrs={"opf:role":"aut"},
+                                                text=auth))
+            else:
+                metadata.appendChild(newTag(contentdom,"dc:creator",
+                                            attrs={"opf:role":"aut"},
+                                            text=self.getMetadata('author')))

        metadata.appendChild(newTag(contentdom,"dc:contributor",text="fanficdownloader [http://fanficdownloader.googlecode.com]",attrs={"opf:role":"bkp"}))
        metadata.appendChild(newTag(contentdom,"dc:rights",text=""))
@@ -241,7 +344,7 @@ ${value}<br />
            metadata.appendChild(newTag(contentdom,"dc:description",text=
                                        self.getMetadata('description')))

-        for subject in self.getTags():
+        for subject in self.story.getSubjectTags():
            metadata.appendChild(newTag(contentdom,"dc:subject",text=subject))

                    
@@ -262,6 +365,37 @@ ${value}<br />
        items.append(("ncx","toc.ncx","application/x-dtbncx+xml",None)) ## we'll generate the toc.ncx file,
                                                                   ## but it needs to be in the items manifest.

+        guide = None
+        coverIO = None
+                
+        imgid = "image0000"
+        if not self.story.cover and self.story.oldcover:
+            print("writer_epub: no new cover, has old cover, write image.")
+            (oldcoverhtmlhref,
+             oldcoverhtmltype,
+             oldcoverhtmldata,
+             oldcoverimghref,
+             oldcoverimgtype,
+             oldcoverimgdata) = self.story.oldcover
+            outputepub.writestr(oldcoverhtmlhref,oldcoverhtmldata)
+            outputepub.writestr(oldcoverimghref,oldcoverimgdata)
+            
+            imgid = "image0"
+            items.append((imgid,
+                          oldcoverimghref,
+                          oldcoverimgtype,
+                          None))
+            items.append(("cover",oldcoverhtmlhref,oldcoverhtmltype,None))
+            itemrefs.append("cover")
+            metadata.appendChild(newTag(contentdom,"meta",{"content":"image0",
+                                                           "name":"cover"}))
+            guide = newTag(contentdom,"guide")
+            guide.appendChild(newTag(contentdom,"reference",attrs={"type":"cover",
+                                                                   "title":"Cover",
+                                                                   "href":oldcoverhtmlhref}))
+            
+            
+
        if self.getConfig('include_images'):
            imgcount=0
            for imgmap in self.story.getImgUrls():
@@ -276,9 +410,6 @@ ${value}<br />
        
        items.append(("style","OEBPS/stylesheet.css","text/css",None))

-        guide = None
-        coverIO = None
-
        if self.story.cover:
            # Note that the id of the cover xhmtl *must* be 'cover'
            # for it to work on Nook.
@@ -315,6 +446,11 @@ div { margin: 0pt; padding: 0pt; }
        if len(self.story.getChapters()) > 1 and self.getConfig("include_tocpage") and not self.metaonly :
            items.append(("toc_page","OEBPS/toc_page.xhtml","application/xhtml+xml","Table of Contents"))
            itemrefs.append("toc_page")
+
+        if self.getConfig("include_logpage"):
+            items.append(("log_page","OEBPS/log_page.xhtml","application/xhtml+xml","Update Log"))
+            itemrefs.append("log_page")
+            
        for index, (title,html) in enumerate(self.story.getChapters()):
            if html:
                i=index+1
@@ -347,8 +483,8 @@ div { margin: 0pt; padding: 0pt; }
        contentxml = contentdom.toxml(encoding='utf-8')
        
        # tweak for brain damaged Nook STR.  Nook insists on name before content.
-        contentxml = contentxml.replace('<meta content="image0000" name="cover"/>',
-                                        '<meta name="cover" content="image0000"/>')
+        contentxml = contentxml.replace('<meta content="%s" name="cover"/>'%imgid,
+                                        '<meta name="cover" content="%s"/>'%imgid)
        outputepub.writestr("content.opf",contentxml)

        contentdom.unlink()
@@ -405,7 +541,7 @@ div { margin: 0pt; padding: 0pt; }
        del tocncxdom

        # write stylesheet.css file.
-        outputepub.writestr("OEBPS/stylesheet.css",self.EPUB_CSS.substitute(self.story.metadata)) 
+        outputepub.writestr("OEBPS/stylesheet.css",self.EPUB_CSS.substitute(self.story.getAllMetadata())) 

        # write title page.
        if self.getConfig("titlepage_use_table"):
@@ -446,6 +582,13 @@ div { margin: 0pt; padding: 0pt; }
            outputepub.writestr("OEBPS/toc_page.xhtml",tocpageIO.getvalue())
        tocpageIO.close()

+        # write log page.
+        logpageIO = StringIO.StringIO()
+        self.writeLogPage(logpageIO)
+        if logpageIO.getvalue(): # will be false if no log page.
+            outputepub.writestr("OEBPS/log_page.xhtml",logpageIO.getvalue())
+        logpageIO.close()
+        
        for index, (title,html) in enumerate(self.story.getChapters()):
            if html:
                logging.debug('Writing chapter text for: %s' % title)
@@ -458,6 +601,9 @@ div { margin: 0pt; padding: 0pt; }
                outputepub.writestr("OEBPS/file%04d.xhtml"%(index+1),fullhtml.encode('utf-8'))
                del fullhtml

+        if self.story.calibrebookmark:
+            outputepub.writestr("META-INF/calibre_bookmarks.txt",self.story.calibrebookmark)
+
 	# declares all the files created by Windows.  otherwise, when
        # it runs in appengine, windows unzips the files as 000 perms.
        for zf in outputepub.filelist:
--- a/Show More
+++ b/Show More