Fix 'Status: Complete' showing up as a character on ffnet stories.

Moved tag FanFictionDownLoader-4.4.17 to changeset 6dca1af79eaf (from changeset af6b3a11df60)
Bump previous link.
2026-09-13 12:11:20 +08:00 · 2012-06-28 21:34:15 -05:00 · 2012-06-28 13:58:58 -05:00 · 2012-06-28 13:58:35 -05:00 · 2012-06-28 13:57:08 -05:00 · 2012-06-28 13:56:29 -05:00
104 changed files with 16693 additions and 1155 deletions
@@ -0,0 +1,78 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01//EN">
+<html>
+  <head>
+    <link href="/css/index.css" rel="stylesheet" type="text/css">
+    <title>FanFictionDownLoader (fanfiction.net, fanficauthors, fictionalley, ficwad to epub and HTML)</title>
+    <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
+    <script type="text/javascript">
+
+      var _gaq = _gaq || [];
+      _gaq.push(['_setAccount', 'UA-12136939-1']);
+      _gaq.push(['_trackPageview']);
+
+      (function() {
+      var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true;
+      ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
+      var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s);
+      })();
+
+    </script>
+  </head>
+  <body>
+    <div id='main'>
+      <h1>
+	<a href="/" style="text-decoration: none; color: black;">FanFictionDownLoader</a>
+      </h1>
+      
+      <script type="text/javascript"><!--
+                                        google_ad_client = "ca-pub-0320924304307555";
+                                        /* Standard */
+                                        google_ad_slot = "8974025478";
+                                        google_ad_width = 468;
+                                        google_ad_height = 60;
+                                        //-->
+      </script>
+      <script type="text/javascript"
+              src="http://pagead2.googlesyndication.com/pagead/show_ads.js">
+      </script>
+      <!--			<div id='yourfile'> -->
+      {{yourfile}}
+      <!--			</div> -->
+      
+      <div id='helpbox'>
+	{% for fic in fics %}
+	<p>
+  	  <a href="{{ fic.url }}" title="Link to original story"><span class="recent"><i>{{ fic.title }}</i></span></a>
+	      by <a href="{{ fic.authorUrl }}">{{ fic.author }}</a>  <b>Download Count:</b> {{ fic.count }} <br />
+	  <b>Word Count:</b> {{ fic.numWords }} <b>Chapter Count:</b> {{ fic.numChapters }}<br />
+	  {% if fic.category %} <b>Categories:</b> {{ fic.category }} <br /> {% endif %}
+	  {% if fic.genre %} <b>Genres:</b> {{ fic.genre }} <br /> {% endif %}
+	  {% if fic.language %} <b>Language:</b> {{ fic.language }} <br /> {% endif %}
+	  {% if fic.series %} <b>Series:</b> {{ fic.series }} <br /> {% endif %}
+	  {% if fic.characters %} <b>Characters:</b> {{ fic.characters }} <br /> {% endif %}
+	  {% if fic.status %} <b>Status:</b> {{ fic.status }} <br /> {% endif %}
+	  {% if fic.datePublished %} <b>Published:</b> {{ fic.datePublished }} <br /> {% endif %}
+	  {% if fic.dateUpdated %} <b>Last Updated:</b> {{ fic.dateUpdated }} <br /> {% endif %}
+	  {% if fic.dateCreated %} <b>Last Downloaded:</b> {{ fic.dateCreated }} <br /> {% endif %}
+	  {% if fic.rating %} <b>Rating:</b> {{ fic.rating }} <br /> {% endif %}
+	  {% if fic.warnings %} <b>Warnings:</b> {{ fic.warnings }} <br /> {% endif %}
+	  {% if fic.description %} <b>Summary:</b> {{ fic.description }} <br /> {% endif %}
+	</p>
+	{% endfor %}
+      </div>
+      
+      <script type="text/javascript"><!--
+                                        google_ad_client = "ca-pub-0320924304307555";
+                                        /* Standard */
+                                        google_ad_slot = "8974025478";
+                                        google_ad_width = 468;
+                                        google_ad_height = 60;
+                                        //-->
+      </script>
+      <script type="text/javascript"
+              src="http://pagead2.googlesyndication.com/pagead/show_ads.js">
+      </script>				
+      
+    </div>
+  </body>
+</html>
@@ -1,20 +1,22 @@
-# fanfictionloader ffd-retief
-application: fanfictionloader
-version: 4-0-7
-runtime: python
+# ffd-retief-hrd fanfictiondownloader
+application: fanfictiondownloader
+version: 4-4-17
+runtime: python27
 api_version: 1
+threadsafe: true

 handlers:
+
 - url: /r3m0v3r.*
-  script: utils/remover.py
+  script: utils.remover.app
  login: admin

 - url: /tally.*
-  script: utils/tally.py
+  script: utils.tally.app
  login: admin

 - url: /fdownloadtask
-  script: main.py
+  script: main.app
  login: admin
  
 - url: /css
@@ -31,7 +33,14 @@ handlers:
  upload: static/favicon\.ico

 - url: /.*
-  script: main.py
+  script: main.app

-builtins:
- datastore_admin: on
+#builtins:
+#- datastore_admin: on
+
+libraries:
+- name: django
+  version: "1.2"
+
+- name: PIL
+  version: "1.1.7"
@@ -0,0 +1,90 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+# -*- coding: utf-8 -*-
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2011, Jim Miller'
+__docformat__ = 'restructuredtext en'
+
+# The class that all Interface Action plugin wrappers must inherit from
+from calibre.customize import InterfaceActionBase
+
+## Apparently the name for this class doesn't matter--it was still
+## 'demo' for the first few versions.
+class FanFictionDownLoaderBase(InterfaceActionBase):
+    '''
+    This class is a simple wrapper that provides information about the
+    actual plugin class. The actual interface plugin class is called
+    InterfacePlugin and is defined in the ffdl_plugin.py file, as
+    specified in the actual_plugin field below.
+
+    The reason for having two classes is that it allows the command line
+    calibre utilities to run without needing to load the GUI libraries.
+    '''
+    name                = 'FanFictionDownLoader'
+    description         = 'UI plugin to download FanFiction stories from various sites.'
+    supported_platforms = ['windows', 'osx', 'linux']
+    author              = 'Jim Miller'
+    version             = (1, 5, 39)
+    minimum_calibre_version = (0, 8, 30)
+
+    #: This field defines the GUI plugin class that contains all the code
+    #: that actually does something. Its format is module_path:class_name
+    #: The specified class must be defined in the specified module.
+    actual_plugin       = 'calibre_plugins.fanfictiondownloader_plugin.ffdl_plugin:FanFictionDownLoaderPlugin'
+
+    def is_customizable(self):
+        '''
+        This method must return True to enable customization via
+        Preferences->Plugins
+        '''
+        return True
+
+    def config_widget(self):
+        '''
+        Implement this method and :meth:`save_settings` in your plugin to
+        use a custom configuration dialog.
+
+        This method, if implemented, must return a QWidget. The widget can have
+        an optional method validate() that takes no arguments and is called
+        immediately after the user clicks OK. Changes are applied if and only
+        if the method returns True.
+
+        If for some reason you cannot perform the configuration at this time,
+        return a tuple of two strings (message, details), these will be
+        displayed as a warning dialog to the user and the process will be
+        aborted.
+
+        The base class implementation of this method raises NotImplementedError
+        so by default no user configuration is possible.
+        '''
+        # It is important to put this import statement here rather than at the
+        # top of the module as importing the config class will also cause the
+        # GUI libraries to be loaded, which we do not want when using calibre
+        # from the command line
+        from calibre_plugins.fanfictiondownloader_plugin.config import ConfigWidget
+        return ConfigWidget(self.actual_plugin_)
+
+    def save_settings(self, config_widget):
+        '''
+        Save the settings specified by the user with config_widget.
+
+        :param config_widget: The widget returned by :meth:`config_widget`.
+        '''
+        config_widget.save_settings()
+
+        # Apply the changes
+        ac = self.actual_plugin_
+        if ac is not None:
+            ac.apply_settings()
+
+# For testing, run from command line with this:
+# calibre-debug -e __init__.py
+# 
+if __name__ == '__main__':
+    from PyQt4.Qt import QApplication
+    from calibre.gui2.preferences import test_widget
+    app = QApplication([])
+    test_widget('Advanced', 'Plugins')
@@ -0,0 +1,28 @@
+<hr />
+
+<p>Plugin created by Jim Miller, borrowing heavily from Grant Drake's
+'<a href="http://www.mobileread.com/forums/showthread.php?t=134856">Reading List</a>',
+'<a href="http://www.mobileread.com/forums/showthread.php?t=126727">Extract ISBN</a>' and
+'<a href="http://www.mobileread.com/forums/showthread.php?t=134000">Count Pages</a>'
+plugins.</p>
+
+<p>
+Calibre officially distributes plugins from the mobileread.com forum site.
+The official distro channel for this plugin is there: <a href="http://www.mobileread.com/forums/showthread.php?t=163261">FanFictionDownLoader</a>
+</p>
+
+<p> I also monitor the
+<a href="http://groups.google.com/group/fanfic-downloader">general users
+group</a> for the downloader.  That covers the web application and CLI, too.
+</p>
+
+The source for this plugin is available at it's 
+<a href="http://code.google.com/p/fanficdownloader">project home</a>.
+<hr />
+
+<p>
+See the <a href="http://code.google.com/p/fanficdownloader/wiki/FanFictionDownloaderSupportedsites">list of supported sites</a>.
+</p>
+<p>
+Read the <a href="http://code.google.com/p/fanficdownloader/wiki/FanFictionDownloaderFAQs">FAQs</a>.
+</p>
@@ -0,0 +1,447 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2011, Grant Drake <grant.drake@gmail.com>'
+__docformat__ = 'restructuredtext en'
+
+import os
+from PyQt4 import QtGui
+from PyQt4.Qt import (Qt, QIcon, QPixmap, QLabel, QDialog, QHBoxLayout,
+                      QTableWidgetItem, QFont, QLineEdit, QComboBox,
+                      QVBoxLayout, QDialogButtonBox, QStyledItemDelegate, QDateTime)
+from calibre.constants import iswindows
+from calibre.gui2 import gprefs, error_dialog, UNDEFINED_QDATETIME
+from calibre.gui2.actions import menu_action_unique_name
+from calibre.gui2.keyboard import ShortcutConfig
+from calibre.utils.config import config_dir
+from calibre.utils.date import now, format_date, qt_to_dt, UNDEFINED_DATE
+
+# Global definition of our plugin name. Used for common functions that require this.
+plugin_name = None
+# Global definition of our plugin resources. Used to share between the xxxAction and xxxBase
+# classes if you need any zip images to be displayed on the configuration dialog.
+plugin_icon_resources = {}
+
+
+def set_plugin_icon_resources(name, resources):
+    '''
+    Set our global store of plugin name and icon resources for sharing between
+    the InterfaceAction class which reads them and the ConfigWidget
+    if needed for use on the customization dialog for this plugin.
+    '''
+    global plugin_icon_resources, plugin_name
+    plugin_name = name
+    plugin_icon_resources = resources
+
+
+def get_icon(icon_name):
+    '''
+    Retrieve a QIcon for the named image from the zip file if it exists,
+    or if not then from Calibre's image cache.
+    '''
+    if icon_name:
+        pixmap = get_pixmap(icon_name)
+        if pixmap is None:
+            # Look in Calibre's cache for the icon
+            return QIcon(I(icon_name))
+        else:
+            return QIcon(pixmap)
+    return QIcon()
+
+
+def get_pixmap(icon_name):
+    '''
+    Retrieve a QPixmap for the named image
+    Any icons belonging to the plugin must be prefixed with 'images/'
+    '''
+    global plugin_icon_resources, plugin_name
+
+    if not icon_name.startswith('images/'):
+        # We know this is definitely not an icon belonging to this plugin
+        pixmap = QPixmap()
+        pixmap.load(I(icon_name))
+        return pixmap
+
+    # Check to see whether the icon exists as a Calibre resource
+    # This will enable skinning if the user stores icons within a folder like:
+    # ...\AppData\Roaming\calibre\resources\images\Plugin Name\
+    if plugin_name:
+        local_images_dir = get_local_images_dir(plugin_name)
+        local_image_path = os.path.join(local_images_dir, icon_name.replace('images/', ''))
+        if os.path.exists(local_image_path):
+            pixmap = QPixmap()
+            pixmap.load(local_image_path)
+            return pixmap
+
+    # As we did not find an icon elsewhere, look within our zip resources
+    if icon_name in plugin_icon_resources:
+        pixmap = QPixmap()
+        pixmap.loadFromData(plugin_icon_resources[icon_name])
+        return pixmap
+    return None
+
+
+def get_local_images_dir(subfolder=None):
+    '''
+    Returns a path to the user's local resources/images folder
+    If a subfolder name parameter is specified, appends this to the path
+    '''
+    images_dir = os.path.join(config_dir, 'resources/images')
+    if subfolder:
+        images_dir = os.path.join(images_dir, subfolder)
+    if iswindows:
+        images_dir = os.path.normpath(images_dir)
+    return images_dir
+
+
+def create_menu_item(ia, parent_menu, menu_text, image=None, tooltip=None,
+                     shortcut=(), triggered=None, is_checked=None):
+    '''
+    Create a menu action with the specified criteria and action
+    Note that if no shortcut is specified, will not appear in Preferences->Keyboard
+    This method should only be used for actions which either have no shortcuts,
+    or register their menus only once. Use create_menu_action_unique for all else.
+    '''
+    if shortcut is not None:
+        if len(shortcut) == 0:
+            shortcut = ()
+        else:
+            shortcut = _(shortcut)
+    ac = ia.create_action(spec=(menu_text, None, tooltip, shortcut),
+        attr=menu_text)
+    if image:
+        ac.setIcon(get_icon(image))
+    if triggered is not None:
+        ac.triggered.connect(triggered)
+    if is_checked is not None:
+        ac.setCheckable(True)
+        if is_checked:
+            ac.setChecked(True)
+
+    parent_menu.addAction(ac)
+    return ac
+
+
+def create_menu_action_unique(ia, parent_menu, menu_text, image=None, tooltip=None,
+                       shortcut=None, triggered=None, is_checked=None, shortcut_name=None,
+                       unique_name=None):
+    '''
+    Create a menu action with the specified criteria and action, using the new
+    InterfaceAction.create_menu_action() function which ensures that regardless of
+    whether a shortcut is specified it will appear in Preferences->Keyboard
+    '''
+    orig_shortcut = shortcut
+    kb = ia.gui.keyboard
+    if unique_name is None:
+        unique_name = menu_text
+    if not shortcut == False:
+        full_unique_name = menu_action_unique_name(ia, unique_name)
+        if full_unique_name in kb.shortcuts:
+            shortcut = False
+        else:
+            if shortcut is not None and not shortcut == False:
+                if len(shortcut) == 0:
+                    shortcut = None
+                else:
+                    shortcut = _(shortcut)
+
+    if shortcut_name is None:
+        shortcut_name = menu_text.replace('&','')
+
+    ac = ia.create_menu_action(parent_menu, unique_name, menu_text, icon=None, shortcut=shortcut,
+        description=tooltip, triggered=triggered, shortcut_name=shortcut_name)
+    if shortcut == False and not orig_shortcut == False:
+        if ac.calibre_shortcut_unique_name in ia.gui.keyboard.shortcuts:
+            kb.replace_action(ac.calibre_shortcut_unique_name, ac)
+    if image:
+        ac.setIcon(get_icon(image))
+    if is_checked is not None:
+        ac.setCheckable(True)
+        if is_checked:
+            ac.setChecked(True)
+    return ac
+
+
+def swap_author_names(author):
+    if author.find(',') == -1:
+        return author
+    name_parts = author.strip().partition(',')
+    return name_parts[2].strip() + ' ' + name_parts[0]
+
+
+def get_library_uuid(db):
+    try:
+        library_uuid = db.library_id
+    except:
+        library_uuid = ''
+    return library_uuid
+
+
+class ImageLabel(QLabel):
+
+    def __init__(self, parent, icon_name, size=16):
+        QLabel.__init__(self, parent)
+        pixmap = get_pixmap(icon_name)
+        self.setPixmap(pixmap)
+        self.setMaximumSize(size, size)
+        self.setScaledContents(True)
+
+
+class ImageTitleLayout(QHBoxLayout):
+    '''
+    A reusable layout widget displaying an image followed by a title
+    '''
+    def __init__(self, parent, icon_name, title):
+        QHBoxLayout.__init__(self)
+        title_image_label = QLabel(parent)
+        pixmap = get_pixmap(icon_name)
+        if pixmap is None:
+            pixmap = get_pixmap('library.png')
+            # error_dialog(parent, _('Restart required'),
+            #              _('You must restart Calibre before using this plugin!'), show=True)
+        else:
+            title_image_label.setPixmap(pixmap)
+        title_image_label.setMaximumSize(32, 32)
+        title_image_label.setScaledContents(True)
+        self.addWidget(title_image_label)
+
+        title_font = QFont()
+        title_font.setPointSize(16)
+        shelf_label = QLabel(title, parent)
+        shelf_label.setFont(title_font)
+        self.addWidget(shelf_label)
+        self.insertStretch(-1)
+
+
+class SizePersistedDialog(QDialog):
+    '''
+    This dialog is a base class for any dialogs that want their size/position
+    restored when they are next opened.
+    '''
+    def __init__(self, parent, unique_pref_name):
+        QDialog.__init__(self, parent)
+        self.unique_pref_name = unique_pref_name
+        self.geom = gprefs.get(unique_pref_name, None)
+        self.finished.connect(self.dialog_closing)
+
+    def resize_dialog(self):
+        if self.geom is None:
+            self.resize(self.sizeHint())
+        else:
+            self.restoreGeometry(self.geom)
+
+    def dialog_closing(self, result):
+        geom = bytearray(self.saveGeometry())
+        gprefs[self.unique_pref_name] = geom
+
+
+class ReadOnlyTableWidgetItem(QTableWidgetItem):
+
+    def __init__(self, text):
+        if text is None:
+            text = ''
+        QTableWidgetItem.__init__(self, text, QtGui.QTableWidgetItem.UserType)
+        self.setFlags(Qt.ItemIsSelectable|Qt.ItemIsEnabled)
+
+
+class RatingTableWidgetItem(QTableWidgetItem):
+
+    def __init__(self, rating, is_read_only=False):
+        QTableWidgetItem.__init__(self, '', QtGui.QTableWidgetItem.UserType)
+        self.setData(Qt.DisplayRole, rating)
+        if is_read_only:
+            self.setFlags(Qt.ItemIsSelectable|Qt.ItemIsEnabled)
+
+
+class DateTableWidgetItem(QTableWidgetItem):
+
+    def __init__(self, date_read, is_read_only=False, default_to_today=False):
+        if date_read == UNDEFINED_DATE and default_to_today:
+            date_read = now()
+        if is_read_only:
+            QTableWidgetItem.__init__(self, format_date(date_read, None), QtGui.QTableWidgetItem.UserType)
+            self.setFlags(Qt.ItemIsSelectable|Qt.ItemIsEnabled)
+        else:
+            QTableWidgetItem.__init__(self, '', QtGui.QTableWidgetItem.UserType)
+            self.setData(Qt.DisplayRole, QDateTime(date_read))
+
+
+class NoWheelComboBox(QComboBox):
+
+    def wheelEvent (self, event):
+        # Disable the mouse wheel on top of the combo box changing selection as plays havoc in a grid
+        event.ignore()
+
+
+class CheckableTableWidgetItem(QTableWidgetItem):
+
+    def __init__(self, checked=False, is_tristate=False):
+        QTableWidgetItem.__init__(self, '')
+        self.setFlags(Qt.ItemFlags(Qt.ItemIsSelectable | Qt.ItemIsUserCheckable | Qt.ItemIsEnabled ))
+        if is_tristate:
+            self.setFlags(self.flags() | Qt.ItemIsTristate)
+        if checked:
+            self.setCheckState(Qt.Checked)
+        else:
+            if is_tristate and checked is None:
+                self.setCheckState(Qt.PartiallyChecked)
+            else:
+                self.setCheckState(Qt.Unchecked)
+
+    def get_boolean_value(self):
+        '''
+        Return a boolean value indicating whether checkbox is checked
+        If this is a tristate checkbox, a partially checked value is returned as None
+        '''
+        if self.checkState() == Qt.PartiallyChecked:
+            return None
+        else:
+            return self.checkState() == Qt.Checked
+
+
+class TextIconWidgetItem(QTableWidgetItem):
+
+    def __init__(self, text, icon):
+        QTableWidgetItem.__init__(self, text)
+        if icon:
+            self.setIcon(icon)
+
+
+class ReadOnlyTextIconWidgetItem(ReadOnlyTableWidgetItem):
+
+    def __init__(self, text, icon):
+        ReadOnlyTableWidgetItem.__init__(self, text)
+        if icon:
+            self.setIcon(icon)
+
+
+class ReadOnlyLineEdit(QLineEdit):
+
+    def __init__(self, text, parent):
+        if text is None:
+            text = ''
+        QLineEdit.__init__(self, text, parent)
+        self.setEnabled(False)
+
+
+class KeyValueComboBox(QComboBox):
+
+    def __init__(self, parent, values, selected_key):
+        QComboBox.__init__(self, parent)
+        self.values = values
+        self.populate_combo(selected_key)
+
+    def populate_combo(self, selected_key):
+        self.clear()
+        selected_idx = idx = -1
+        for key, value in self.values.iteritems():
+            idx = idx + 1
+            self.addItem(value)
+            if key == selected_key:
+                selected_idx = idx
+        self.setCurrentIndex(selected_idx)
+
+    def selected_key(self):
+        for key, value in self.values.iteritems():
+            if value == unicode(self.currentText()).strip():
+                return key
+
+
+class CustomColumnComboBox(QComboBox):
+
+    def __init__(self, parent, custom_columns, selected_column, initial_items=['']):
+        QComboBox.__init__(self, parent)
+        self.populate_combo(custom_columns, selected_column, initial_items)
+
+    def populate_combo(self, custom_columns, selected_column, initial_items=['']):
+        self.clear()
+        self.column_names = initial_items
+        if len(initial_items) > 0:
+            self.addItems(initial_items)
+        selected_idx = 0
+        for idx, value in enumerate(initial_items):
+            if value == selected_column:
+                selected_idx = idx
+        for key in sorted(custom_columns.keys()):
+            self.column_names.append(key)
+            self.addItem('%s (%s)'%(key, custom_columns[key]['name']))
+            if key == selected_column:
+                selected_idx = len(self.column_names) - 1
+        self.setCurrentIndex(selected_idx)
+
+    def get_selected_column(self):
+        return self.column_names[self.currentIndex()]
+
+
+class KeyboardConfigDialog(SizePersistedDialog):
+    '''
+    This dialog is used to allow editing of keyboard shortcuts.
+    '''
+    def __init__(self, gui, group_name):
+        SizePersistedDialog.__init__(self, gui, 'Keyboard shortcut dialog')
+        self.gui = gui
+        self.setWindowTitle('Keyboard shortcuts')
+        layout = QVBoxLayout(self)
+        self.setLayout(layout)
+
+        self.keyboard_widget = ShortcutConfig(self)
+        layout.addWidget(self.keyboard_widget)
+        self.group_name = group_name
+
+        button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
+        button_box.accepted.connect(self.commit)
+        button_box.rejected.connect(self.reject)
+        layout.addWidget(button_box)
+
+        # Cause our dialog size to be restored from prefs or created on first usage
+        self.resize_dialog()
+        self.initialize()
+
+    def initialize(self):
+        self.keyboard_widget.initialize(self.gui.keyboard)
+        self.keyboard_widget.highlight_group(self.group_name)
+
+    def commit(self):
+        self.keyboard_widget.commit()
+        self.accept()
+
+
+class DateDelegate(QStyledItemDelegate):
+    '''
+    Delegate for dates. Because this delegate stores the
+    format as an instance variable, a new instance must be created for each
+    column. This differs from all the other delegates.
+    '''
+    def __init__(self, parent):
+        QStyledItemDelegate.__init__(self, parent)
+        self.format = 'dd MMM yyyy'
+
+    def displayText(self, val, locale):
+        d = val.toDateTime()
+        if d <= UNDEFINED_QDATETIME:
+            return ''
+        return format_date(qt_to_dt(d, as_utc=False), self.format)
+
+    def createEditor(self, parent, option, index):
+        qde = QStyledItemDelegate.createEditor(self, parent, option, index)
+        qde.setDisplayFormat(self.format)
+        qde.setMinimumDateTime(UNDEFINED_QDATETIME)
+        qde.setSpecialValueText(_('Undefined'))
+        qde.setCalendarPopup(True)
+        return qde
+
+    def setEditorData(self, editor, index):
+        val = index.model().data(index, Qt.DisplayRole).toDateTime()
+        if val is None or val == UNDEFINED_QDATETIME:
+            val = now()
+        editor.setDateTime(val)
+
+    def setModelData(self, editor, model, index):
+        val = editor.dateTime()
+        if val <= UNDEFINED_QDATETIME:
+            model.setData(index, UNDEFINED_QDATETIME, Qt.EditRole)
+        else:
+            model.setData(index, QDateTime(val), Qt.EditRole)
@@ -0,0 +1,710 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2011, Jim Miller'
+__docformat__ = 'restructuredtext en'
+
+import traceback, copy
+
+from PyQt4.Qt import (QDialog, QWidget, QVBoxLayout, QHBoxLayout, QLabel, QLineEdit, QFont, QWidget,
+                      QTextEdit, QComboBox, QCheckBox, QPushButton, QTabWidget, QVariant, QScrollArea)
+
+from calibre.gui2 import dynamic, info_dialog
+from calibre.utils.config import JSONConfig
+from calibre.gui2.ui import get_gui
+
+from calibre_plugins.fanfictiondownloader_plugin.dialogs \
+    import (UPDATE, UPDATEALWAYS, OVERWRITE, collision_order)
+
+from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.adapters import getConfigSections
+
+from calibre_plugins.fanfictiondownloader_plugin.common_utils \
+    import ( get_library_uuid, KeyboardConfigDialog )
+
+from calibre.gui2.complete import MultiCompleteLineEdit
+
+# This is where all preferences for this plugin will be stored
+# Remember that this name (i.e. plugins/fanfictiondownloader_plugin) is also
+# in a global namespace, so make it as unique as possible.
+# You should always prefix your config file name with plugins/,
+# so as to ensure you dont accidentally clobber a calibre config file
+all_prefs = JSONConfig('plugins/fanfictiondownloader_plugin')
+
+# Set defaults used by all.  Library specific settings continue to
+# take from here.
+all_prefs.defaults['personal.ini'] = get_resources('plugin-example.ini')
+
+all_prefs.defaults['updatemeta'] = True
+all_prefs.defaults['updatecover'] = False
+all_prefs.defaults['updateepubcover'] = False
+all_prefs.defaults['keeptags'] = False
+all_prefs.defaults['urlsfromclip'] = True
+all_prefs.defaults['updatedefault'] = True
+all_prefs.defaults['fileform'] = 'epub'
+all_prefs.defaults['collision'] = OVERWRITE
+all_prefs.defaults['deleteotherforms'] = False
+all_prefs.defaults['adddialogstaysontop'] = False
+all_prefs.defaults['includeimages'] = False
+all_prefs.defaults['lookforurlinhtml'] = False
+all_prefs.defaults['injectseries'] = False
+
+all_prefs.defaults['send_lists'] = ''
+all_prefs.defaults['read_lists'] = ''
+all_prefs.defaults['addtolists'] = False
+all_prefs.defaults['addtoreadlists'] = False
+all_prefs.defaults['addtolistsonread'] = False
+
+all_prefs.defaults['gcnewonly'] = False
+all_prefs.defaults['gc_site_settings'] = {}
+all_prefs.defaults['allow_gc_from_ini'] = True
+
+all_prefs.defaults['custom_cols'] = {}
+
+# The list of settings to copy from all_prefs or the previous library
+# when config is called for the first time on a library.
+copylist = ['personal.ini',
+            'updatemeta',
+            'updatecover',
+            'updateepubcover',
+            'keeptags',
+            'urlsfromclip',
+            'updatedefault',
+            'fileform',
+            'collision',
+            'deleteotherforms',
+            'adddialogstaysontop',
+            'includeimages',
+            'lookforurlinhtml',
+            'injectseries',
+            'gcnewonly',
+            'gc_site_settings',
+            'allow_gc_from_ini']
+
+# fake out so I don't have to change the prefs calls anywhere.  The
+# Java programmer in me is offended by op-overloading, but it's very
+# tidy.
+class PrefsFacade():
+    def __init__(self,all_prefs):
+        self.all_prefs = all_prefs
+        self.lastlibid = None
+
+    def _get_copylist_prefs(self,frompref):
+        return filter( lambda x : x[0] in copylist, frompref.items() )
+        
+    def _get_prefs(self):
+        libraryid = get_library_uuid(get_gui().current_db)
+        if libraryid not in self.all_prefs:
+            if self.lastlibid == None:
+                self.all_prefs[libraryid] = dict(self._get_copylist_prefs(self.all_prefs))
+            else:
+                self.all_prefs[libraryid] = dict(self._get_copylist_prefs(self.all_prefs[self.lastlibid]))
+                
+        self.lastlibid = libraryid
+            
+        return self.all_prefs[libraryid]
+
+    def _save_prefs(self,prefs):
+        libraryid = get_library_uuid(get_gui().current_db)
+        self.all_prefs[libraryid] = prefs
+        
+    def __getitem__(self,k):            
+        prefs = self._get_prefs()
+        if k not in prefs:
+            # pulls from all_prefs.defaults automatically if not set
+            # in all_prefs
+            return self.all_prefs[k]
+        return prefs[k]
+
+    def __setitem__(self,k,v):
+        prefs = self._get_prefs()
+        prefs[k]=v
+        self._save_prefs(prefs)
+
+    # to be avoided--can cause unexpected results as possibly ancient
+    # all_pref settings may be pulled.
+    def __delitem__(self,k):
+        prefs = self._get_prefs()
+        del prefs[k]
+        self._save_prefs(prefs)
+
+prefs = PrefsFacade(all_prefs)
+    
+class ConfigWidget(QWidget):
+
+    def __init__(self, plugin_action):
+        QWidget.__init__(self)
+        self.plugin_action = plugin_action
+        
+        self.l = QVBoxLayout()
+        self.setLayout(self.l)
+
+        label = QLabel('<a href="http://code.google.com/p/fanficdownloader/wiki/FanFictionDownloaderSupportedsites">List of Supported Sites</a> -- <a href="http://code.google.com/p/fanficdownloader/wiki/FanFictionDownloaderFAQs">FAQs</a>')
+        label.setOpenExternalLinks(True)
+        self.l.addWidget(label)
+
+        tab_widget = QTabWidget(self)
+        self.l.addWidget(tab_widget)
+
+        self.basic_tab = BasicTab(self, plugin_action)
+        tab_widget.addTab(self.basic_tab, 'Basic')
+
+        self.personalini_tab = PersonalIniTab(self, plugin_action)
+        tab_widget.addTab(self.personalini_tab, 'personal.ini')
+        
+        self.readinglist_tab = ReadingListTab(self, plugin_action)
+        tab_widget.addTab(self.readinglist_tab, 'Reading Lists')
+        if 'Reading List' not in plugin_action.gui.iactions:
+            self.readinglist_tab.setEnabled(False)
+
+        self.generatecover_tab = GenerateCoverTab(self, plugin_action)
+        tab_widget.addTab(self.generatecover_tab, 'Generate Cover')
+        if 'Generate Cover' not in plugin_action.gui.iactions:
+            self.generatecover_tab.setEnabled(False)
+
+        self.columns_tab = ColumnsTab(self, plugin_action)
+        tab_widget.addTab(self.columns_tab, 'Custom Columns')
+
+        self.other_tab = OtherTab(self, plugin_action)
+        tab_widget.addTab(self.other_tab, 'Other')
+
+
+    def save_settings(self):
+
+        # basic
+        prefs['fileform'] = unicode(self.basic_tab.fileform.currentText())
+        prefs['collision'] = unicode(self.basic_tab.collision.currentText())
+        prefs['updatemeta'] = self.basic_tab.updatemeta.isChecked()
+        prefs['updatecover'] = self.basic_tab.updatecover.isChecked()
+        prefs['updateepubcover'] = self.basic_tab.updateepubcover.isChecked()
+        prefs['keeptags'] = self.basic_tab.keeptags.isChecked()
+        prefs['urlsfromclip'] = self.basic_tab.urlsfromclip.isChecked()
+        prefs['updatedefault'] = self.basic_tab.updatedefault.isChecked()
+        prefs['deleteotherforms'] = self.basic_tab.deleteotherforms.isChecked()
+        prefs['adddialogstaysontop'] = self.basic_tab.adddialogstaysontop.isChecked()
+        prefs['includeimages'] = self.basic_tab.includeimages.isChecked()
+        prefs['lookforurlinhtml'] = self.basic_tab.lookforurlinhtml.isChecked()
+        prefs['injectseries'] = self.basic_tab.injectseries.isChecked()
+
+        if self.readinglist_tab:
+            # lists
+            prefs['send_lists'] = ', '.join(map( lambda x : x.strip(), filter( lambda x : x.strip() != '', unicode(self.readinglist_tab.send_lists_box.text()).split(','))))
+            prefs['read_lists'] = ', '.join(map( lambda x : x.strip(), filter( lambda x : x.strip() != '', unicode(self.readinglist_tab.read_lists_box.text()).split(','))))
+            # print("send_lists: %s"%prefs['send_lists'])
+            # print("read_lists: %s"%prefs['read_lists'])
+            prefs['addtolists'] = self.readinglist_tab.addtolists.isChecked()
+            prefs['addtoreadlists'] = self.readinglist_tab.addtoreadlists.isChecked()
+            prefs['addtolistsonread'] = self.readinglist_tab.addtolistsonread.isChecked()
+
+        # personal.ini
+        ini = unicode(self.personalini_tab.ini.toPlainText())
+        if ini:
+            prefs['personal.ini'] = ini
+        else:
+            # if they've removed everything, reset to default.
+            prefs['personal.ini'] = get_resources('plugin-example.ini')
+
+        # Generate Covers tab
+        prefs['gcnewonly'] = self.generatecover_tab.gcnewonly.isChecked()
+        gc_site_settings = {}
+        for (site,combo) in self.generatecover_tab.gc_dropdowns.iteritems():
+            val = unicode(combo.itemData(combo.currentIndex()).toString())
+            if val != 'none':
+                gc_site_settings[site] = val
+                #print("gc_site_settings[%s]:%s"%(site,gc_site_settings[site]))
+        prefs['gc_site_settings'] = gc_site_settings
+        prefs['allow_gc_from_ini'] = self.generatecover_tab.allow_gc_from_ini.isChecked()
+
+        # Custom Columns tab
+        colsmap = {}
+        for (col,combo) in self.columns_tab.custcol_dropdowns.iteritems():
+            val = unicode(combo.itemData(combo.currentIndex()).toString())
+            if val != 'none':
+                colsmap[col] = val
+                #print("colsmap[%s]:%s"%(col,colsmap[col]))
+        prefs['custom_cols'] = colsmap
+
+    def edit_shortcuts(self):
+        self.save_settings()
+        # Force the menus to be rebuilt immediately, so we have all our actions registered
+        self.plugin_action.rebuild_menus()
+        d = KeyboardConfigDialog(self.plugin_action.gui, self.plugin_action.action_spec[0])
+        if d.exec_() == d.Accepted:
+            self.plugin_action.gui.keyboard.finalize()
+
+class BasicTab(QWidget):
+
+    def __init__(self, parent_dialog, plugin_action):
+        self.parent_dialog = parent_dialog
+        self.plugin_action = plugin_action
+        QWidget.__init__(self)
+        
+        self.l = QVBoxLayout()
+        self.setLayout(self.l)
+
+        label = QLabel('These settings control the basic features of the plugin--downloading FanFiction.')
+        label.setWordWrap(True)
+        self.l.addWidget(label)
+        self.l.addSpacing(5)
+
+        tooltip = "On each download, FFDL offers an option to select the output format. <br />This sets what that option will default to."
+        horz = QHBoxLayout()
+        label = QLabel('Default Output &Format:')
+        label.setToolTip(tooltip)
+        horz.addWidget(label)
+        self.fileform = QComboBox(self)
+        self.fileform.addItem('epub')
+        self.fileform.addItem('mobi')
+        self.fileform.addItem('html')
+        self.fileform.addItem('txt')
+        self.fileform.setCurrentIndex(self.fileform.findText(prefs['fileform']))
+        self.fileform.setToolTip(tooltip)
+        self.fileform.activated.connect(self.set_collisions)
+        label.setBuddy(self.fileform)
+        horz.addWidget(self.fileform)
+        self.l.addLayout(horz)
+
+        tooltip = "On each download, FFDL offers an option of what happens if that story already exists. <br />This sets what that option will default to."
+        horz = QHBoxLayout()
+        label = QLabel('Default If Story Already Exists?')
+        label.setToolTip(tooltip)
+        horz.addWidget(label)
+        self.collision = QComboBox(self)
+        # add collision options
+        self.set_collisions()
+        i = self.collision.findText(prefs['collision'])
+        if i > -1:
+            self.collision.setCurrentIndex(i)
+        self.collision.setToolTip(tooltip)
+        label.setBuddy(self.collision)
+        horz.addWidget(self.collision)
+        self.l.addLayout(horz)
+
+        self.updatemeta = QCheckBox('Default Update Calibre &Metadata?',self)
+        self.updatemeta.setToolTip("On each download, FFDL offers an option to update Calibre's metadata (title, author, URL, tags, custom columns, etc) from the web site. <br />This sets whether that will default to on or off.")
+        self.updatemeta.setChecked(prefs['updatemeta'])
+        self.l.addWidget(self.updatemeta)
+
+        self.updateepubcover = QCheckBox('Default Update EPUB Cover when Updating EPUB?',self)
+        self.updateepubcover.setToolTip("On each download, FFDL offers an option to update the book cover image <i>inside</i> the EPUB from the web site when the EPUB is updated.<br />This sets whether that will default to on or off.")
+        self.updateepubcover.setChecked(prefs['updateepubcover'])
+        self.l.addWidget(self.updateepubcover)
+
+        self.updatecover = QCheckBox('Update Calibre Cover when Updating Metadata?',self)
+        self.updatecover.setToolTip("Update calibre book cover image from EPUB when metadata is updated.  (EPUB only.)\nDoesn't go looking for new images on 'Update Calibre Metadata Only'.")
+        self.updatecover.setChecked(prefs['updatecover'])
+        self.l.addWidget(self.updatecover)
+
+        self.keeptags = QCheckBox('Keep Existing Tags when Updating Metadata?',self)
+        self.keeptags.setToolTip('Existing tags will be kept and any new tags added.\nCompleted and In-Progress tags will be still be updated, if known.\nLast Updated tags will be updated if lastupdate in include_subject_tags.')
+        self.keeptags.setChecked(prefs['keeptags'])
+        self.l.addWidget(self.keeptags)
+
+        self.urlsfromclip = QCheckBox('Take URLs from Clipboard?',self)
+        self.urlsfromclip.setToolTip('Prefill URLs from valid URLs in Clipboard when Adding New.')
+        self.urlsfromclip.setChecked(prefs['urlsfromclip'])
+        self.l.addWidget(self.urlsfromclip)
+
+        self.updatedefault = QCheckBox('Default to Update when books selected?',self)
+        self.updatedefault.setToolTip('The top FanFictionDownLoader plugin button will start Update if\n'+
+                                      'books are selected.  If unchecked, it will always bring up \'Add New\'.')
+        self.updatedefault.setChecked(prefs['updatedefault'])
+        self.l.addWidget(self.updatedefault)
+
+        self.deleteotherforms = QCheckBox('Delete other existing formats?',self)
+        self.deleteotherforms.setToolTip('Check this to automatically delete all other ebook formats when updating an existing book.\nHandy if you have both a Nook(epub) and Kindle(mobi), for example.')
+        self.deleteotherforms.setChecked(prefs['deleteotherforms'])
+        self.l.addWidget(self.deleteotherforms)
+        
+        self.adddialogstaysontop = QCheckBox("Keep 'Add New from URL(s)' dialog on top?",self)
+        self.adddialogstaysontop.setToolTip("Instructs the OS and Window Manager to keep the 'Add New from URL(s)'\ndialog on top of all other windows.  Useful for dragging URLs onto it.")
+        self.adddialogstaysontop.setChecked(prefs['adddialogstaysontop'])
+        self.l.addWidget(self.adddialogstaysontop)
+
+        # this is a cheat to make it easier for users to realize there's a new include_images features.
+        self.includeimages = QCheckBox("Include images in EPUBs?",self)
+        self.includeimages.setToolTip("Download and include images in EPUB stories.  This is equivalent to adding:\n\n[epub]\ninclude_images:true\nkeep_summary_html:true\nmake_firstimage_cover:true\n\n ...to the top of personal.ini.  Your settings in personal.ini will override this.")
+        self.includeimages.setChecked(prefs['includeimages'])
+        self.l.addWidget(self.includeimages)
+
+        self.lookforurlinhtml = QCheckBox("Search EPUB text for Story URL?",self)
+        self.lookforurlinhtml.setToolTip("Look for first valid story URL inside EPUB text if not found in metadata.\nSomewhat risky, could find wrong URL depending on EPUB content.\nAlso finds and corrects bad ffnet URLs from ficsaver.com files.")
+        self.lookforurlinhtml.setChecked(prefs['lookforurlinhtml'])
+        self.l.addWidget(self.lookforurlinhtml)
+
+        self.injectseries = QCheckBox("Inject calibre Series when none found?",self)
+        self.injectseries.setToolTip("If no series is found, inject the calibre series (if there is one) so it appears on the FFDL title page(not cover).")
+        self.injectseries.setChecked(prefs['injectseries'])
+        self.l.addWidget(self.injectseries)
+
+        self.l.insertStretch(-1)
+        
+    def set_collisions(self):
+        prev=self.collision.currentText()
+        self.collision.clear()
+        for o in collision_order:
+            if self.fileform.currentText() == 'epub' or o not in [UPDATE,UPDATEALWAYS]:
+                self.collision.addItem(o)
+        i = self.collision.findText(prev)
+        if i > -1:
+            self.collision.setCurrentIndex(i)
+        
+    def show_defaults(self):
+        text = get_resources('plugin-defaults.ini')
+        ShowDefaultsIniDialog(self.windowIcon(),text,self).exec_()
+        
+class PersonalIniTab(QWidget):
+
+    def __init__(self, parent_dialog, plugin_action):
+        self.parent_dialog = parent_dialog
+        self.plugin_action = plugin_action
+        QWidget.__init__(self)
+        
+        self.l = QVBoxLayout()
+        self.setLayout(self.l)
+
+        label = QLabel('These settings provide more detailed control over what metadata will be displayed inside the ebook as well as let you set is_adult and user/password for different sites.')
+        label.setWordWrap(True)
+        self.l.addWidget(label)
+        self.l.addSpacing(5)
+        
+        self.label = QLabel('personal.ini:')
+        self.l.addWidget(self.label)
+
+        self.ini = QTextEdit(self)
+        try:
+            self.ini.setFont(QFont("Courier",
+                                   self.plugin_action.gui.font().pointSize()+1));
+        except Exception as e:
+            print("Couldn't get font: %s"%e)
+        self.ini.setLineWrapMode(QTextEdit.NoWrap)
+        self.ini.setText(prefs['personal.ini'])
+        self.l.addWidget(self.ini)
+
+        self.defaults = QPushButton('View Defaults', self)
+        self.defaults.setToolTip("View all of the plugin's configurable settings\nand their default settings.")
+        self.defaults.clicked.connect(self.show_defaults)
+        self.l.addWidget(self.defaults)
+        
+        # self.l.insertStretch(-1)
+        # let edit box fill the space.
+        
+    def show_defaults(self):
+        text = get_resources('plugin-defaults.ini')
+        ShowDefaultsIniDialog(self.windowIcon(),text,self).exec_()
+
+class ShowDefaultsIniDialog(QDialog):
+
+    def __init__(self, icon, text, parent=None):
+        QDialog.__init__(self, parent)
+        self.resize(600, 500)
+        self.l = QVBoxLayout()
+        self.setLayout(self.l)
+        self.label = QLabel("Plugin Defaults (Read-Only)")
+        self.label.setToolTip("These are all of the plugin's configurable options\nand their default settings.")
+        self.setWindowTitle(_('Plugin Defaults'))
+        self.setWindowIcon(icon)
+        self.l.addWidget(self.label)
+        
+        self.ini = QTextEdit(self)
+        self.ini.setToolTip("These are all of the plugin's configurable options\nand their default settings.")
+        try:
+            self.ini.setFont(QFont("Courier",
+                                   get_gui().font().pointSize()+1));
+        except Exception as e:
+            print("Couldn't get font: %s"%e)
+        self.ini.setLineWrapMode(QTextEdit.NoWrap)
+        self.ini.setText(text)
+        self.ini.setReadOnly(True)
+        self.l.addWidget(self.ini)
+        
+        self.ok_button = QPushButton('OK', self)
+        self.ok_button.clicked.connect(self.hide)
+        self.l.addWidget(self.ok_button)
+        
+class ReadingListTab(QWidget):
+
+    def __init__(self, parent_dialog, plugin_action):
+        self.parent_dialog = parent_dialog
+        self.plugin_action = plugin_action
+        QWidget.__init__(self)
+        
+        self.l = QVBoxLayout()
+        self.setLayout(self.l)
+
+        try:
+            rl_plugin = plugin_action.gui.iactions['Reading List']
+            reading_lists = rl_plugin.get_list_names()
+        except KeyError:
+            reading_lists= []
+            
+        label = QLabel('These settings provide integration with the Reading List Plugin.  Reading List can automatically send to devices and change custom columns.  You have to create and configure the lists in Reading List to be useful.')
+        label.setWordWrap(True)
+        self.l.addWidget(label)
+        self.l.addSpacing(5)
+        
+        self.addtolists = QCheckBox('Add new/updated stories to "Send to Device" Reading List(s).',self)
+        self.addtolists.setToolTip('Automatically add new/updated stories to these lists in the Reading List plugin.')
+        self.addtolists.setChecked(prefs['addtolists'])
+        self.l.addWidget(self.addtolists)
+            
+        horz = QHBoxLayout()
+        label = QLabel('"Send to Device" Reading Lists')
+        label.setToolTip("When enabled, new/updated stories will be automatically added to these lists.")
+        horz.addWidget(label)        
+        self.send_lists_box = MultiCompleteLineEdit(self)
+        self.send_lists_box.setToolTip("When enabled, new/updated stories will be automatically added to these lists.")
+        self.send_lists_box.update_items_cache(reading_lists)
+        self.send_lists_box.setText(prefs['send_lists'])
+        horz.addWidget(self.send_lists_box)
+        self.l.addLayout(horz)
+        
+        self.addtoreadlists = QCheckBox('Add new/updated stories to "To Read" Reading List(s).',self)
+        self.addtoreadlists.setToolTip('Automatically add new/updated stories to these lists in the Reading List plugin.\nAlso offers menu option to remove stories from the "To Read" lists.')
+        self.addtoreadlists.setChecked(prefs['addtoreadlists'])
+        self.l.addWidget(self.addtoreadlists)
+            
+        horz = QHBoxLayout()
+        label = QLabel('"To Read" Reading Lists')
+        label.setToolTip("When enabled, new/updated stories will be automatically added to these lists.")
+        horz.addWidget(label)        
+        self.read_lists_box = MultiCompleteLineEdit(self)
+        self.read_lists_box.setToolTip("When enabled, new/updated stories will be automatically added to these lists.")
+        self.read_lists_box.update_items_cache(reading_lists)
+        self.read_lists_box.setText(prefs['read_lists'])
+        horz.addWidget(self.read_lists_box)
+        self.l.addLayout(horz)
+        
+        self.addtolistsonread = QCheckBox('Add stories back to "Send to Device" Reading List(s) when marked "Read".',self)
+        self.addtolistsonread.setToolTip('Menu option to remove from "To Read" lists will also add stories back to "Send to Device" Reading List(s)')
+        self.addtolistsonread.setChecked(prefs['addtolistsonread'])
+        self.l.addWidget(self.addtolistsonread)
+            
+        self.l.insertStretch(-1)
+        
+class GenerateCoverTab(QWidget):
+
+    def __init__(self, parent_dialog, plugin_action):
+        self.parent_dialog = parent_dialog
+        self.plugin_action = plugin_action
+        QWidget.__init__(self)
+        
+        self.l = QVBoxLayout()
+        self.setLayout(self.l)
+
+        try:
+            gc_plugin = plugin_action.gui.iactions['Generate Cover']
+            gc_settings = gc_plugin.get_saved_setting_names()
+        except KeyError:
+            gc_settings= []
+            
+        label = QLabel('The Generate Cover plugin can create cover images for books using various metadata and configurations.  If you have GC installed, FFDL can run GC on new downloads and metadata updates.  Pick a GC setting by site or Default.')
+        label.setWordWrap(True)
+        self.l.addWidget(label)
+        self.l.addSpacing(5)
+
+        scrollable = QScrollArea()
+        scrollcontent = QWidget()
+        scrollable.setWidget(scrollcontent)
+        scrollable.setWidgetResizable(True)
+        self.l.addWidget(scrollable)
+
+        self.sl = QVBoxLayout()
+        scrollcontent.setLayout(self.sl)
+        
+        self.gc_dropdowns = {}
+
+        sitelist = getConfigSections()
+        sitelist.sort()
+        sitelist.insert(0,u"Default")
+        for site in sitelist:
+            horz = QHBoxLayout()
+            label = QLabel(site)
+            if site == u"Default":
+                s = "On Metadata update, run Generate Cover with this setting, if not selected for specific site."
+            else:
+                s = "On Metadata update, run Generate Cover with this setting for %s stories."%site
+
+            label.setToolTip(s)
+            horz.addWidget(label)
+            dropdown = QComboBox(self)
+            dropdown.setToolTip(s)
+            dropdown.addItem('',QVariant('none'))
+            for setting in gc_settings:
+                dropdown.addItem(setting,QVariant(setting))
+            self.gc_dropdowns[site] = dropdown
+            if site in prefs['gc_site_settings']:
+                dropdown.setCurrentIndex(dropdown.findData(QVariant(prefs['gc_site_settings'][site])))
+
+            horz.addWidget(dropdown)
+            self.sl.addLayout(horz)
+        
+        self.gcnewonly = QCheckBox("Run Generate Cover Only on New Books",self)
+        self.gcnewonly.setToolTip("Default is to run GC any time the calibre metadata is updated.")
+        self.gcnewonly.setChecked(prefs['gcnewonly'])
+        self.l.addWidget(self.gcnewonly)
+
+        self.allow_gc_from_ini = QCheckBox('Allow generate_cover_settings from personal.ini to override.',self)
+        self.allow_gc_from_ini.setToolTip("The personal.ini parameter generate_cover_settings allows you to choose a GC setting based on metadata rather than site, but it's much more complex.<br \>generate_cover_settings is ignored when this is off.")
+        self.allow_gc_from_ini.setChecked(prefs['allow_gc_from_ini'])
+        self.l.addWidget(self.allow_gc_from_ini)
+            
+        self.l.insertStretch(-1)
+        
+class OtherTab(QWidget):
+
+    def __init__(self, parent_dialog, plugin_action):
+        self.parent_dialog = parent_dialog
+        self.plugin_action = plugin_action
+        QWidget.__init__(self)
+        
+        self.l = QVBoxLayout()
+        self.setLayout(self.l)
+
+        label = QLabel("These controls aren't plugin settings as such, but convenience buttons for setting Keyboard shortcuts and getting all the FanFictionDownLoader confirmation dialogs back again.")
+        label.setWordWrap(True)
+        self.l.addWidget(label)
+        self.l.addSpacing(5)
+        
+        keyboard_shortcuts_button = QPushButton('Keyboard shortcuts...', self)
+        keyboard_shortcuts_button.setToolTip(_(
+                    'Edit the keyboard shortcuts associated with this plugin'))
+        keyboard_shortcuts_button.clicked.connect(parent_dialog.edit_shortcuts)
+        self.l.addWidget(keyboard_shortcuts_button)
+
+        reset_confirmation_button = QPushButton(_('Reset disabled &confirmation dialogs'), self)
+        reset_confirmation_button.setToolTip(_(
+                    'Reset all show me again dialogs for the FanFictionDownLoader plugin'))
+        reset_confirmation_button.clicked.connect(self.reset_dialogs)
+        self.l.addWidget(reset_confirmation_button)
+        
+        self.l.insertStretch(-1)
+        
+    def reset_dialogs(self):
+        for key in dynamic.keys():
+            if key.startswith('fanfictiondownloader_') and key.endswith('_again') \
+                                                  and dynamic[key] is False:
+                dynamic[key] = True
+        info_dialog(self, _('Done'),
+                    _('Confirmation dialogs have all been reset'),
+                    show=True,
+                    show_copy_button=False)
+
+permitted_values = {
+    'int' : ['numWords','numChapters'],
+    'float' : ['numWords','numChapters'],
+    'bool' : ['status-C','status-I'],
+    'datetime' : ['datePublished', 'dateUpdated', 'dateCreated'],
+    'series' : ['series'],
+    'enumeration' : ['category',
+                     'genre',
+                     'language',
+                     'series',
+                     'characters',
+                     'status',
+                     'datePublished',
+                     'dateUpdated',
+                     'dateCreated',
+                     'rating',
+                     'warnings',
+                     'numChapters',
+                     'numWords',
+                     'site',
+                     'storyId',
+                     'authorId',
+                     'extratags',
+                     'title',
+                     'storyUrl',
+                     'description',
+                     'author',
+                     'authorUrl',
+                     'formatname'
+                     #,'formatext'   # not useful information.
+                     #,'siteabbrev'
+                     #,'version'
+                     ]
+    }
+# no point copying the whole list.
+permitted_values['text'] = permitted_values['enumeration']
+permitted_values['comments'] = permitted_values['enumeration']
+
+titleLabels = {
+    'category':'Category',
+    'genre':'Genre',
+    'language':'Language',
+    'status':'Status',
+    'status-C':'Status:Completed',
+    'status-I':'Status:In-Progress',
+    'series':'Series',
+    'characters':'Characters',
+    'datePublished':'Published',
+    'dateUpdated':'Updated',
+    'dateCreated':'Packaged',
+    'rating':'Rating',
+    'warnings':'Warnings',
+    'numChapters':'Chapters',
+    'numWords':'Words',
+    'site':'Site',
+    'storyId':'Story ID',
+    'authorId':'Author ID',
+    'extratags':'Extra Tags',
+    'title':'Title',
+    'storyUrl':'Story URL',
+    'description':'Summary',
+    'author':'Author',
+    'authorUrl':'Author URL',
+    'formatname':'File Format',
+    'formatext':'File Extension',
+    'siteabbrev':'Site Abbrev',
+    'version':'FFDL Version'
+    }
+
+class ColumnsTab(QWidget):
+
+    def __init__(self, parent_dialog, plugin_action):
+        self.parent_dialog = parent_dialog
+        self.plugin_action = plugin_action
+        QWidget.__init__(self)
+        
+        self.l = QVBoxLayout()
+        self.setLayout(self.l)
+
+        label = QLabel("If you have custom columns defined, they will be listed below.  Choose a metadata value type to fill your columns automatically.")
+        label.setWordWrap(True)
+        self.l.addWidget(label)
+        self.l.addSpacing(5)
+        
+        self.custcol_dropdowns = {}
+
+        custom_columns = self.plugin_action.gui.library_view.model().custom_columns
+
+        for key, column in custom_columns.iteritems():
+
+            if column['datatype'] in permitted_values:
+                # print("\n============== %s ===========\n"%key)
+                # for (k,v) in column.iteritems():
+                #     print("column['%s'] => %s"%(k,v))
+                horz = QHBoxLayout()
+                label = QLabel('%s(%s)'%(column['name'],key))
+                label.setToolTip("Update this %s column with..."%column['datatype'])
+                horz.addWidget(label)
+                dropdown = QComboBox(self)
+                dropdown.addItem('',QVariant('none'))
+                for md in permitted_values[column['datatype']]:
+                    dropdown.addItem(titleLabels[md],QVariant(md))
+                self.custcol_dropdowns[key] = dropdown
+                if key in prefs['custom_cols']:
+                    dropdown.setCurrentIndex(dropdown.findData(QVariant(prefs['custom_cols'][key])))
+                if column['datatype'] == 'enumeration':
+                    dropdown.setToolTip("Metadata values valid for this type of column.\nValues that aren't valid for this enumeration column will be ignored.")
+                else:
+                    dropdown.setToolTip("Metadata values valid for this type of column.")
+
+                horz.addWidget(dropdown)
+                self.l.addLayout(horz)
+        
+        self.l.insertStretch(-1)
+
+        #print("prefs['custom_cols'] %s"%prefs['custom_cols'])
@@ -0,0 +1,715 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2011, Jim Miller'
+__docformat__ = 'restructuredtext en'
+
+import traceback
+
+from PyQt4 import QtGui
+from PyQt4.Qt import (QDialog, QTableWidget, QMessageBox, QVBoxLayout, QHBoxLayout, QGridLayout,
+                      QPushButton, QProgressDialog, QString, QLabel, QCheckBox, QIcon, QTextCursor,
+                      QTextEdit, QLineEdit, QInputDialog, QComboBox, QClipboard, QVariant,
+                      QProgressDialog, QTimer, QDialogButtonBox, QPixmap, Qt, QAbstractItemView )
+
+from calibre.gui2 import error_dialog, warning_dialog, question_dialog, info_dialog
+from calibre.gui2.dialogs.confirm_delete import confirm
+
+from calibre import confirm_config_name
+from calibre.gui2 import dynamic
+
+from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapters,writers,exceptions
+from calibre_plugins.fanfictiondownloader_plugin.common_utils \
+    import (ReadOnlyTableWidgetItem, ReadOnlyTextIconWidgetItem, SizePersistedDialog,
+            ImageTitleLayout, get_icon)
+
+SKIP='Skip'
+ADDNEW='Add New Book'
+UPDATE='Update EPUB if New Chapters'
+UPDATEALWAYS='Update EPUB Always'
+OVERWRITE='Overwrite if Newer'
+OVERWRITEALWAYS='Overwrite Always'
+CALIBREONLY='Update Calibre Metadata Only'
+collision_order=[SKIP,
+                 ADDNEW,
+                 UPDATE,
+                 UPDATEALWAYS,
+                 OVERWRITE,
+                 OVERWRITEALWAYS,
+                 CALIBREONLY,]
+        
+class NotGoingToDownload(Exception):
+    def __init__(self,error,icon='dialog_error.png'):
+        self.error=error
+        self.icon=icon
+
+    def __str__(self):
+        return self.error
+
+class DroppableQTextEdit(QTextEdit):
+    def __init__(self,parent):
+        QTextEdit.__init__(self,parent)
+        
+    def canInsertFromMimeData(self, source):
+        if source.hasUrls():
+            return True;
+        else:
+            return QTextEdit.canInsertFromMimeData(self,source)
+
+    def insertFromMimeData(self, source):
+        if source.hasText():
+            self.append(source.text())
+        else:
+            return QTextEdit.insertFromMimeData(self, source)
+                            
+class AddNewDialog(SizePersistedDialog):
+
+    def __init__(self, gui, prefs, icon, url_list_text):
+        SizePersistedDialog.__init__(self, gui, 'FanFictionDownLoader plugin:add new dialog')
+        self.gui = gui
+
+        if prefs['adddialogstaysontop']:
+            QDialog.setWindowFlags ( self, Qt.Dialog|Qt.WindowStaysOnTopHint )
+        
+        self.setMinimumWidth(300)
+        self.l = QVBoxLayout()
+        self.setLayout(self.l)
+
+        self.setWindowTitle('FanFictionDownLoader')
+        self.setWindowIcon(icon)
+
+        self.l.addWidget(QLabel('Story URL(s), one per line:'))
+        self.url = DroppableQTextEdit(self)
+        self.url.setToolTip('URLs for stories, one per line.\nWill take URLs from clipboard, but only valid URLs.\nAdd [1,5] after the URL to limit the download to chapters 1-5.')
+        self.url.setLineWrapMode(QTextEdit.NoWrap)
+        self.url.setText(url_list_text)
+        self.l.addWidget(self.url)
+        
+        horz = QHBoxLayout()
+        label = QLabel('Output &Format:')
+        horz.addWidget(label)
+        self.fileform = QComboBox(self)
+        self.fileform.addItem('epub')
+        self.fileform.addItem('mobi')
+        self.fileform.addItem('html')
+        self.fileform.addItem('txt')
+        self.fileform.setCurrentIndex(self.fileform.findText(prefs['fileform']))
+        self.fileform.setToolTip('Choose output format to create.  May set default from plugin configuration.')
+        self.fileform.activated.connect(self.set_collisions)
+        
+        label.setBuddy(self.fileform)
+        horz.addWidget(self.fileform)
+        self.l.addLayout(horz)
+
+        horz = QHBoxLayout()
+        label = QLabel('If Story Already Exists?')
+        label.setToolTip("What to do if there's already an existing story with the same title and author.")
+        horz.addWidget(label)
+        self.collision = QComboBox(self)
+        # add collision options
+        self.set_collisions()
+        i = self.collision.findText(prefs['collision'])
+        if i > -1:
+            self.collision.setCurrentIndex(i)
+        # self.collision.setToolTip(OVERWRITE+' will replace the existing story.\n'+
+        #                           UPDATE+' will download new chapters only and add to existing EPUB.\n'+
+        #                           ADDNEW+' will create a new story with the same title and author.\n'+
+        #                           SKIP+' will not download existing stories.\n'+
+        #                           CALIBREONLY+' will not download stories, but will update Calibre metadata.')
+        label.setBuddy(self.collision)
+        horz.addWidget(self.collision)
+        self.l.addLayout(horz)
+
+        horz = QHBoxLayout()
+        self.updatemeta = QCheckBox('Update Calibre &Metadata?',self)
+        self.updatemeta.setToolTip('Update metadata for story in Calibre from web site?')
+        self.updatemeta.setChecked(prefs['updatemeta'])
+        horz.addWidget(self.updatemeta)
+
+        self.updateepubcover = QCheckBox('Update EPUB Cover?',self)
+        self.updateepubcover.setToolTip('Update book cover image from site or defaults (if found) <i>inside</i> the EPUB when EPUB is updated.')
+        self.updateepubcover.setChecked(prefs['updateepubcover'])
+        horz.addWidget(self.updateepubcover)
+        
+        self.l.addLayout(horz)
+
+        button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
+        button_box.accepted.connect(self.accept)
+        button_box.rejected.connect(self.reject)
+        self.l.addWidget(button_box)
+        
+        if url_list_text:
+            button_box.button(QDialogButtonBox.Ok).setFocus()
+
+        # restore saved size.
+        self.resize_dialog()
+        #self.resize(self.sizeHint())
+
+    def set_collisions(self):
+        prev=self.collision.currentText()
+        self.collision.clear()
+        for o in collision_order:
+            if self.fileform.currentText() == 'epub' or o not in [UPDATE,UPDATEALWAYS]:
+                self.collision.addItem(o)
+        i = self.collision.findText(prev)
+        if i > -1:
+            self.collision.setCurrentIndex(i)
+        
+    def get_ffdl_options(self):
+        return {
+            'fileform': unicode(self.fileform.currentText()),
+            'collision': unicode(self.collision.currentText()),
+            'updatemeta': self.updatemeta.isChecked(),
+            'updateepubcover': self.updateepubcover.isChecked(),
+            }
+
+    def get_urlstext(self):
+        return unicode(self.url.toPlainText())
+
+
+class FakeLineEdit():
+    def __init__(self):
+        pass
+    
+    def text(self):
+        pass
+    
+class CollectURLDialog(QDialog):
+    '''
+    Collect single url for get urls.
+    '''
+    def __init__(self, gui, title):
+        QDialog.__init__(self, gui)
+        self.gui = gui
+        self.status=False
+
+        self.l = QGridLayout()
+        self.setLayout(self.l)
+
+        self.setWindowTitle(title)
+        self.l.addWidget(QLabel(title),0,0,1,2)
+        
+        self.l.addWidget(QLabel("URL:"),1,0)
+        self.url = QLineEdit(self)
+        self.l.addWidget(self.url,1,1)
+   
+        self.ok_button = QPushButton('OK', self)
+        self.ok_button.clicked.connect(self.ok)
+        self.l.addWidget(self.ok_button,2,0)
+
+        self.cancel_button = QPushButton('Cancel', self)
+        self.cancel_button.clicked.connect(self.cancel)
+        self.l.addWidget(self.cancel_button,2,1)
+
+        self.resize(self.sizeHint())
+
+    def ok(self):
+        self.status=True
+        self.hide()
+
+    def cancel(self):
+        self.status=False
+        self.hide()
+
+class UserPassDialog(QDialog):
+    '''
+    Need to collect User/Pass for some sites.
+    '''
+    def __init__(self, gui, site, exception=None):
+        QDialog.__init__(self, gui)
+        self.gui = gui
+        self.status=False
+
+        self.l = QGridLayout()
+        self.setLayout(self.l)
+
+        if exception.passwdonly:
+            self.setWindowTitle('Password')
+            self.l.addWidget(QLabel("Author requires a password for this story(%s)."%exception.url),0,0,1,2)
+            # user isn't used, but it's easier to still have it for
+            # post processing.
+            self.user = FakeLineEdit()
+        else:
+            self.setWindowTitle('User/Password')
+            self.l.addWidget(QLabel("%s requires you to login to download this story."%site),0,0,1,2)
+        
+            self.l.addWidget(QLabel("User:"),1,0)
+            self.user = QLineEdit(self)
+            self.l.addWidget(self.user,1,1)
+   
+        self.l.addWidget(QLabel("Password:"),2,0)
+        self.passwd = QLineEdit(self)
+        self.passwd.setEchoMode(QLineEdit.Password)
+        self.l.addWidget(self.passwd,2,1)
+   
+        self.ok_button = QPushButton('OK', self)
+        self.ok_button.clicked.connect(self.ok)
+        self.l.addWidget(self.ok_button,3,0)
+
+        self.cancel_button = QPushButton('Cancel', self)
+        self.cancel_button.clicked.connect(self.cancel)
+        self.l.addWidget(self.cancel_button,3,1)
+
+        self.resize(self.sizeHint())
+
+    def ok(self):
+        self.status=True
+        self.hide()
+
+    def cancel(self):
+        self.status=False
+        self.hide()
+
+class LoopProgressDialog(QProgressDialog):
+    '''
+    ProgressDialog displayed while fetching metadata for each story.
+    '''
+    def __init__(self, gui,
+                 book_list,
+                 foreach_function,
+                 finish_function,
+                 init_label="Fetching metadata for stories...",
+                 win_title="Downloading metadata for stories",
+                 status_prefix="Fetched metadata for"):
+        QProgressDialog.__init__(self,
+                                 init_label,
+                                 QString(), 0, len(book_list), gui)
+        self.setWindowTitle(win_title)
+        self.setMinimumWidth(500)
+        self.gui = gui
+        self.book_list = book_list
+        self.foreach_function = foreach_function
+        self.finish_function = finish_function
+        self.status_prefix = status_prefix
+        self.i = 0
+        
+        ## self.do_loop does QTimer.singleShot on self.do_loop also.
+        ## A weird way to do a loop, but that was the example I had.
+        QTimer.singleShot(0, self.do_loop)
+        self.exec_()
+
+    def updateStatus(self):
+        self.setLabelText("%s %d of %d"%(self.status_prefix,self.i+1,len(self.book_list)))
+        self.setValue(self.i+1)
+        print(self.labelText())
+
+    def do_loop(self):
+
+        if self.i == 0:
+            self.setValue(0)
+
+        book = self.book_list[self.i]
+        try:
+            ## collision spec passed into getadapter by partial from ffdl_plugin
+            ## no retval only if it exists, but collision is SKIP
+            self.foreach_function(book)
+            
+        except NotGoingToDownload as d:
+            book['good']=False
+            book['comment']=unicode(d)
+            book['icon'] = d.icon
+
+        except Exception as e:
+            book['good']=False
+            book['comment']=unicode(e)
+            print("Exception: %s:%s"%(book,unicode(e)))
+            traceback.print_exc()
+            
+        self.updateStatus()
+        self.i += 1
+            
+        if self.i >= len(self.book_list) or self.wasCanceled():
+            return self.do_when_finished()
+        else:
+            QTimer.singleShot(0, self.do_loop)
+
+    def do_when_finished(self):
+        self.hide()
+        self.gui = None        
+        # Queues a job to process these books in the background.
+        self.finish_function(self.book_list)
+
+class AboutDialog(QDialog):
+
+    def __init__(self, parent, icon, text):
+        QDialog.__init__(self, parent)
+        self.resize(400, 250)
+        self.l = QGridLayout()
+        self.setLayout(self.l)
+        self.logo = QLabel()
+        self.logo.setMaximumWidth(110)
+        self.logo.setPixmap(QPixmap(icon.pixmap(100,100)))
+        self.label = QLabel(text)
+        self.label.setOpenExternalLinks(True)
+        self.label.setWordWrap(True)
+        self.setWindowTitle(_('About FanFictionDownLoader'))
+        self.setWindowIcon(icon)
+        self.l.addWidget(self.logo, 0, 0)
+        self.l.addWidget(self.label, 0, 1)
+        self.bb = QDialogButtonBox(self)
+        b = self.bb.addButton(_('OK'), self.bb.AcceptRole)
+        b.setDefault(True)
+        self.l.addWidget(self.bb, 2, 0, 1, -1)
+        self.bb.accepted.connect(self.accept)
+
+class IconWidgetItem(ReadOnlyTextIconWidgetItem):
+    def __init__(self, text, icon, sort_key):
+        ReadOnlyTextIconWidgetItem.__init__(self, text, icon)
+        self.sort_key = sort_key
+
+    #Qt uses a simple < check for sorting items, override this to use the sortKey
+    def __lt__(self, other):
+        return self.sort_key < other.sort_key
+
+class AuthorTableWidgetItem(ReadOnlyTableWidgetItem):
+    def __init__(self, text, sort_key):
+        ReadOnlyTableWidgetItem.__init__(self, text)
+        self.sort_key = sort_key
+
+    #Qt uses a simple < check for sorting items, override this to use the sortKey
+    def __lt__(self, other):
+        return self.sort_key < other.sort_key
+
+class UpdateExistingDialog(SizePersistedDialog):
+    def __init__(self, gui, header, prefs, icon, books,
+                 save_size_name='fanfictiondownloader_plugin:update list dialog'):
+        SizePersistedDialog.__init__(self, gui, save_size_name)
+        self.gui = gui
+        
+        self.setWindowTitle(header)
+        self.setWindowIcon(icon)
+        
+        layout = QVBoxLayout(self)
+        self.setLayout(layout)
+        title_layout = ImageTitleLayout(self, 'images/icon.png',
+                                        header)
+        layout.addLayout(title_layout)
+        books_layout = QHBoxLayout()
+        layout.addLayout(books_layout)
+
+        self.books_table = StoryListTableWidget(self)
+        books_layout.addWidget(self.books_table)
+
+        button_layout = QVBoxLayout()
+        books_layout.addLayout(button_layout)
+        # self.move_up_button = QtGui.QToolButton(self)
+        # self.move_up_button.setToolTip('Move selected books up the list')
+        # self.move_up_button.setIcon(QIcon(I('arrow-up.png')))
+        # self.move_up_button.clicked.connect(self.books_table.move_rows_up)
+        # button_layout.addWidget(self.move_up_button)
+        spacerItem = QtGui.QSpacerItem(20, 40, QtGui.QSizePolicy.Minimum, QtGui.QSizePolicy.Expanding)
+        button_layout.addItem(spacerItem)
+        self.remove_button = QtGui.QToolButton(self)
+        self.remove_button.setToolTip('Remove selected books from the list')
+        self.remove_button.setIcon(get_icon('list_remove.png'))
+        self.remove_button.clicked.connect(self.remove_from_list)
+        button_layout.addWidget(self.remove_button)
+        spacerItem1 = QtGui.QSpacerItem(20, 40, QtGui.QSizePolicy.Minimum, QtGui.QSizePolicy.Expanding)
+        button_layout.addItem(spacerItem1)
+        # self.move_down_button = QtGui.QToolButton(self)
+        # self.move_down_button.setToolTip('Move selected books down the list')
+        # self.move_down_button.setIcon(QIcon(I('arrow-down.png')))
+        # self.move_down_button.clicked.connect(self.books_table.move_rows_down)
+        # button_layout.addWidget(self.move_down_button)
+
+        options_layout = QHBoxLayout()
+
+        label = QLabel('Output &Format:')
+        options_layout.addWidget(label)
+        self.fileform = QComboBox(self)
+        self.fileform.addItem('epub')
+        self.fileform.addItem('mobi')
+        self.fileform.addItem('html')
+        self.fileform.addItem('txt')
+        self.fileform.setCurrentIndex(self.fileform.findText(prefs['fileform']))
+        self.fileform.setToolTip('Choose output format to create.  May set default from plugin configuration.')
+        self.fileform.activated.connect(self.set_collisions)
+        label.setBuddy(self.fileform)
+        options_layout.addWidget(self.fileform)
+        
+        label = QLabel('Update Mode:')
+        label.setToolTip("What sort of update to perform.  May set default from plugin configuration.")
+        options_layout.addWidget(label)
+        self.collision = QComboBox(self)
+        # add collision options
+        self.set_collisions()
+        i = self.collision.findText(prefs['collision'])
+        if i > -1:
+            self.collision.setCurrentIndex(i)
+        # self.collision.setToolTip('Overwrite will replace the existing story.  Add New will create a new story with the same title and author.')
+        label.setBuddy(self.collision)
+        options_layout.addWidget(self.collision)
+
+        self.updatemeta = QCheckBox('Update Calibre &Metadata?',self)
+        self.updatemeta.setToolTip('Update metadata for story in Calibre from web site?  May set default from plugin configuration.')
+        self.updatemeta.setChecked(prefs['updatemeta'])
+        options_layout.addWidget(self.updatemeta)
+                
+        self.updateepubcover = QCheckBox('Update EPUB Cover?',self)
+        self.updateepubcover.setToolTip('Update book cover image from site or defaults (if found) <i>inside</i> the EPUB when EPUB is updated.')
+        self.updateepubcover.setChecked(prefs['updateepubcover'])
+        options_layout.addWidget(self.updateepubcover)
+        
+        button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
+        button_box.accepted.connect(self.accept)
+        button_box.rejected.connect(self.reject)
+        options_layout.addWidget(button_box)
+        
+        layout.addLayout(options_layout)
+        
+        # Cause our dialog size to be restored from prefs or created on first usage
+        self.resize_dialog()
+        self.books_table.populate_table(books)
+
+    def set_collisions(self):
+        prev=self.collision.currentText()
+        self.collision.clear()
+        for o in collision_order:
+            if o not in [ADDNEW,SKIP] and \
+                    (self.fileform.currentText() == 'epub' or o not in [UPDATE,UPDATEALWAYS]):
+                self.collision.addItem(o)        
+        i = self.collision.findText(prev)
+        if i > -1:
+            self.collision.setCurrentIndex(i)
+        
+    def remove_from_list(self):
+        self.books_table.remove_selected_rows()
+
+    def get_books(self):
+        return self.books_table.get_books()
+
+    def get_ffdl_options(self):
+        return {
+            'fileform': unicode(self.fileform.currentText()),
+            'collision': unicode(self.collision.currentText()),
+            'updatemeta': self.updatemeta.isChecked(),
+            'updateepubcover': self.updateepubcover.isChecked(),
+            }
+
+def display_story_list(gui, header, prefs, icon, books,
+                       label_text='',
+                       save_size_name='fanfictiondownloader_plugin:display list dialog',
+                       offer_skip=False):
+    all_good = True
+    for b in books:
+        if not b['good']:
+            all_good=False
+            break
+        
+    ## 
+    if all_good and not dynamic.get(confirm_config_name(save_size_name), True):
+        return True
+        pass
+            ## fake accept?
+    d = DisplayStoryListDialog(gui, header, prefs, icon, books,
+                               label_text,
+                               save_size_name,
+                               offer_skip and all_good)
+    d.exec_()
+    return d.result() == d.Accepted
+
+class DisplayStoryListDialog(SizePersistedDialog):
+    def __init__(self, gui, header, prefs, icon, books,
+                 label_text='',
+                 save_size_name='fanfictiondownloader_plugin:display list dialog',
+                 offer_skip=False):
+        SizePersistedDialog.__init__(self, gui, save_size_name)
+        self.name = save_size_name
+        self.gui = gui
+        
+        self.setWindowTitle(header)
+        self.setWindowIcon(icon)
+        
+        layout = QVBoxLayout(self)
+        self.setLayout(layout)
+        title_layout = ImageTitleLayout(self, 'images/icon.png',
+                                        header)
+        layout.addLayout(title_layout)
+
+        self.books_table = StoryListTableWidget(self)
+        layout.addWidget(self.books_table)
+
+        options_layout = QHBoxLayout()
+        self.label = QLabel(label_text)
+        #self.label.setOpenExternalLinks(True)
+        #self.label.setWordWrap(True)
+        options_layout.addWidget(self.label)
+        
+        if offer_skip:
+            spacerItem1 = QtGui.QSpacerItem(2, 4, QtGui.QSizePolicy.Expanding, QtGui.QSizePolicy.Minimum)
+            options_layout.addItem(spacerItem1)
+            self.again = QCheckBox('Show this again?',self)
+            self.again.setChecked(True)
+            self.again.stateChanged.connect(self.toggle)
+            self.again.setToolTip('Uncheck to skip review and update stories immediately when no problems.')
+            options_layout.addWidget(self.again)
+        
+        button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
+        button_box.accepted.connect(self.accept)
+        button_box.rejected.connect(self.reject)
+
+        options_layout.addWidget(button_box)
+        
+        layout.addLayout(options_layout)
+        
+        # Cause our dialog size to be restored from prefs or created on first usage
+        self.resize_dialog()
+        self.books_table.populate_table(books)
+
+    def get_books(self):
+        return self.books_table.get_books()
+
+    def toggle(self, *args):
+        dynamic[confirm_config_name(self.name)] = self.again.isChecked()
+
+    
+
+class StoryListTableWidget(QTableWidget):
+
+    def __init__(self, parent):
+        QTableWidget.__init__(self, parent)
+        self.setSelectionBehavior(QAbstractItemView.SelectRows)
+
+    def populate_table(self, books):
+        self.clear()
+        self.setAlternatingRowColors(True)
+        self.setRowCount(len(books))
+        header_labels = ['','Title', 'Author', 'URL', 'Comment']
+        self.setColumnCount(len(header_labels))
+        self.setHorizontalHeaderLabels(header_labels)
+        self.horizontalHeader().setStretchLastSection(True)
+        #self.verticalHeader().setDefaultSectionSize(24)
+        self.verticalHeader().hide()
+
+        self.books={}
+        for row, book in enumerate(books):
+            self.populate_table_row(row, book)
+            self.books[row] = book
+
+        # turning True breaks up/down.  Do we need either sorting or up/down?
+        self.setSortingEnabled(True)
+        self.resizeColumnsToContents()
+        self.setMinimumColumnWidth(1, 100)
+        self.setMinimumColumnWidth(2, 100)
+        self.setMinimumColumnWidth(3, 100)
+        self.setMinimumSize(300, 0)
+        # if len(books) > 0:
+        #     self.selectRow(0)
+        self.sortItems(1)
+        self.sortItems(0)
+
+    def setMinimumColumnWidth(self, col, minimum):
+        if self.columnWidth(col) < minimum:
+            self.setColumnWidth(col, minimum)
+
+    def populate_table_row(self, row, book):
+        if book['good']:
+            icon = get_icon('ok.png')
+            val = 0
+        else:
+            icon = get_icon('minus.png')
+            val = 1
+        if 'icon' in book:
+            icon = get_icon(book['icon'])
+
+        status_cell = IconWidgetItem(None,icon,val)
+        status_cell.setData(Qt.UserRole, QVariant(val))
+        self.setItem(row, 0, status_cell)
+        
+        title_cell = ReadOnlyTableWidgetItem(book['title'])
+        title_cell.setData(Qt.UserRole, QVariant(row))
+        self.setItem(row, 1, title_cell)
+        
+        self.setItem(row, 2, AuthorTableWidgetItem(book['author'], book['author_sort']))
+        
+        url_cell = ReadOnlyTableWidgetItem(book['url'])
+        #url_cell.setData(Qt.UserRole, QVariant(book['url']))
+        self.setItem(row, 3, url_cell)
+        
+        comment_cell = ReadOnlyTableWidgetItem(book['comment'])
+        #comment_cell.setData(Qt.UserRole, QVariant(book))
+        self.setItem(row, 4, comment_cell)
+
+    def get_books(self):
+        books = []
+        #print("=========================\nbooks:%s"%self.books)
+        for row in range(self.rowCount()):
+            rnum = self.item(row, 1).data(Qt.UserRole).toPyObject()
+            book = self.books[rnum]
+            books.append(book)
+        return books
+
+    def remove_selected_rows(self):
+        self.setFocus()
+        rows = self.selectionModel().selectedRows()
+        if len(rows) == 0:
+            return
+        message = '<p>Are you sure you want to remove this book from the list?'
+        if len(rows) > 1:
+            message = '<p>Are you sure you want to remove the selected %d books from the list?'%len(rows)
+        if not confirm(message,'fanfictiondownloader_delete_item', self):
+            return
+        first_sel_row = self.currentRow()
+        for selrow in reversed(rows):
+            self.removeRow(selrow.row())
+        if first_sel_row < self.rowCount():
+            self.select_and_scroll_to_row(first_sel_row)
+        elif self.rowCount() > 0:
+            self.select_and_scroll_to_row(first_sel_row - 1)
+
+    def select_and_scroll_to_row(self, row):
+        self.selectRow(row)
+        self.scrollToItem(self.currentItem())
+
+    def move_rows_up(self):
+        self.setFocus()
+        rows = self.selectionModel().selectedRows()
+        if len(rows) == 0:
+            return
+        first_sel_row = rows[0].row()
+        if first_sel_row <= 0:
+            return
+        # Workaround for strange selection bug in Qt which "alters" the selection
+        # in certain circumstances which meant move down only worked properly "once"
+        selrows = []
+        for row in rows:
+            selrows.append(row.row())
+        selrows.sort()
+        for selrow in selrows:
+            self.swap_row_widgets(selrow - 1, selrow + 1)
+        scroll_to_row = first_sel_row - 1
+        if scroll_to_row > 0:
+            scroll_to_row = scroll_to_row - 1
+        self.scrollToItem(self.item(scroll_to_row, 0))
+
+    def move_rows_down(self):
+        self.setFocus()
+        rows = self.selectionModel().selectedRows()
+        if len(rows) == 0:
+            return
+        last_sel_row = rows[-1].row()
+        if last_sel_row == self.rowCount() - 1:
+            return
+        # Workaround for strange selection bug in Qt which "alters" the selection
+        # in certain circumstances which meant move down only worked properly "once"
+        selrows = []
+        for row in rows:
+            selrows.append(row.row())
+        selrows.sort()
+        for selrow in reversed(selrows):
+            self.swap_row_widgets(selrow + 2, selrow)
+        scroll_to_row = last_sel_row + 1
+        if scroll_to_row < self.rowCount() - 1:
+            scroll_to_row = scroll_to_row + 1
+        self.scrollToItem(self.item(scroll_to_row, 0))
+
+    def swap_row_widgets(self, src_row, dest_row):
+        self.blockSignals(True)
+        self.insertRow(dest_row)
+        for col in range(0, self.columnCount()):
+            self.setItem(dest_row, col, self.takeItem(src_row, col))
+        self.removeRow(src_row)
+        self.blockSignals(False)
@@ -0,0 +1,184 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2012, Jim Miller'
+__copyright__ = '2011, Grant Drake <grant.drake@gmail.com>'
+__docformat__ = 'restructuredtext en'
+
+import time, os, traceback
+
+from ConfigParser import SafeConfigParser
+from StringIO import StringIO
+
+from calibre.utils.ipc.server import Server
+from calibre.utils.ipc.job import ParallelJob
+from calibre.utils.logging import Log
+
+from calibre_plugins.fanfictiondownloader_plugin.dialogs import (NotGoingToDownload,
+    OVERWRITE, OVERWRITEALWAYS, UPDATE, UPDATEALWAYS, ADDNEW, SKIP, CALIBREONLY)
+from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapters, writers, exceptions
+from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.epubutils import get_update_data
+
+# ------------------------------------------------------------------------------
+#
+#              Functions to perform downloads using worker jobs
+#
+# ------------------------------------------------------------------------------
+
+def do_download_worker(book_list, options,
+                       cpus, notification=lambda x,y:x):
+    '''
+    Master job, to launch child jobs to extract ISBN for a set of books
+    This is run as a worker job in the background to keep the UI more
+    responsive and get around the memory leak issues as it will launch
+    a child job for each book as a worker process
+    '''
+    server = Server(pool_size=cpus)
+
+    print(options['version'])
+    total = 0
+    # Queue all the jobs
+    print("Adding jobs for URLs:")
+    for book in book_list:
+        if book['good']:
+            print("%s"%book['url'])
+            total += 1
+            args = ['calibre_plugins.fanfictiondownloader_plugin.jobs',
+                    'do_download_for_worker',
+                    (book,options)]
+            job = ParallelJob('arbitrary',
+                              "url:(%s) id:(%s)"%(book['url'],book['calibre_id']),
+                              done=None,
+                              args=args)
+            job._book = book
+            # job._book_id = book_id
+            # job._title = title
+            # job._modified_date = modified_date
+            # job._existing_isbn = existing_isbn
+            server.add_job(job)
+    
+    # This server is an arbitrary_n job, so there is a notifier available.
+    # Set the % complete to a small number to avoid the 'unavailable' indicator
+    notification(0.01, 'Downloading FanFiction Stories')
+
+    # dequeue the job results as they arrive, saving the results
+    count = 0
+    while True:
+        job = server.changed_jobs_queue.get()
+        # A job can 'change' when it is not finished, for example if it
+        # produces a notification. Ignore these.
+        job.update()
+        if not job.is_finished:
+            continue
+        # A job really finished. Get the information.
+        output_book = job.result
+        #print("output_book:%s"%output_book)
+        book_list.remove(job._book)
+        book_list.append(job.result)
+        book_id = job._book['calibre_id']
+        #title = job._title
+        count = count + 1
+        notification(float(count)/total, 'Downloaded Story')
+        # Add this job's output to the current log
+        print('Logfile for book ID %s (%s)'%(book_id, job._book['title']))
+        print(job.details)
+
+        if count >= total:
+            # All done!  Output some lists for convenience of some users.
+            print("Successfully downloaded:")
+            for book in book_list:
+                if book['good']:
+                    print(book['title'])
+            print("\nUnsuccessful:")
+            for book in book_list:
+                if not book['good']:
+                    print(book['title'])
+            break
+
+    server.close()
+    
+    # return the book list as the job result
+    return book_list
+
+def do_download_for_worker(book,options):
+    '''
+    Child job, to extract isbn from formats for this specific book,
+    when run as a worker job
+    '''
+    try:
+        book['comment'] = 'Download started...'
+
+        ffdlconfig = SafeConfigParser()
+        ffdlconfig.readfp(StringIO(get_resources("plugin-defaults.ini")))
+        ffdlconfig.readfp(StringIO(options['personal.ini']))
+        
+        if not options['updateepubcover'] and 'epub_for_update' in book and options['collision'] in (UPDATE, UPDATEALWAYS):
+            ffdlconfig.set("overrides","never_make_cover","true")
+        
+        adapter = adapters.getAdapter(ffdlconfig,book['url'],options['fileform'])
+        adapter.is_adult = book['is_adult'] 
+        adapter.username = book['username'] 
+        adapter.password = book['password']
+        
+        story = adapter.getStoryMetadataOnly()
+        if 'calibre_series' in book:
+            # print("calibre_series:%s [%d]"%book['calibre_series'])
+            adapter.setSeries(book['calibre_series'][0],book['calibre_series'][1])
+        # else:
+        #     print("no calibre_series")
+        writer = writers.getWriter(options['fileform'],adapter.config,adapter)
+
+        outfile = book['outfile']
+
+        ## No need to download at all.  Shouldn't ever get down here.
+        if options['collision'] in (CALIBREONLY):
+            print("Skipping CALIBREONLY 'update' down inside worker--this shouldn't be happening...")
+            book['comment'] = 'Metadata collected.'
+            
+        ## checks were done earlier, it's new or not dup or newer--just write it.
+        elif options['collision'] in (ADDNEW, SKIP, OVERWRITE, OVERWRITEALWAYS) or \
+                ('epub_for_update' not in book and options['collision'] in (UPDATE, UPDATEALWAYS)):
+
+            adapter.setChaptersRange(book['begin'],book['end'])
+            
+            print("write to %s"%outfile)
+            writer.writeStory(outfilename=outfile, forceOverwrite=True)
+            book['comment'] = 'Download %s completed, %s chapters.'%(options['fileform'],story.getMetadata("numChapters"))
+            
+        ## checks were done earlier, just update it.
+        elif 'epub_for_update' in book and options['collision'] in (UPDATE, UPDATEALWAYS):
+
+            # update now handled by pre-populating the old images and
+            # chapters in the adapter rather than merging epubs.
+            urlchaptercount = int(story.getMetadata('numChapters'))
+            (url,chaptercount,
+             adapter.oldchapters,
+             adapter.oldimgs,
+             adapter.oldcover,
+             adapter.calibrebookmark) = get_update_data(book['epub_for_update'])
+
+            print("Do update - epub(%d) vs url(%d)" % (chaptercount, urlchaptercount))
+            print("write to %s"%outfile)
+
+            writer.writeStory(outfilename=outfile, forceOverwrite=True)
+            
+            book['comment'] = 'Update %s completed, added %s chapters for %s total.'%\
+                (options['fileform'],(urlchaptercount-chaptercount),urlchaptercount)
+        
+    except NotGoingToDownload as d:
+        book['good']=False
+        book['comment']=unicode(d)
+        book['icon'] = d.icon
+
+    except Exception as e:
+        book['good']=False
+        book['comment']=unicode(e)
+        book['icon']='dialog_error.png'
+        print("Exception: %s:%s"%(book,unicode(e)))
+        traceback.print_exc()
+
+    #time.sleep(10)
+    return book
@@ -3,6 +3,8 @@ cron:
  url: /r3m0v3r
  schedule: every 2 hours

- description: orphan cleanup job
-  url: /r3m0v3rOrphans
-  schedule: every 4 hours
+# There's a bug in the Python 2.7 runtime that prevents this from
+# working properly.  In theory, there should never be orphans anyway.
+#- description: orphan cleanup job
+#  url: /r3m0v3rOrphans
+#  schedule: every 4 hours
@@ -1,4 +1,4 @@
-# Copyright 2011 Fanficdownloader team
+# Copyright 2012 Fanficdownloader team
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -36,7 +36,9 @@ formatext_label:File Extension
 ## Sometimes there are multiple categories and/or genres.
 category_label:Category
 genre_label:Genre
+language_label:Language
 characters_label:Characters
+series_label:Series
 ## Completed/In-Progress
 status_label:Status
 ## Dates story first published, last updated, and downloaded(last with time).
@@ -61,12 +63,19 @@ authorId_label:Author ID
 extratags_label:Extra Tags
 ## The version of fanficdownloader
 ##
-version_label:FFD Version
+version_label:FFDL Version

 ## items to include in the title page
 ## Empty entries will *not* appear, even if in the list.
 ## All current formats already include title and author.
-titlepage_entries: category,genre,characters,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,description
+titlepage_entries: series,category,genre,language,characters,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,description
+
+## Try to collect series name and number of this story in series.
+## Some sites (ab)use 'series' for reading lists and personal
+## collections.  This lets us turn it on and off by site without
+## keeping a lengthy titlepage_entries per site and prevents it
+## updating in the plugin.
+collect_series: true

 ## include title page as first page.
 include_titlepage: true
@@ -88,8 +97,10 @@ include_tocpage: true
 #output_filename: books/${title}-${siteabbrev}_${storyId}${formatext}
 #output_filename: books/${formatname}/${siteabbrev}/${authorId}/${title}-${siteabbrev}_${storyId}${formatext}
 output_filename: ${title}-${siteabbrev}_${storyId}${formatext}
+
 ## Make directories as needed.
 make_directories: true
+
 ## Always overwrite output files.  Otherwise, the downloader checks
 ## the timestamp on the existing file and only overwrites if the story
 ## has been updated more recently.  Command line version only
@@ -97,28 +108,83 @@ make_directories: true

 ## put output (with output_filename) in a zip file zip_filename.
 zip_output: false
+
 ## Can include directories.  .zip will be added if not in name somewhere
 zip_filename: ${title}-${siteabbrev}_${storyId}${formatext}.zip

-## Normally, try to make the output file name 'safe' by removing
-## invalid filename chars.  Applies to both output_filename &
-## zip_filename.  
+## Normally, try to make the filenames 'safe' by removing invalid
+## filename chars.  Applies to default_cover_image, output_filename &
+## zip_filename.
 allow_unsafe_filename: false

+## entries to make epub subjects and calibre tags
+## lastupdate creates two tags: "Last Update Year/Month: %Y/%m" and "Last Update: %Y/%m/%d"
+include_subject_tags: extratags, genre, category, characters, lastupdate, status
+
 ## extra tags (comma separated) to include, primarily for epub.
 extratags: FanFiction

 ## number of seconds to sleep between calls to the story site.  May by
 ## useful if pulling large numbers of stories or if the site is slow.
-## Primarily for commandline.
 #slow_down_sleep_time:0.5

+## For use only with stand-alone CLI version--run a command on the
+## generated file after it's produced.  All of the titlepage_entries
+## values are available, plus output_filename.
+#post_process_cmd: addbook -f "${output_filename}" -t "${title}"
+
+## Use regular expressions to find and replace (or remove) metadata.
+## For example, you could change Sci-Fi=>SF, remove *-Centered tags,
+## etc.  See http://docs.python.org/library/re.html (look for re.sub)
+## for regexp details.
+## Make sure to keep at least one space at the start of each line and
+## to escape % to %%, if used.
+#replace_metadata:
+# Sci-Fi=>SF
+# Puella Magi Madoka Magica.* => Madoka
+# Comedy=>Humor
+# Crossover: (.*)=>\1
+# (.*)Great(.*)=>\1Moderate\2
+# .*-Centered=>
+
+## Some readers don't show horizontal rule (<hr />) tags correctly.
+## This replaces them all with a centered '* * *'.  (Note centering
+## doesn't work on some devices either.)
+#replace_hr: false
+
 ## Each output format has a section that overrides [defaults]
 [html]

+## output background color--only used by html and epub (and ignored in
+## epub by many readers).  Included below in output_css--will be
+## ignored if not in output_css.
+background_color: ffffff
+
+## Allow customization of CSS.  Make sure to keep at least one space
+## at the start of each line and to escape % to %%.  Also need
+## background_color to be in the same section, if included in CSS.
+output_css:
+ body { background-color: #%(background_color)s; }
+ .CI {
+     text-align:center;
+     margin-top:0px;
+     margin-bottom:0px;
+     padding:0px;
+     }
+ .center   {text-align: center;}
+ .cover    {text-align: center;}
+ .full     {width: 100%%; }
+ .quarter  {width: 25%%; }
+ .smcap    {font-variant: small-caps;}
+ .u        {text-decoration: underline;}
+ .bold     {font-weight: bold;}
+
 [txt]
 ## Add URLs since there aren't links.
-titlepage_entries: category,genre,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,storyUrl, authorUrl, description
+titlepage_entries: series,category,genre,language,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,storyUrl, authorUrl, description
+
+## Width to word wrap text output.  0 indicates no wrapping.
+wrap_width: 78

 ## use \r\n for line endings, the windows convention.  text output only.
 windows_eol: true
@@ -128,10 +194,6 @@ windows_eol: true
 ## epub is already a zip file.
 zip_output: false

-## entries to make epub subject tags
-## lastupdate creates two tags: "Last Update Year/Month: %Y/%m" and "Last Update: %Y/%m/%d"
-include_subject_tags: extratags, genre, category, characters, lastupdate, status
-
 ## epub carries the TOC in metadata.
 ## mobi generated from epub will have a TOC at the end.
 include_tocpage: false
@@ -142,11 +204,95 @@ titlepage_use_table: false
 ## When using tables, make these span both columns.
 wide_titlepage_entries: description, storyUrl, author URL

+## output background color--only used by html and epub (and ignored in
+## epub by many readers).  Included below in output_css--will be
+## ignored if not in output_css.
+background_color: ffffff
+
+## Allow customization of CSS.  Make sure to keep at least one space
+## at the start of each line and to escape % to %%.  Also need
+## background_color to be in the same section, if included in CSS.
+output_css: 
+ body { background-color: #%(background_color)s;
+        text-align: justify;
+        margin: 2%%; }
+ pre { font-size: x-small; }
+ sml { font-size: small; }
+ h1 { text-align: center; }
+ h2 { text-align: center; }
+ h3 { text-align: center; }
+ h4 { text-align: center; }
+ h5 { text-align: center; }
+ h6 { text-align: center; }
+ .CI {
+     text-align:center;
+     margin-top:0px;
+     margin-bottom:0px;
+     padding:0px;
+     }
+ .center   {text-align: center;}
+ .cover    {text-align: center;}
+ .full     {width: 100%%; }
+ .quarter  {width: 25%%; }
+ .smcap    {font-variant: small-caps;}
+ .u        {text-decoration: underline;}
+ .bold     {font-weight: bold;}
+
+## include images from img tags in the body and summary of
+## stories.  Images will be converted to jpg for size if possible.
+#include_images:false
+
+## If not set, the summary will have all html stripped for safety.
+## Both this and include_images must be true to get images in the
+## summary.
+#keep_summary_html:false
+
+## If set, the first image found will be made the cover image.  If
+## keep_summary_html is true, any images in summary will be before any
+## in chapters.
+#make_firstimage_cover: false
+
+## If set, the epub will never have a cover, even include_images is on
+## and the site has specific cover images.
+#never_make_cover: false
+
+## If set, and there isn't already a cover image from the adapter or
+## from make_firstimage_cover, this image will be made the cover.
+## It can be either a 'file:' or 'http:' url.
+## Note that if you enable make_firstimage_cover in [epub], but want
+## to use default_cover_image for a specific site, use the site:format
+## section, for example: [www.ficwad.com:epub]
+## default_cover_image is a python string Template string with
+## ${title}, ${author} etc, same as titlepage_entries. Unless
+## allow_unsafe_filename is true, invalid filename chars will be
+## removed from metadata fields
+#default_cover_image:file:///C:/Users/username/Desktop/nook/images/icon.png
+#default_cover_image:file:///C:/Users/username/Desktop/nook/images/${title}/icon.png
+#default_cover_image:http://www.somesite.com/someimage.gif
+
+## some sites include images that we don't ever want becoming the
+## cover image.  This lets you exclude them.
+#cover_exclusion_regexp:/stories/999/images/.*?_trophy.png
+
+## Resize images down to width, height, preserving aspect ratio.
+## Nook size, with margin.
+image_max_size: 580, 725
+
+## Change image to grayscale, if graphics library allows, to save
+## space.
+#grayscale_images: false
+
+## if the <img> tag doesn't have a div or a p around it, nook gets
+## confused and displays it on every page after that under the text
+## for the rest of the chapter.  I doubt adding a div around the img
+## will break any other readers, but in case it does, the fix can be
+## turned off.
+nook_img_fix:true
+
 [mobi]
 ## mobi TOC cannot be turned off right now.
 #include_tocpage: true

-
 ## Each site has a section that overrides [defaults] *and* the format
 ## sections test1.com specifically is not a real story site.  Instead,
 ## it is a fake site for testing configuration and output.  It uses
@@ -163,7 +309,200 @@ extratags: FanFiction,Testing,Text
 [test1.com:html]
 extratags: FanFiction,Testing,HTML

+[archive.skyehawke.com]
+
+[ashwinder.sycophanthex.com]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+[castlefans.org]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
+[chaos.sycophanthex.com]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+## some sites include images that we don't ever want becoming the
+## cover image.  This lets you exclude them.
+cover_exclusion_regexp:/images/.*?ribbon.gif
+
+[dramione.org]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
+[erosnsappho.sycophanthex.com]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+## some sites include images that we don't ever want becoming the
+## cover image.  This lets you exclude them.
+cover_exclusion_regexp:/images/.*?ribbon.gif
+
+[fanfiction.mugglenet.com]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
+[fanfiction.portkey.org]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+[fanfiction.tenhawkpresents.com]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+[lumos.sycophanthex.com]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+[nfacommunity.com]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+[occlumency.sycophanthex.com]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+[onedirectionfanfiction.com]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
+[ponyfictionarchive.net]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+[samdean.archive.nu]
+
+[sg1-heliopolis.com]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+[stargate-atlantis.org]
+
+[thehexfiles.net]
+
+[thequidditchpitch.org]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
+[www.adastrafanfic.com]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+[www.thealphagate.com]
+
+[www.archiveofourown.org]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+[www.checkmated.com]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+[www.destinysgateway.com]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+[www.dokuga.com]
+
 [www.fanfiction.net]
+## fanfiction.net's 'cover' images are really just tiny thumbnails.
+## Comment this out or change it to false to use them anyway.
+never_make_cover: true
+
+[www.fanfiktion.de]
+
+[www.ficbook.net]
+
+[www.fictionalley.org]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+## fictionally.org storyIds are not unique.  Combine with authorId.
+output_filename: ${title}-${siteabbrev}_${authorId}_${storyId}${formatext}

 [www.fictionpress.com]
 ## Clear FanFiction from defaults, fictionpress.com is original fiction.
@@ -177,7 +516,30 @@ extratags:
 #username:YourName
 #password:yourpassword

-[www.twilighted.net]
+[www.fimfiction.net]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+## fimfiction.net stories can be locked requiring individual
+## passwords.  If fail_on_password is set, the downloader will fail
+## when a password is required rather than prompting every time.
+#fail_on_password: false
+
+[www.harrypotterfanfiction.com]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+[www.hpfandom.net]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+[www.ik-eternal.net]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -185,18 +547,58 @@ extratags:
 #username:YourName
 #password:yourpassword

-[www.twiwrite.net]
-## Some sites require login (or login for some rated stories) The
-## program can prompt you, or you can save it in config.  In
-## commandline version, this should go in your personal.ini, not
-## defaults.ini.
-#username:YourName
-#password:yourpassword
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true

-[www.whofic.com]
+[www.libraryofmoria.com]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true

 [www.mediaminer.org]

+[www.midnightwhispers.ca]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+## some sites include images that we don't ever want becoming the
+## cover image.  This lets you exclude them.
+cover_exclusion_regexp:/stories/999/images/.*?_trophy.png
+
+[www.ncisfiction.com]
+
+[www.phoenixsong.net]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+[www.potionsandsnitches.net]
+
+[www.prisonbreakfic.net]
+
+[www.siye.co.uk]
+
+[www.squidge.org/peja]
+# www.squidge.org/peja calls it Fandom <shrug>
+category_label:Fandom
+# Remove numWords -- www.squidge.org/peja word counts are inaccurate
+titlepage_entries: series,category,genre,language,characters,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,site,description
+
+[www.squidge.org/peja:txt]
+## Add URLs since there aren't links.
+# Remove numWords -- www.squidge.org/peja word counts are inaccurate
+titlepage_entries: series,category,genre,language,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,site,storyUrl, authorUrl, description
+
+[www.storiesofarda.com]
+
 [www.thewriterscoffeeshop.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -210,7 +612,23 @@ extratags:
 ## personal.ini, not defaults.ini.
 #is_adult:true

-[www.ficwad.com]
+## thewriterscoffeeshop.com (ab)uses series as personal reading lists.
+collect_series: false
+
+[www.tthfanfic.org]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+## tth is a little unusual--it doesn't require user/pass, but the site
+## keeps track of which chapters you've read and won't send another
+## update until it thinks you're up to date.  This way, on download,
+## it thinks you're up to date.
+#username:YourName
+#password:yourpassword
+
+[www.twilighted.net]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -218,28 +636,35 @@ extratags:
 #username:YourName
 #password:yourpassword

-[www.adastrafanfic.com]
+## twilighted.net (ab)uses series as personal reading lists.
+collect_series: false
+
+[www.twiwrite.net]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## twiwrite.net (ab)uses series as personal reading lists.
+collect_series: false
+
+[www.walkingtheplank.org]
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true

-[www.fictionalley.org]
+[www.whofic.com]
+
+[www.wraithbait.com]
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true

-## fictionally.org storyIds are not unique.  Combine with authorId.
-output_filename: ${title}-${siteabbrev}_${authorId}_${storyId}${formatext}
-
-[www.harrypotterfanfiction.com]
-## Some sites do not require a login, but do require the user to
-## confirm they are adult for adult content.  In commandline version,
-## this should go in your personal.ini, not defaults.ini.
-#is_adult:true
-
-[www.fimfiction.net]
+[www.yourfanfiction.com]
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
@@ -17,31 +17,33 @@

 import logging
 ## XXX cli option for logging level.
-logging.basicConfig(level=logging.DEBUG,format="%(levelname)s:%(filename)s(%(lineno)d):%(message)s")
+logging.basicConfig(level=logging.INFO,format="%(levelname)s:%(filename)s(%(lineno)d):%(message)s")

 import sys, os
 from os.path import normpath, expanduser, isfile, join
 from StringIO import StringIO
 from optparse import OptionParser      
 import getpass
+import string
+import ConfigParser
+from subprocess import call

-from epubmerge import doMerge
+from fanficdownloader import adapters,writers,exceptions
+from fanficdownloader.epubutils import get_dcsource_chaptercount, get_update_data
+from fanficdownloader.geturls import get_urls_from_page

 if sys.version_info < (2, 5):
    print "This program requires Python 2.5 or newer."
    sys.exit(1)

-from fanficdownloader import adapters,writers,exceptions
-
-import ConfigParser
-
 def writeStory(config,adapter,writeformat,metaonly=False,outstream=None):
    writer = writers.getWriter(writeformat,config,adapter)
    writer.writeStory(outstream=outstream,metaonly=metaonly)
+    output_filename=writer.getOutputFileName()
    del writer
+    return output_filename

 def main():
-
   # read in args, anything starting with -- will be treated as --<varible>=<value>
   usage = "usage: %prog [options] storyurl"
   parser = OptionParser(usage)
@@ -62,10 +64,16 @@ def main():
                     help="Retrieve metadata and stop.  Or, if --update-epub, update metadata title page only.",)
   parser.add_option("-u", "--update-epub",
                     action="store_true", dest="update",
-                     help="Update an existing epub with new chapter, give epub filename instead of storyurl.  Not compatible with inserted TOC.",)
+                     help="Update an existing epub with new chapter, give epub filename instead of storyurl.",)
+   parser.add_option("--update-cover",
+                     action="store_true", dest="updatecover",
+                     help="Update cover in an existing epub, otherwise existing cover (if any) is used on update.  Only valid with --update-epub.",)
   parser.add_option("--force",
                     action="store_true", dest="force",
-                     help="Force update of an existing epub, download and overwrite all chapters.",)
+                     help="Force overwrite or update of an existing epub, download and overwrite all chapters.",)
+   parser.add_option("-l", "--list",
+                     action="store_true", dest="list",
+                     help="Get list of valid story URLs from page given.",)
   
   (options, args) = parser.parse_args()

@@ -79,14 +87,17 @@ def main():

   conflist = []
   homepath = join(expanduser("~"),".fanficdownloader")
+   
   if isfile(join(homepath,"defaults.ini")):
       conflist.append(join(homepath,"defaults.ini"))
-   if isfile(join(homepath,"personal.ini")):
-       conflist.append(join(homepath,"personal.ini"))
   if isfile("defaults.ini"):
       conflist.append("defaults.ini")
+       
+   if isfile(join(homepath,"personal.ini")):
+       conflist.append(join(homepath,"personal.ini"))
   if isfile("personal.ini"):
       conflist.append("personal.ini")
+       
   if options.configfile:
       conflist.extend(options.configfile)
      
@@ -97,42 +108,64 @@ def main():
       config.add_section("overrides")
   except ConfigParser.DuplicateSectionError:
       pass
+
+   if options.force:
+       config.set("overrides","always_overwrite","true")
+
+   if options.update and not options.updatecover:
+       config.set("overrides","never_make_cover","true")
+       
   if options.options:
       for opt in options.options:
           (var,val) = opt.split('=')
           config.set("overrides",var,val)

+   if options.list:
+       retlist = get_urls_from_page(args[0])
+       print "\n".join(retlist)
+               
+       return
+
   try:
       ## Attempt to update an existing epub.
       if options.update:
-           updateio = StringIO()
-           (url,chaptercount) = doMerge(updateio,
-                                        args,
-                                        titlenavpoints=False,
-                                        striptitletoc=True,
-                                        forceunique=False)
+           (url,chaptercount) = get_dcsource_chaptercount(args[0])
           print "Updating %s, URL: %s" % (args[0],url)
-           filename = args[0]
+           output_filename = args[0]
           config.set("overrides","output_filename",args[0])
       else:
           url = args[0]

-       adapter = adapters.getAdapter(config,url)
-           
-       try:
-           adapter.getStoryMetadataOnly()
-       except exceptions.FailedToLogin:
-           print "Login Failed, Need Username/Password."
-           sys.stdout.write("Username: ")
-           adapter.username = sys.stdin.readline().strip()
-           adapter.password = getpass.getpass(prompt='Password: ')
-           #print("Login: `%s`, Password: `%s`" % (adapter.username, adapter.password))
-           adapter.getStoryMetadataOnly()
-       except exceptions.AdultCheckRequired:
-           print "Please confirm you are an adult in your locale: (y/n)?"
-           if sys.stdin.readline().strip().lower().startswith('y'):
-               adapter.is_adult=True
-           adapter.getStoryMetadataOnly()
+       adapter = adapters.getAdapter(config,url,options.format)
+
+       ## Check for include_images and absence of PIL, give warning.
+       if adapter.getConfig('include_images'):
+           try:
+               import Image
+           except:
+               print "You have include_images enabled, but Python Image Library(PIL) isn't found.\nImages will be included full size in original format.\nContinue? (y/n)?"
+               if not sys.stdin.readline().strip().lower().startswith('y'):
+                   return
+               
+
+       ## three tries, that's enough if both user/pass & is_adult needed,
+       ## or a couple tries of one or the other
+       for x in range(0,2):
+           try:
+               adapter.getStoryMetadataOnly()
+           except exceptions.FailedToLogin, f:
+               if f.passwdonly:
+                   print "Story requires a password."
+               else:
+                   print "Login Failed, Need Username/Password."
+                   sys.stdout.write("Username: ")
+                   adapter.username = sys.stdin.readline().strip()
+               adapter.password = getpass.getpass(prompt='Password: ')
+               #print("Login: `%s`, Password: `%s`" % (adapter.username, adapter.password))
+           except exceptions.AdultCheckRequired:
+               print "Please confirm you are an adult in your locale: (y/n)?"
+               if sys.stdin.readline().strip().lower().startswith('y'):
+                   adapter.is_adult=True

       if options.update and not options.force:
           urlchaptercount = int(adapter.getStoryMetadataOnly().getMetadata('numChapters'))
@@ -143,40 +176,19 @@ def main():
               print "%s contains %d chapters, more than source: %d." % (args[0],chaptercount,urlchaptercount)
           else:
               print "Do update - epub(%d) vs url(%d)" % (chaptercount, urlchaptercount)
-               ## Get updated title page/metadata by itself in an epub.
-               ## Even if the title page isn't included, this carries the metadata.
-               titleio = StringIO()
-               writeStory(config,adapter,"epub",metaonly=True,outstream=titleio)
-
-               newchaptersio = None
               if not options.metaonly:
-                   ## Go get the new chapters only in another epub.
-                   newchaptersio = StringIO()
-                   adapter.setChaptersRange(chaptercount+1,urlchaptercount)
-                   config.set("overrides",'include_tocpage','false')
-                   config.set("overrides",'include_titlepage','false')
-                   writeStory(config,adapter,"epub",outstream=newchaptersio)
-               
-               # out = open("testing/titleio.epub","wb")
-               # out.write(titleio.getvalue())
-               # out.close()
-               
-               # out = open("testing/updateio.epub","wb")
-               # out.write(updateio.getvalue())
-               # out.close()
-               
-               # out = open("testing/newchaptersio.epub","wb")
-               # out.write(newchaptersio.getvalue())
-               # out.close()
-               
-               ## Merge the three epubs together.
-               doMerge(args[0],
-                       [titleio,updateio,newchaptersio],
-                       fromfirst=True,
-                       titlenavpoints=False,
-                       striptitletoc=False,
-                       forceunique=False)

+                   # update now handled by pre-populating the old
+                   # images and chapters in the adapter rather than
+                   # merging epubs.
+                   (url,chaptercount,
+                    adapter.oldchapters,
+                    adapter.oldimgs,
+                    adapter.oldcover,
+                    adapter.calibrebookmark) = get_update_data(args[0])
+
+                   writeStory(config,adapter,"epub")
+                   
       else:
           # regular download
           if options.metaonly:
@@ -184,16 +196,14 @@ def main():
           
           adapter.setChaptersRange(options.begin,options.end)
           
-           if options.format == "all":
-               ## For testing.  Doing all three formats actually causes
-               ## some interesting config issues with format-specific
-               ## sections.  But it should rarely be an issue.
-               writeStory(config,adapter,"epub",options.metaonly)
-               writeStory(config,adapter,"html",options.metaonly)
-               writeStory(config,adapter,"txt",options.metaonly)
-           else:
-               writeStory(config,adapter,options.format,options.metaonly)
+           output_filename=writeStory(config,adapter,options.format,options.metaonly)
       
+       if not options.metaonly and adapter.getConfig("post_process_cmd"):
+           metadata = adapter.story.metadata
+           metadata['output_filename']=output_filename
+           call(string.Template(adapter.getConfig("post_process_cmd"))
+                .substitute(metadata), shell=True)
+           
       del adapter
   
   except exceptions.InvalidStoryURL, isu:
@@ -202,6 +212,9 @@ def main():
       print dne
   except exceptions.UnknownSite, us:
       print us
-   
+
 if __name__ == "__main__":
+    #import time
+    #start = time.time()
    main()
+    #print("Total time seconds:%f"%(time.time()-start))
@@ -2,7 +2,7 @@
 <html>
  <head>
    <link href="/css/index.css" rel="stylesheet" type="text/css">
-    <title>Fanfiction Downloader - read fanfiction from twilighted.net, fanfiction.net, fictionpress.com, fictionalley.org, ficwad.com, potionsandsnitches.net, harrypotterfanfiction.com, mediaminer.org on Kindle, Nook, Sony Reader, iPad, iPhone, Android, Aldiko, Stanza</title>
+    <title>FanFictionDownLoader - read fanfiction from twilighted.net, fanfiction.net, fictionpress.com, fictionalley.org, ficwad.com, potionsandsnitches.net, harrypotterfanfiction.com, mediaminer.org on Kindle, Nook, Sony Reader, iPad, iPhone, Android, Aldiko, Stanza</title>
    <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
    <meta name="google-site-verification" content="kCFc-G4bka_pJN6Rv8CapPBcwmq0hbAUZPkKWqRsAYU" />
    <script type="text/javascript">
@@ -22,7 +22,7 @@
  <body>
    <div id='main' style="width: 80%; margin-left: 10%;">
      <h1>
-	<a href="/" style="text-decoration: none; color: black;">FanFiction Downloader</a>
+	<a href="/" style="text-decoration: none; color: black;">FanFictionDownLoader</a>
      </h1>

      <div style="text-align: center">
@@ -67,7 +67,7 @@
 	<img src="http://code.google.com/appengine/images/appengine-silver-120x30.gif"
 	     alt="Powered by Google App Engine" />
 	<br/><br/>
-	FanfictionLoader is a web front-end to <A href="http://code.google.com/p/fanficdownloader/">fanficdownloader</a><br/>
+	This is a web front-end to <A href="http://code.google.com/p/fanficdownloader/">FanFictionDownLoader</a><br/>
 	Copyright &copy; Fanficdownloader team
      </div>

@@ -16,363 +16,10 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
-import sys
-import os
-import re
-#import StringIO
-from optparse import OptionParser      
-
-import zlib
-import zipfile
-from zipfile import ZipFile, ZIP_STORED, ZIP_DEFLATED
-from time import time
-
-from exceptions import KeyError
-
-from xml.dom.minidom import parse, parseString, getDOMImplementation
-        
-def main(argv):
-    # read in args, anything starting with -- will be treated as --<varible>=<value>
-    usage = "usage: %prog [options] <input epub> [<input epub>...]"
-    parser = OptionParser(usage)
-    parser.add_option("-o", "--output", dest="outputopt", default="merge.epub",
-                      help="Set OUTPUT file, Default: merge.epub", metavar="OUTPUT")
-    parser.add_option("-t", "--title", dest="titleopt", default=None,
-                      help="Use TITLE as the metadata title.  Default: '<first epub title> Anthology'", metavar="TITLE")
-    parser.add_option("-d", "--description", dest="descopt", default=None,
-                      help="Use DESC as the metadata description.  Default: '<epub title> by <author>' for each epub.", metavar="DESC")
-    parser.add_option("-a", "--author",
-                      action="append", dest="authoropts", default=[],
-                      help="Use AUTHOR as a metadata author, multiple authors may be given, Default: <All authors from epubs>", metavar="AUTHOR")
-    parser.add_option("-f", "--first",
-                      action="store_true", dest="fromfirst", default=False,
-                      help="Take all metadata from first input epub",)
-    parser.add_option("-n", "--titles-in-toc",
-                      action="store_true", dest="titlenavpoints",
-                      help="Put an entry in the TOC for each epub, in addition to each epub's chapters.",)
-    parser.add_option("-s", "--strip-title-toc",
-                      action="store_true", dest="striptitletoc",
-                      help="Strip any title_page.xhtml and toc_page.xhtml files.",)
-    
-    (options, args) = parser.parse_args()
-
-    ## Add .epub if not already there.
-    if not options.outputopt.lower().endswith(".epub"):
-        options.outputopt=options.outputopt+".epub"
-
-    print "output file: "+options.outputopt
-    doMerge(options.outputopt,
-            args,
-            options.authoropts,
-            options.titleopt,
-            options.descopt,
-            options.fromfirst,
-            options.titlenavpoints,
-            options.striptitletoc)
-
-    # output = StringIO.StringIO()
-    # files = []
-    # for file in args:
-    #     f = open(file,"rb")
-    #     fio = StringIO.StringIO(f.read())
-    #     f.close()
-    #     files.append(fio)
-
-    # doMerge(output,files,authoropts,titleopt,descopt,fromfirst,titlenavpoints,striptitletoc)
-
-    # out = open(outputopt,"wb")
-    # out.write(output.getvalue())
-    
-def doMerge(outputio,files,authoropts=[],titleopt=None,descopt=None,
-            fromfirst=False,
-            titlenavpoints=True,
-            striptitletoc=False,
-            forceunique=True):
-    '''
-    outputio = output file name or StringIO.
-    files = list of input file names or StringIOs.
-    authoropts = list of authors to use, otherwise add from all input
-    titleopt = title, otherwise '<first title> Anthology'
-    descopt = description, otherwise '<title> by <author>' list for all input
-    fromfirst if true, take all metadata (including author, title, desc) from first input
-    titlenavpoints if true, put in a new TOC entry for each epub
-    striptitletoc if true, strip out any (title|toc)_page.xhtml files
-    forceunique if true, guarantee uniqueness of contents by adding a dir for each input
-    '''
-    ## Python 2.5 ZipFile is rather more primative than later
-    ## versions.  It can operate on a file, or on a StringIO, but
-    ## not on an open stream.  OTOH, I suspect we would have had
-    ## problems with closing and opening again to change the
-    ## compression type anyway.
-
-    filecount=0
-    source=None
-    
-    ## Write mimetype file, must be first and uncompressed.
-    ## Older versions of python(2.4/5) don't allow you to specify
-    ## compression by individual file.
-    ## Overwrite if existing output file.
-    outputepub = ZipFile(outputio, "w", compression=ZIP_STORED)
-    outputepub.debug = 3
-    outputepub.writestr("mimetype", "application/epub+zip")
-    outputepub.close()
-
-    ## Re-open file for content.
-    outputepub = ZipFile(outputio, "a", compression=ZIP_DEFLATED)
-    outputepub.debug = 3
-
-    ## Create META-INF/container.xml file.  The only thing it does is
-    ## point to content.opf
-    containerdom = getDOMImplementation().createDocument(None, "container", None)
-    containertop = containerdom.documentElement
-    containertop.setAttribute("version","1.0")
-    containertop.setAttribute("xmlns","urn:oasis:names:tc:opendocument:xmlns:container")
-    rootfiles = containerdom.createElement("rootfiles")
-    containertop.appendChild(rootfiles)
-    rootfiles.appendChild(newTag(containerdom,"rootfile",{"full-path":"content.opf",
-                                                          "media-type":"application/oebps-package+xml"}))
-    outputepub.writestr("META-INF/container.xml",containerdom.toprettyxml(indent='   ',encoding='utf-8'))    
-
-    ## Process input epubs.
-    
-    items = [] # list of (id, href, type) tuples(all strings) -- From .opfs' manifests
-    items.append(("ncx","toc.ncx","application/x-dtbncx+xml")) ## we'll generate the toc.ncx file,
-                                                               ## but it needs to be in the items manifest.
-    itemrefs = [] # list of strings -- idrefs from .opfs' spines
-    navmaps = [] # list of navMap DOM elements -- TOC data for each from toc.ncx files
-
-    booktitles = [] # list of strings -- Each book's title
-    allauthors = [] # list of lists of strings -- Each book's list of authors.
-
-    filelist = []
-    
-    booknum=1
-    firstmetadom = None
-    for file in files:
-        if file == None : continue
-        
-        book = "%d" % booknum
-        bookdir = ""
-        bookid = ""
-        if forceunique:
-            bookdir = "%d/" % booknum
-            bookid = "a%d" % booknum
-        #print "book %d" % booknum
-        
-        epub = ZipFile(file, 'r')
-
-        ## Find the .opf file.
-        container = epub.read("META-INF/container.xml")
-        containerdom = parseString(container)
-        rootfilenodelist = containerdom.getElementsByTagName("rootfile")
-        rootfilename = rootfilenodelist[0].getAttribute("full-path")
-
-        ## Save the path to the .opf file--hrefs inside it are relative to it.
-        relpath = os.path.dirname(rootfilename)
-        if( len(relpath) > 0 ):
-            relpath=relpath+"/"
-            
-        metadom = parseString(epub.read(rootfilename))
-        if booknum==1:
-            firstmetadom = metadom.getElementsByTagName("metadata")[0]
-            try:
-                source=firstmetadom.getElementsByTagName("dc:source")[0].firstChild.data.encode("utf-8")
-            except:
-                source=""
-            #print "Source:%s"%source
-
-        ## Save indiv book title
-        booktitles.append(metadom.getElementsByTagName("dc:title")[0].firstChild.data)
-
-        ## Save authors.
-        authors=[]
-        for creator in metadom.getElementsByTagName("dc:creator"):
-            if( creator.getAttribute("opf:role") == "aut" ):
-                authors.append(creator.firstChild.data)
-        allauthors.append(authors)
-
-        for item in metadom.getElementsByTagName("item"):
-            if( item.getAttribute("media-type") == "application/x-dtbncx+xml" ):
-                # TOC file is only one with this type--as far as I know.
-                # grab the whole navmap, deal with it later.
-                tocdom = parseString(epub.read(relpath+item.getAttribute("href")))
-                
-                for navpoint in tocdom.getElementsByTagName("navPoint"):
-                    navpoint.setAttribute("id",bookid+navpoint.getAttribute("id"))
-
-                for content in tocdom.getElementsByTagName("content"):
-                    content.setAttribute("src",bookdir+relpath+content.getAttribute("src"))
-
-                navmaps.append(tocdom.getElementsByTagName("navMap")[0])
-            else:
-                id=bookid+item.getAttribute("id")
-                href=bookdir+relpath+item.getAttribute("href")
-                href=href.encode('utf8')
-                #print "href:"+href
-                if not striptitletoc or not re.match(r'.*/(title|toc)_page\.xhtml',
-                                                      item.getAttribute("href")):
-                    if href not in filelist:
-                        try:
-                            outputepub.writestr(href,
-                                                epub.read(relpath+item.getAttribute("href")))
-                            if re.match(r'.*/(file|chapter)\d+\.xhtml',href):
-                                filecount+=1
-                            items.append((id,href,item.getAttribute("media-type")))
-                            filelist.append(href)
-                        except KeyError, ke:
-                            pass # Skip missing files.
-                
-        for itemref in metadom.getElementsByTagName("itemref"):
-            if not striptitletoc or not re.match(r'(title|toc)_page', itemref.getAttribute("idref")):
-                itemrefs.append(bookid+itemref.getAttribute("idref"))
-
-        booknum=booknum+1;
-
-    ## create content.opf file. 
-    uniqueid="epubmerge-uid-%d" % time() # real sophisticated uid scheme.
-    contentdom = getDOMImplementation().createDocument(None, "package", None)
-    package = contentdom.documentElement
-    if fromfirst and firstmetadom:
-        metadata = firstmetadom
-        firstpackage = firstmetadom.parentNode
-        package.setAttribute("version",firstpackage.getAttribute("version"))
-        package.setAttribute("xmlns",firstpackage.getAttribute("xmlns"))
-        package.setAttribute("unique-identifier",firstpackage.getAttribute("unique-identifier"))
-    else:
-        package.setAttribute("version","2.0")
-        package.setAttribute("xmlns","http://www.idpf.org/2007/opf")
-        package.setAttribute("unique-identifier","epubmerge-id")
-        metadata=newTag(contentdom,"metadata",
-                        attrs={"xmlns:dc":"http://purl.org/dc/elements/1.1/",
-                               "xmlns:opf":"http://www.idpf.org/2007/opf"})
-        metadata.appendChild(newTag(contentdom,"dc:identifier",text=uniqueid,attrs={"id":"epubmerge-id"}))
-        if( titleopt is None ):
-            titleopt = booktitles[0]+" Anthology"
-        metadata.appendChild(newTag(contentdom,"dc:title",text=titleopt))
-    
-        # If cmdline authors, use those instead of those collected from the epubs
-        # (allauthors kept for TOC & description gen below.
-        if( len(authoropts) > 1  ):
-            useauthors=[authoropts]
-        else:
-            useauthors=allauthors
-            
-        usedauthors=dict()
-        for authorlist in useauthors:
-            for author in authorlist:
-                if( not usedauthors.has_key(author) ):
-                    usedauthors[author]=author
-                    metadata.appendChild(newTag(contentdom,"dc:creator",
-                                                attrs={"opf:role":"aut"},
-                                                text=author))
-    
-        metadata.appendChild(newTag(contentdom,"dc:contributor",text="epubmerge",attrs={"opf:role":"bkp"}))
-        metadata.appendChild(newTag(contentdom,"dc:rights",text="Copyrights as per source stories"))
-        metadata.appendChild(newTag(contentdom,"dc:language",text="en"))
-    
-        if not descopt:
-            # created now, but not filled in until TOC generation to save loops.
-            description = newTag(contentdom,"dc:description",text="Anthology containing:\n")
-        else:
-            description = newTag(contentdom,"dc:description",text=descopt)
-        metadata.appendChild(description)
-    
-    package.appendChild(metadata)
-    
-    manifest = contentdom.createElement("manifest")
-    package.appendChild(manifest)
-    for item in items:
-        (id,href,type)=item
-        manifest.appendChild(newTag(contentdom,"item",
-                                       attrs={'id':id,
-                                              'href':href,
-                                              'media-type':type}))
-        
-    spine = newTag(contentdom,"spine",attrs={"toc":"ncx"})
-    package.appendChild(spine)
-    for itemref in itemrefs:
-        spine.appendChild(newTag(contentdom,"itemref",
-                                    attrs={"idref":itemref,
-                                           "linear":"yes"}))
-
-    ## create toc.ncx file
-    tocncxdom = getDOMImplementation().createDocument(None, "ncx", None)
-    ncx = tocncxdom.documentElement
-    ncx.setAttribute("version","2005-1")
-    ncx.setAttribute("xmlns","http://www.daisy.org/z3986/2005/ncx/")
-    head = tocncxdom.createElement("head")
-    ncx.appendChild(head)
-    head.appendChild(newTag(tocncxdom,"meta",
-                            attrs={"name":"dtb:uid", "content":uniqueid}))
-    head.appendChild(newTag(tocncxdom,"meta",
-                            attrs={"name":"dtb:depth", "content":"1"}))
-    head.appendChild(newTag(tocncxdom,"meta",
-                            attrs={"name":"dtb:totalPageCount", "content":"0"}))
-    head.appendChild(newTag(tocncxdom,"meta",
-                            attrs={"name":"dtb:maxPageNumber", "content":"0"}))
-    
-    docTitle = tocncxdom.createElement("docTitle")
-    docTitle.appendChild(newTag(tocncxdom,"text",text=titleopt))
-    ncx.appendChild(docTitle)
-    
-    tocnavMap = tocncxdom.createElement("navMap")
-    ncx.appendChild(tocnavMap)
-
-    ## TOC navPoints can be nested, but this flattens them for
-    ## simplicity, plus adds a navPoint for each epub.
-    booknum=0
-    for navmap in navmaps:
-        navpoints = navmap.getElementsByTagName("navPoint")
-        if titlenavpoints:
-           ## Copy first navPoint of each epub, give a different id and
-           ## text: bookname by authorname
-            newnav = navpoints[0].cloneNode(True)
-            newnav.setAttribute("id","book"+newnav.getAttribute("id"))
-            ## For purposes of TOC titling & desc, use first book author
-            newtext = newTag(tocncxdom,"text",text=booktitles[booknum]+" by "+allauthors[booknum][0])            
-            text = newnav.getElementsByTagName("text")[0]
-            text.parentNode.replaceChild(newtext,text)
-            tocnavMap.appendChild(newnav)
-        
-        if not descopt and not fromfirst:
-            description.appendChild(contentdom.createTextNode(booktitles[booknum]+" by "+allauthors[booknum][0]+"\n"))
-            
-        for navpoint in navpoints:
-            #print "navpoint:%s"%navpoint.getAttribute("id")
-            if not striptitletoc or not re.match(r'(title|toc)_page',navpoint.getAttribute("id")):
-                tocnavMap.appendChild(navpoint)
-        booknum=booknum+1;
-
-    ## Force strict ordering of playOrder
-    playorder=1
-    for navpoint in tocncxdom.getElementsByTagName("navPoint"):
-        navpoint.setAttribute("playOrder","%d" % playorder)
-        if( not navpoint.getAttribute("id").startswith("book") ):
-            playorder = playorder + 1
-
-    ## content.opf written now due to description being filled in
-    ## during TOC generation to save loops.
-    outputepub.writestr("content.opf",contentdom.toxml('utf-8'))
-    outputepub.writestr("toc.ncx",tocncxdom.toxml('utf-8'))
-    
-    # declares all the files created by Windows.  otherwise, when
-    # it runs in appengine, windows unzips the files as 000 perms.
-    for zf in outputepub.filelist:
-        zf.create_system = 0
-    outputepub.close()
-
-    return (source,filecount)
-
-## Utility method for creating new tags.
-def newTag(dom,name,attrs=None,text=None):
-    tag = dom.createElement(name)
-    if( attrs is not None ):
-        for attr in attrs.keys():
-            tag.setAttribute(attr,attrs[attr])
-    if( text is not None ):
-        tag.appendChild(dom.createTextNode(text))
-    return tag
    
 if __name__ == "__main__":
-    main(sys.argv[1:])
+    print('''
+The this utility has been split out into it's own project. 
+See: http://code.google.com/p/epubmerge/
+...for a CLI epubmerge.py program and calibre plugin.
+''')
@@ -1,6 +1,12 @@
 ## This is an example of what your personal configuration might look
 ## like.

+[defaults]
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
 ## Most common, I expect will be using this to save username/passwords
 ## for different sites.
 [www.twilighted.net]
@@ -1,78 +1,136 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2011 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import os, re, sys, glob
-from os.path import dirname, basename, normpath
-import logging
-import urlparse as up
-
-import fanficdownloader.exceptions as exceptions
-
-## This bit of complexity allows adapters to be added by just adding
-## the source file.  It eliminates the long if/else clauses we used to
-## need to pick out the adapter.
-    
-## List of registered site adapters.
-    
-__class_list = []
-
-def getAdapter(config,url):
-    ## fix up leading protocol.
-    fixedurl = re.sub(r"(?i)^[htp]+[:/]+","http://",url.strip())
-    if not fixedurl.startswith("http"):
-        fixedurl = "http://%s"%url
-    ## remove any trailing '#' locations.
-    fixedurl = re.sub(r"#.*$","",fixedurl)
-    
-    ## remove any trailing '&' parameters--?sid=999 will be left.
-    ## that's all that any of the current adapters need or want.
-    fixedurl = re.sub(r"&.*$","",fixedurl)
-    
-    parsedUrl = up.urlparse(fixedurl)
-    domain = parsedUrl.netloc.lower()
-    if( domain != parsedUrl.netloc ):
-        fixedurl = fixedurl.replace(parsedUrl.netloc,domain)
-
-    logging.debug("site:"+domain)
-    cls = getClassFor(domain)
-    if not cls:
-        logging.debug("trying site:www."+domain)
-        cls = getClassFor("www."+domain)
-        fixedurl = fixedurl.replace("http://","http://www.")
-    if cls:
-        adapter = cls(config,fixedurl) # raises InvalidStoryURL
-        return adapter
-    # No adapter found.
-    raise exceptions.UnknownSite( url, [cls.getSiteDomain() for cls in __class_list] )
-
-def getClassFor(domain):
-    for cls in __class_list:
-        if cls.matchesSite(domain):
-            return cls
-
-## Automatically import each adapter_*.py file.
-## Each implement getClass() to their class
-
-filelist = glob.glob(dirname(__file__)+'/adapter_*.py')
-sys.path.insert(0,normpath(dirname(__file__)))
-
-for file in filelist:
-    #print "file: "+basename(file)[:-3]
-    module = __import__(basename(file)[:-3])
-    __class_list.append(module.getClass())
-
-del sys.path[0]
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import os, re, sys, glob, types
+from os.path import dirname, basename, normpath
+import logging
+import urlparse as up
+
+from .. import exceptions as exceptions
+
+## must import each adapter here.
+
+import adapter_test1
+import adapter_fanfictionnet
+import adapter_castlefansorg
+import adapter_fictionalleyorg
+import adapter_fictionpresscom
+import adapter_ficwadcom
+import adapter_fimfictionnet
+import adapter_harrypotterfanfictioncom
+import adapter_mediaminerorg
+import adapter_potionsandsnitchesnet
+import adapter_tenhawkpresentscom
+import adapter_adastrafanficcom
+import adapter_thewriterscoffeeshopcom
+import adapter_tthfanficorg
+import adapter_twilightednet
+import adapter_twiwritenet
+import adapter_whoficcom
+import adapter_siyecouk
+import adapter_archiveofourownorg
+import adapter_ficbooknet
+import adapter_portkeyorg
+import adapter_mugglenetcom
+import adapter_hpfandomnet
+import adapter_thequidditchpitchorg
+import adapter_nfacommunitycom
+import adapter_midnightwhispersca
+import adapter_ksarchivecom
+import adapter_archiveskyehawkecom
+import adapter_squidgeorgpeja
+import adapter_libraryofmoriacom
+import adapter_wraithbaitcom
+import adapter_checkmatedcom
+import adapter_chaossycophanthexcom
+import adapter_dramioneorg
+import adapter_erosnsapphosycophanthexcom
+import adapter_lumossycophanthexcom
+import adapter_occlumencysycophanthexcom
+import adapter_phoenixsongnet
+import adapter_walkingtheplankorg
+import adapter_ashwindersycophanthexcom
+import adapter_thehexfilesnet
+import adapter_dokugacom
+import adapter_iketernalnet
+import adapter_onedirectionfanfictioncom
+import adapter_prisonbreakficnet
+import adapter_storiesofardacom
+import adapter_yourfanfictioncom
+import adapter_samdeanarchivenu
+import adapter_destinysgatewaycom
+import adapter_ncisfictioncom
+import adapter_stargateatlantisorg
+import adapter_thealphagatecom
+import adapter_fanfiktionde
+import adapter_ponyfictionarchivenet
+import adapter_sg1heliopoliscom
+
+## This bit of complexity allows adapters to be added by just adding
+## importing.  It eliminates the long if/else clauses we used to need
+## to pick out the adapter.
+    
+## List of registered site adapters.
+__class_list = []
+
+def imports():
+    for name, val in globals().items():
+        if isinstance(val, types.ModuleType):
+            yield val.__name__
+
+for x in imports():
+    if "fanficdownloader.adapters.adapter_" in x:
+        #print x
+        __class_list.append(sys.modules[x].getClass())
+            
+def getAdapter(config,url,fileform=None):
+    ## fix up leading protocol.
+    fixedurl = re.sub(r"(?i)^[htp]+[:/]+","http://",url.strip())
+    if not fixedurl.startswith("http"):
+        fixedurl = "http://%s"%url
+    ## remove any trailing '#' locations.
+    fixedurl = re.sub(r"#.*$","",fixedurl)
+    
+    ## remove any trailing '&' parameters--?sid=999 will be left.
+    ## that's all that any of the current adapters need or want.
+    fixedurl = re.sub(r"&.*$","",fixedurl)
+    
+    parsedUrl = up.urlparse(fixedurl)
+    domain = parsedUrl.netloc.lower()
+    if( domain != parsedUrl.netloc ):
+        fixedurl = fixedurl.replace(parsedUrl.netloc,domain)
+
+    logging.debug("site:"+domain)
+    cls = getClassFor(domain)
+    if not cls:
+        logging.debug("trying site:www."+domain)
+        cls = getClassFor("www."+domain)
+        fixedurl = fixedurl.replace("http://","http://www.")
+    if cls:
+        adapter = cls(config,fixedurl) # raises InvalidStoryURL
+        adapter.setSectionOrder(adapter.getConfigSection(),fileform)
+        return adapter
+    # No adapter found.
+    raise exceptions.UnknownSite( url, [cls.getSiteDomain() for cls in __class_list] )
+
+def getConfigSections():
+    return [cls.getConfigSection() for cls in __class_list]
+
+def getClassFor(domain):
+    for cls in __class_list:
+        if cls.matchesSite(domain):
+            return cls
@@ -21,11 +21,11 @@ import re
 import urllib
 import urllib2

-import fanficdownloader.BeautifulSoup as bs
-from fanficdownloader.htmlcleanup import stripHTML
-import fanficdownloader.exceptions as exceptions
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions

-from base_adapter import BaseSiteAdapter, utf8FromSoup, makeDate
+from base_adapter import BaseSiteAdapter,  makeDate

 class AdAstraFanficComSiteAdapter(BaseSiteAdapter):

@@ -79,6 +79,11 @@ class AdAstraFanficComSiteAdapter(BaseSiteAdapter):
        if "Content is only suitable for mature adults. May contain explicit language and adult themes. Equivalent of NC-17." in data:
            raise exceptions.AdultCheckRequired(self.url)
            
+        # problems with some stories, but only in calibre.  I suspect
+        # issues with different SGML parsers in python.  This is a
+        # nasty hack, but it works.
+        data = data[data.index("<body"):]
+        
        # use BeautifulSoup HTML parser to make everything easier to find.
        soup = bs.BeautifulSoup(data)

@@ -128,7 +133,8 @@ class AdAstraFanficComSiteAdapter(BaseSiteAdapter):
                # sometimes poorly formated desc (<p> w/o </p>) leads
                # to all labels being included.
                svalue=svalue[:svalue.find('<span class="label">')]
-                self.story.setMetadata('description',stripHTML(svalue))
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))

            if 'Rated' in label:
                self.story.setMetadata('rating', value)
@@ -142,6 +148,12 @@ class AdAstraFanficComSiteAdapter(BaseSiteAdapter):
                for cat in catstext:
                    self.story.addToList('category',cat.string)

+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                charstext = [char.string for char in chars]
+                for char in charstext:
+                    self.story.addToList('characters',char.string)
+
            if 'Genre' in label:
                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
                genrestext = [genre.string for genre in genres]
@@ -163,19 +175,45 @@ class AdAstraFanficComSiteAdapter(BaseSiteAdapter):
                    self.story.setMetadata('status', 'In-Progress')

            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(value.strip(), "%m/%d/%Y"))
+                self.story.setMetadata('datePublished', makeDate(value.strip(), "%d %b %Y"))
            
            if 'Updated' in label:
                # there's a stray [ at the end.
                #value = value[0:-1]
-                self.story.setMetadata('dateUpdated', makeDate(value.strip(), "%m/%d/%Y"))
+                self.story.setMetadata('dateUpdated', makeDate(value.strip(), "%d %b %Y"))

+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            

    def getChapterText(self, url):

        logging.debug('Getting chapter text from: %s' % url)

-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+        data = self._fetchUrl(url)
+        # problems with some stories, but only in calibre.  I suspect
+        # issues with different SGML parsers in python.  This is a
+        # nasty hack, but it works.
+        data = data[data.index("<body"):]
+        
+        soup = bs.BeautifulStoneSoup(data,
                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
        
        span = soup.find('div', {'id' : 'story'})
@@ -183,7 +221,7 @@ class AdAstraFanficComSiteAdapter(BaseSiteAdapter):
        if None == span:
            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
    
-        return utf8FromSoup(span)
+        return self.utf8FromSoup(url,span)

 def getClass():
    return AdAstraFanficComSiteAdapter
@@ -0,0 +1,267 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return ArchiveOfOurOwnOrgAdapter
+
+
+class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["utf8",
+                       "Windows-1252"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+							   
+							   
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/works/'+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','ao3')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%Y-%b-%d"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.archiveofourown.org'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/works/123456"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/works/")+r"\d+(/chapters/\d+)?/?$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            addurl = "?view_adult=true"
+        else:
+            addurl=""
+
+        meta = self.url+addurl
+        url = self.url+'/navigate'+addurl
+        logging.debug("URL: "+meta)
+
+        try:
+            data = self._fetchUrl(url)
+            meta = self._fetchUrl(meta)
+
+            if "This work could have adult content. If you proceed you have agreed that you are willing to see such content." in meta:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.meta)
+            else:
+                raise e
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        metasoup = bs.BeautifulSoup(meta)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r"^/works/\d+$"))
+        self.story.setMetadata('title',a.string)
+		
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"^/users/\w+/pseuds/\w+"))
+        if a == None: # ao3 allows for author 'Anonymous' with no author link.
+            self.story.setMetadata('author','Anonymous')
+            self.story.setMetadata('authorUrl',self.url)
+        else:
+            self.story.setMetadata('authorId',a['href'].split('/')[2])
+            self.story.setMetadata('authorUrl','http://'+self.host+a['href'])
+            self.story.setMetadata('author',a.text)
+
+        # Find the chapters:
+        chapters=soup.findAll('a', href=re.compile(r'/works/'+self.story.getMetadata('storyId')+"/chapters/\d+$"))
+        self.story.setMetadata('numChapters',len(chapters))
+        logging.debug("numChapters: (%s)"%self.story.getMetadata('numChapters'))
+        for x in range(0,len(chapters)):
+            # just in case there's tags, like <i> in chapter titles.
+            chapter=chapters[x]
+            if len(chapters)==1:
+                self.chapterUrls.append((self.story.getMetadata('title'),'http://'+self.host+chapter['href']+addurl))
+            else:
+                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+chapter['href']+addurl))
+
+
+
+        a = metasoup.find('blockquote',{'class':'userstuff'})
+        if a != None:
+            self.setDescription(url,a.text)
+            #self.story.setMetadata('description',a.text)
+		
+        a = metasoup.find('dd',{'class':"rating tags"})
+        if a != None:
+            self.story.setMetadata('rating',stripHTML(a.text))
+		
+        a = metasoup.find('dd',{'class':"fandom tags"})
+        fandoms = a.findAll('a',{'class':"tag"})
+        for fandom in fandoms:
+            self.story.addToList('category',fandom.string)
+		
+        a = metasoup.find('dd',{'class':"warning tags"})
+        if a != None:
+            warnings = a.findAll('a',{'class':"tag"})
+            for warning in warnings:
+                if warning.string == "Author Chose Not To Use Archive Warnings":
+                    warning.string = "No Archive Warnings Apply"
+                if warning.string != "No Archive Warnings Apply":
+                    self.story.addToList('warnings',warning.string)
+		
+        a = metasoup.find('dd',{'class':"freeform tags"})
+        if a != None:
+            genres = a.findAll('a',{'class':"tag"})
+            for genre in genres:
+                self.story.addToList('genre',genre.string)
+        a = metasoup.find('dd',{'class':"category tags"})
+        if a != None:
+            genres = a.findAll('a',{'class':"tag"})
+            for genre in genres:
+                if genre != "Gen":
+                    self.story.addToList('genre',genre.string)
+		
+        a = metasoup.find('dd',{'class':"character tags"})
+        if a != None:
+            chars = a.findAll('a',{'class':"tag"})
+            for char in chars:
+                self.story.addToList('characters',char.string)
+        a = metasoup.find('dd',{'class':"relationship tags"})
+        if a != None:
+            chars = a.findAll('a',{'class':"tag"})
+            for char in chars:
+                self.story.addToList('characters',char.string)
+		
+
+        stats = metasoup.find('dl',{'class':'stats'})
+        dt = stats.findAll('dt')
+        dd = stats.findAll('dd')
+        for x in range(0,len(dt)):
+            label = dt[x].text
+            value = dd[x].text
+
+            if 'Words:' in label:
+                self.story.setMetadata('numWords', value)
+				
+            if 'Chapters:' in label:
+                if value.split('/')[0] == value.split('/')[1]:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+				
+            if 'Completed' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+		
+        try:
+            # Find Series name from series URL.
+            a = metasoup.find('dd',{'class':"series"})
+            b = a.find('a', href=re.compile(r"/series/\d+"))
+            series_name = b.string
+            series_url = 'http://'+self.host+'/fanfic/'+b['href']
+            series_index = int(a.text.split(' ')[1])
+            self.setSeries(series_name, series_index)
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+        logging.debug('Getting chapter text from: %s' % url)
+		
+        chapter=bs.BeautifulSoup('<div class="story"></div>')
+        data = self._fetchUrl(url)
+        soup = bs.BeautifulSoup(data,selfClosingTags=('br','hr'))
+		
+        headnotes = soup.find('div', {'class' : "preface group"}).find('div', {'class' : "notes module"})
+        if headnotes != None:
+            headnotes = headnotes.find('blockquote', {'class' : "userstuff"})
+            if headnotes != None:
+                chapter.append("<b>Author's Note:</b>")
+                chapter.append(headnotes)
+        
+        chapsumm = soup.find('div', {'id' : "summary"})
+        if chapsumm != None:
+            chapsumm = chapsumm.find('blockquote')
+            chapter.append("<b>Summary for the Chapter:</b>")
+            chapter.append(chapsumm)
+        chapnotes = soup.find('div', {'id' : "notes"})
+        if chapnotes != None:
+            chapnotes = chapnotes.find('blockquote')
+            if chapnotes != None:
+                chapter.append("<b>Notes for the Chapter:</b>")
+                chapter.append(chapnotes)
+		
+        text = soup.find('div', {'class' : "userstuff module"})
+        chtext = text.find('h3', {'class' : "landmark heading"})
+        if chtext:
+            chtext.extract()
+        chapter.append(text)
+		
+        chapfoot = soup.find('div', {'class' : "end notes module", 'role' : "complementary"})
+        if chapfoot != None:
+            chapfoot = chapfoot.find('blockquote')
+            chapter.append("<b>Notes for the Chapter:</b>")
+            chapter.append(chapfoot)
+		
+        footnotes = soup.find('div', {'id' : "work_endnotes"})
+        if footnotes != None:
+            footnotes = footnotes.find('blockquote')
+            chapter.append("<b>Author's Note:</b>")
+            chapter.append(footnotes)
+			
+        if None == soup:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+
+        return self.utf8FromSoup(url,chapter)
@@ -0,0 +1,190 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+
+def getClass():
+    return ArchiveSkyeHawkeComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class ArchiveSkyeHawkeComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/story.php?no='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','ash')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%Y-%m-%d"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'archive.skyehawke.com'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/story.php?no=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/story.php?no=")+r"\d+$"
+
+
+        
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        url = self.url
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        data = self._fetchUrl(url)
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('div', {'class':"story border"}).find('span',{'class':'left'})
+        title=a.text.split('"')[1]
+        self.story.setMetadata('title',title)
+        
+        # Find authorid and URL from... author url.
+        author = a.find('a')
+        self.story.setMetadata('authorId',author['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+author['href'])
+        self.story.setMetadata('author',author.string)
+		
+        authorSoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
+		
+        chapter=soup.find('select',{'name':'chapter'}).findAll('option')
+	
+        for i in range(1,len(chapter)):
+            ch=chapter[i]
+            self.chapterUrls.append((stripHTML(ch),ch['value']))
+		
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+		
+        box=soup.find('div', {'class': "container borderridge"})
+        sum=box.find('span').text
+        self.setDescription(url,sum)
+
+        boxes=soup.findAll('div', {'class': "container bordersolid"})
+        for box in boxes:
+            if box.find('b') != None and box.find('b').text == "History and Story Information":
+			
+                for b in box.findAll('b'):
+                    if "words" in b.nextSibling:
+                        self.story.setMetadata('numWords', b.text)
+                    if "archived" in b.previousSibling:
+                        self.story.setMetadata('datePublished', makeDate(stripHTML(b.text), self.dateformat))
+                    if "updated" in b.previousSibling:
+                        self.story.setMetadata('dateUpdated', makeDate(stripHTML(b.text), self.dateformat))
+                    if "fandom" in b.nextSibling:
+                        self.story.addToList('category', b.text)
+						
+                for br in box.findAll('br'):
+                    br.replaceWith('split')
+                genre=box.text.split("Genre:")[1].split("split")[0]
+                if not "Unspecified" in genre:
+                    self.story.addToList('genre',genre)
+			
+			
+            if box.find('span') != None and box.find('span').text == "WARNING":
+			
+                rating=box.findAll('span')[1]
+                rating.find('br').replaceWith('split')
+                rating=rating.text.replace("This story is rated",'').split('split')[0]
+                self.story.setMetadata('rating',rating)
+                logging.debug(self.story.getMetadata('rating'))
+			
+                warnings=box.find('ol')
+                if warnings != None:
+                    warnings=warnings.text.replace(']', '').replace('[', '').split('  ')
+                    for warning in warnings:
+                        self.story.addToList('warnings',warning)
+	
+	
+        for asoup in authorSoup.findAll('div', {'class':"story bordersolid"}):
+            if asoup.find('a')['href'] == 'story.php?no='+self.story.getMetadata('storyId'):
+                if '[ Completed ]' in asoup.text:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+                chars=asoup.findNext('div').text.split('Characters')[1].split(']')[0]
+                for char in chars.split(','):
+                    if not "None" in char:
+                        self.story.addToList('characters',char)
+                break
+	
+	
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div',{'class':"chapter bordersolid"}).findNext('div').findNext('div')
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -0,0 +1,256 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return AshwinderSycophantHexComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class AshwinderSycophantHexComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+		
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','asph')
+
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","Harry Potter")
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%m/%d/%Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'ashwinder.sycophanthex.com'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'This story contains adult content and/or themes.' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['rememberme'] = '1'
+        params['sid'] = ''
+        params['intent'] = ''
+        params['submit'] = 'Submit'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/user.php'
+        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "Logout" not in d : #Member Account
+            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+            
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
+		
+        try:
+            # in case link points somewhere other than the first chapter
+            a = soup.findAll('option')[1]['value']
+            self.story.setMetadata('storyId',a.split('=',)[1])
+            url = 'http://'+self.host+'/'+a
+            soup = bs.BeautifulSoup(self._fetchUrl(url))
+        except:
+            pass
+		
+        for info in asoup.findAll('table', {'width' : '100%', 'bordercolor' : re.compile(r'#')}):
+            a = info.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+            if a != None:
+                self.story.setMetadata('title',a.string)
+                break
+		
+
+        # Find the chapters:
+        chapters=soup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+&i=1$'))
+        if len(chapters) == 0:
+            self.chapterUrls.append((self.story.getMetadata('title'),url))
+        else:
+            for chapter in chapters:
+                # just in case there's tags, like <i> in chapter titles.
+                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d):
+            try:
+                return d.name
+            except:
+                return ""
+				
+        cats = info.findAll('a',href=re.compile('categories.php'))
+        for cat in cats:
+            self.story.addToList('category',cat.string)
+					
+        a = info.find('a', href=re.compile(r'reviews.php\?sid='+self.story.getMetadata('storyId')))
+        val = a.nextSibling
+        svalue = ""
+        while not defaultGetattr(val) == 'br':
+            val = val.nextSibling
+        val = val.nextSibling
+        while not defaultGetattr(val) == 'table':
+            svalue += str(val)
+            val = val.nextSibling
+        self.setDescription(url,svalue)
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = info.findAll('b')
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = stripHTML(labelspan)
+
+            if 'Rating' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word Count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Genres' in label:
+                genres = value.string.split(', ')
+                for genre in genres:
+                    if genre != 'none':
+                        self.story.addToList('genre',genre)
+
+            if 'Warnings' in label:
+                warnings = value.string.split(', ')
+                for warning in warnings:
+                    if warning != ' none':
+                        self.story.addToList('warnings',warning)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+		
+        data = self._fetchUrl(url)
+
+        soup = bs.BeautifulSoup(data, selfClosingTags=('br','hr','span','center')) # some chapters seem to be hanging up on those tags, so it is safer to close them
+        
+        story = soup.find('div', {"align" : "left"})
+
+        if None == story:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,story)
@@ -20,11 +20,11 @@ import logging
 import re
 import urllib2

-import fanficdownloader.BeautifulSoup as bs
-from fanficdownloader.htmlcleanup import stripHTML
-import fanficdownloader.exceptions as exceptions
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions

-from base_adapter import BaseSiteAdapter, utf8FromSoup, makeDate
+from base_adapter import BaseSiteAdapter,  makeDate

 # By virtue of being recent and requiring both is_adult and user/pass,
 # adapter_fanficcastletvnet.py is the best choice for learning to
@@ -51,11 +51,11 @@ from base_adapter import BaseSiteAdapter, utf8FromSoup, makeDate
 # updated to reflect the class below it.  That, plus getSiteDomain()
 # take care of 'Registering'.
 def getClass():
-    return FanficCastleTVNetAdapter # XXX
+    return CastleFansOrgAdapter # XXX

 # Class name has to be unique.  Our convention is camel case the
 # sitename with Adapter at the end.  www is skipped.
-class FanficCastleTVNetAdapter(BaseSiteAdapter): # XXX
+class CastleFansOrgAdapter(BaseSiteAdapter): # XXX

    def __init__(self, config, url):
        BaseSiteAdapter.__init__(self, config, url)
@@ -74,10 +74,11 @@ class FanficCastleTVNetAdapter(BaseSiteAdapter): # XXX
        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
        
        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
+        self._setURL('http://' + self.getSiteDomain() + '/fanfic/viewstory.php?sid='+self.story.getMetadata('storyId'))
        
        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','csltv') # XXX
+        self.story.setMetadata('siteabbrev','cslf') # XXX

        # If all stories from the site fall into the same category,
        # the site itself isn't likely to label them as such, so we
@@ -91,13 +92,13 @@ class FanficCastleTVNetAdapter(BaseSiteAdapter): # XXX
    @staticmethod # must be @staticmethod, don't remove it.
    def getSiteDomain():
        # The site domain.  Does have www here, if it uses it.
-        return 'fanfic.castletv.net' # XXX
+        return 'castlefans.org' # XXX

    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+        return "http://"+self.getSiteDomain()+"/fanfic/viewstory.php?sid=1234"

    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+        return re.escape("http://"+self.getSiteDomain()+"/fanfic/viewstory.php?sid=")+r"\d+$"

    ## Login seems to be reasonably standard across eFiction sites.
    def needToLoginCheck(self, data):
@@ -120,7 +121,7 @@ class FanficCastleTVNetAdapter(BaseSiteAdapter): # XXX
        params['cookiecheck'] = '1'
        params['submit'] = 'Submit'
    
-        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
+        loginUrl = 'http://' + self.getSiteDomain() + '/fanfic/user.php?action=login'
        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                              params['penname']))
    
@@ -192,7 +193,7 @@ class FanficCastleTVNetAdapter(BaseSiteAdapter): # XXX
        # Find the chapters:
        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
            # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/fanfic/'+chapter['href']+addurl))

        self.story.setMetadata('numChapters',len(self.chapterUrls))

@@ -218,7 +219,8 @@ class FanficCastleTVNetAdapter(BaseSiteAdapter): # XXX
                while not defaultGetattr(value,'class') == 'label':
                    svalue += str(value)
                    value = value.nextSibling
-                self.story.setMetadata('description',stripHTML(svalue))
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))

            if 'Rated' in label:
                self.story.setMetadata('rating', value)
@@ -232,6 +234,12 @@ class FanficCastleTVNetAdapter(BaseSiteAdapter): # XXX
                for cat in catstext:
                    self.story.addToList('category',cat.string)

+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                charstext = [char.string for char in chars]
+                for char in charstext:
+                    self.story.addToList('characters',char.string)
+
            ## Not all sites use Genre, but there's no harm to
            ## leaving it in.  Check to make sure the type_id number
            ## is correct, though--it's site specific.
@@ -266,6 +274,26 @@ class FanficCastleTVNetAdapter(BaseSiteAdapter): # XXX
                #value = value[0:-1]
                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))

+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/fanfic/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
    # grab the text for an individual chapter.
    def getChapterText(self, url):

@@ -279,4 +307,4 @@ class FanficCastleTVNetAdapter(BaseSiteAdapter): # XXX
        if None == div:
            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
    
-        return utf8FromSoup(div)
+        return self.utf8FromSoup(url,div)
@@ -0,0 +1,239 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return ChaosSycophantHexComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class ChaosSycophantHexComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+		
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','csph')
+
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        #self.story.addToList("category","Harry Potter")
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%m/%d/%Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'chaos.sycophanthex.com'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=19"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        # The actual text that is used to announce you need to be an
+        # adult varies from site to site.  Again, print data before
+        # the title search to troubleshoot.
+        if "Age Consent Required" in data:
+            raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        pt = soup.find('div', {'id' : 'pagetitle'})
+        a = pt.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.text)
+        
+        # Find authorid and URL from... author url.
+        a = pt.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        rating=pt.text.split('(')[1].split(')')[0]
+        self.story.setMetadata('rating', rating)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+				
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+		
+        labels = soup.findAll('span',{'class':'label'})
+		
+        value = labels[0].previousSibling
+        svalue = ""
+        while value != None:
+            val = value
+            value = value.previousSibling
+        while not defaultGetattr(val,'class') == 'label':
+            svalue += str(val)
+            val = val.nextSibling
+        self.setDescription(url,svalue)
+		
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value.split(' -')[0])
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Complete' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value.split(' -')[0]), self.dateformat))
+            
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -0,0 +1,239 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+
+def getClass():
+    return CheckmatedComAdapter
+
+
+class CheckmatedComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        self._setURL('http://' + self.getSiteDomain() + '/story.php?story='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','chm')
+
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","Harry Potter")
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%b %d, %Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.checkmated.com'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/story.php?story=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/story.php?story=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites. This story is in The Bedchamber
+    def needToLoginCheck(self, data):
+        if 'This story is in The Bedchamber' in data \
+                or 'That username is not in our database' in data \
+                or "That password is not correct, please try again" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['name'] = self.username
+            params['pass'] = self.password
+        else:
+            params['name'] = self.getConfig("username")
+            params['pass'] = self.getConfig("password")
+        params['login'] = 'yes'
+        params['submit'] = 'login'	
+	
+        loginUrl = 'http://' + self.getSiteDomain()+'/login.php'
+        d = self._fetchUrl(loginUrl,params)
+        e = self._fetchUrl(url)
+
+        if "Welcome back," not in d : #Member Account
+            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['name']))
+            raise exceptions.FailedToLogin(url,params['name'])
+            return False
+        elif "This story is in The Bedchamber" in e:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Your account does not have sufficient priviliges to read this story.")
+            return False
+        else:
+            return True
+
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+
+        # The actual text that is used to announce you need to be an
+        # adult varies from site to site.  Again, print data before
+        # the title search to troubleshoot.
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.findAll('span', {'class' : 'storytitle'})
+        self.story.setMetadata('title',a[0].string)
+        
+        # Find authorid and URL from... author url.
+        a = a[1].find('a', href=re.compile(r"authors.php\?name\=\w+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+			
+        a = soup.find('select', {'name' : 'chapter'})
+        if a == None:
+            self.chapterUrls.append((self.story.getMetadata('title'),url))
+        else:
+            for chapter in a.findAll('option'):
+                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/story.php?story='+self.story.getMetadata('storyId')+'&chapter='+chapter['value']))
+			
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+				
+        # website does not keep track of word count, and there is no convenient way to calculate it
+
+        summary = soup.find('fieldset')
+        summary.find('legend').extract()
+        summary.name='div'
+        self.setDescription(url,summary)
+
+		
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        table = soup.findAll('div', {'class' : 'text'})[1]
+        for labels in table.findAll('tr'):
+            value = labels.findAll('td')[1]
+            label = labels.findAll('td')[0]
+
+			
+            if 'Rating' in stripHTML(label):
+                self.story.setMetadata('rating', stripHTML(value))
+
+            if 'Ship' in stripHTML(label):
+                for char in value.string.split('/'):
+                    if char != 'none':
+                        self.story.addToList('characters',char)		
+
+            if 'Status' in stripHTML(label):
+                if value.find('img', {'src' : 'img/incomplete.gif'}) == None:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in stripHTML(label):
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in stripHTML(label):
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+			
+        a = self._fetchUrl(self.story.getMetadata('authorUrl')+'&cat=stories')	
+        for story in bs.BeautifulSoup(a).findAll('table', {'class' : 'storyinfo'}):
+            a = story.find('a', href=re.compile(r"review.php\?s\="+self.story.getMetadata('storyId')+'&act=view'))
+            if a != None:
+                for labels in story.findAll('tr'):
+                    value = labels.findAll('td')[1]
+                    label = labels.findAll('td')[0]
+                    if 'genre' in stripHTML(label):
+                        for genre in value.findAll('img'):
+                            self.story.addToList('genre',genre['title'])
+        
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'resizeableText'})
+        div.find('div', {'class' : 'storyTools'}).extract()
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -0,0 +1,240 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return DestinysGatewayComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class DestinysGatewayComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','dgrfa')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%b %d %Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.destinysgateway.com'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&warning=4"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&amp;ageconsent=ok)?&amp;warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid &amp; error in url.
+                addurl = addurl.replace("&amp;","&")
+                url = self.url+'&index=1'+addurl
+                logging.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -0,0 +1,236 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return DokugaComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class DokugaComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[3])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        self.story.setMetadata('section',self.parsedUrl.path.split('/',)[1])
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/'+self.parsedUrl.path.split('/',)[1]+'/story/'+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','dkg')
+
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","InuYasha")
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        if 'fanfiction' in self.story.getMetadata('section'):
+            self.dateformat = "%d %b %Y"
+        else:
+            self.dateformat = "%m-%d-%y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.dokuga.com'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/fanfiction/story/1234/1 http://"+self.getSiteDomain()+"/spark/story/1234/1"
+
+    def getSiteURLPattern(self):
+        return r"http://"+self.getSiteDomain()+"/(fanfiction|spark)?/story/\d+/?\d+?$"
+
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title and author
+        a = soup.find('div', {'align' : 'center'}).find('h3')
+        
+        # Find authorid and URL from... author url.
+        aut = a.find('a')
+        self.story.setMetadata('authorId',aut['href'].split('=')[1])
+        alink='http://'+self.host+aut['href']
+        self.story.setMetadata('authorUrl','http://'+self.host+aut['href'])
+        self.story.setMetadata('author',aut.string)
+        aut.extract()
+        
+        a = a.string[:(len(a.string)-4)]
+        self.story.setMetadata('title',a)
+
+        # Find the chapters:
+        chapters = soup.find('select').findAll('option')
+        if len(chapters)==1:
+            self.chapterUrls.append((self.story.getMetadata('title'),'http://'+self.host+'/'+self.story.getMetadata('section')+'/story/'+self.story.getMetadata('storyId')+'/1'))
+        else:
+            for chapter in chapters:
+                # just in case there's tags, like <i> in chapter titles. /fanfiction/story/7406/1
+                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+self.story.getMetadata('section')+'/story/'+self.story.getMetadata('storyId')+'/'+chapter['value']))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+        
+        asoup = bs.BeautifulSoup(self._fetchUrl(alink))
+        
+        if 'fanfiction' in self.story.getMetadata('section'):
+            asoup=asoup.find('div', {'id' : 'cb_tabid_52'}).find('div')
+        
+            #grab the rest of the metadata from the author's page
+            for div in asoup.findAll('div'):
+                nav=div.find('a', href=re.compile(r'/fanfiction/story/'+self.story.getMetadata('storyId')+"/1$"))
+                if nav != None:
+                    break
+            div=div.nextSibling
+            self.setDescription(url,div)
+        
+            div=div.nextSibling
+            
+            a=div.text.split('Rating: ')
+            if len(a) == 2: self.story.setMetadata('rating', a[1].split('&')[0])
+        
+            a=div.text.split('Status: ')
+            if len(a)==2:
+                iscomp=a[1].split('&')[0]
+                if 'Complete' in iscomp:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+            
+            a=div.text.split('Category: ')
+            if len(a) == 2: self.story.addToList('category', a[1].split('&')[0])
+            self.story.addToList('category', 'Fanfiction')
+        
+            a=div.text.split('Created: ')
+            if len(a) == 2: self.story.setMetadata('datePublished', makeDate(stripHTML(a[1].split('&')[0]), self.dateformat))
+        
+            a=div.text.split('Updated: ')
+            if len(a) == 2: self.story.setMetadata('dateUpdated', makeDate(stripHTML(a[1]), self.dateformat))
+            
+            div=div.nextSibling.nextSibling
+            a=div.text.split('Words: ')
+            if len(a) == 2: self.story.setMetadata('numWords', a[1].split('&')[0])
+        
+            a=div.text.split('Genre: ')
+            if len(a) == 2:
+                for genre in a[1].split('&')[0].split(', '):
+                    self.story.addToList('genre',genre)
+        
+        else:
+            asoup=asoup.find('div', {'id' : 'maincol'}).find('div', {'class' : 'padding'})
+            for div in asoup.findAll('div'):
+                nav=div.find('a', href=re.compile(r'/spark/story/'+self.story.getMetadata('storyId')+"/1$"))
+                if nav != None:
+                    break
+                    
+            div=div.nextSibling.nextSibling
+            self.setDescription(url,div)
+            self.story.addToList('category', 'Spark')
+            
+            div=div.nextSibling.nextSibling
+            a=div.text.split('Rating: ')
+            if len(a) == 2: self.story.setMetadata('rating', a[1].split(' - ')[0])
+            
+            a=div.text.split('Status: ')
+            if len(a)==2:
+                iscomp=a[1].split(' - ')[0]
+                if 'Complete' in iscomp:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+            
+            a=div.text.split('Genre: ')
+            if len(a)==2:
+                for genre in a[1].split(' - ')[0].split('/'):
+                    self.story.addToList('genre',genre)
+                
+            div=div.nextSibling.nextSibling
+            
+            a=div.text.split('Updated: ')
+            if len(a)==2:
+                date=a[1].split('      -')[0]
+                self.story.setMetadata('dateUpdated', makeDate(date, self.dateformat))
+        
+                # does not have published date anywhere
+                self.story.setMetadata('datePublished', makeDate(date, self.dateformat))
+            
+            a=div.text.split('Words ')
+            if len(a)==2: self.story.setMetadata('numWords', a[1])
+            
+        
+
+
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'chtext'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -0,0 +1,271 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return DramioneOrgAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class DramioneOrgAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','drmn')
+
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","Harry Potter")
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%dth %B %Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'dramione.org'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Registered Users Only' in data \
+                or 'There is no such account on our website' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['cookiecheck'] = '1'
+        params['submit'] = 'Submit'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
+        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "Member Account" not in d : #Member Account
+            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&warning=5"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+
+        # The actual text that is used to announce you need to be an
+        # adult varies from site to site.  Again, print data before
+        # the title search to troubleshoot.
+        if "Stories that are suitable for ages 16 and older" in data:
+            raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+		
+        genres=soup.findAll('a', {'class' : "tag-1"})
+        for genre in genres:
+            self.story.addToList('genre',genre.string)
+
+        warnings=soup.findAll('a', {'class' : "tag-2"})
+        for warning in warnings:
+            self.story.addToList('warnings',warning.string)
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                value=value.replace('st','th').replace('nd','th').replace('rd','th')
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                value=value.replace('st','th').replace('nd','th').replace('rd','th')
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -0,0 +1,257 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return ErosnSapphoSycophantHexComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class ErosnSapphoSycophantHexComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+		
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','essph')
+
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","Harry Potter")
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%d/%m/%y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'erosnsappho.sycophanthex.com'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=18"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&amp;ageconsent=ok)?&amp;warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid &amp; error in url.
+                addurl = addurl.replace("&amp;","&")
+                url = self.url+'&index=1'+addurl
+                logging.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        pt = soup.find('div', {'id' : 'pagetitle'})
+        a = pt.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.text)
+        
+        # Find authorid and URL from... author url.
+        a = pt.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        rating=pt.text.split('(')[1].split(')')[0]
+        self.story.setMetadata('rating', rating)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+				
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+		
+        labels = soup.findAll('span',{'class':'label'})
+		
+        value = labels[0].previousSibling
+        svalue = ""
+        while value != None:
+            val = value
+            value = value.previousSibling
+        while not defaultGetattr(val,'class') == 'label':
+            svalue += str(val)
+            val = val.nextSibling
+        self.setDescription(url,svalue)
+		
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value.split(' -')[0])
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Complete' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value.split(' -')[0]), self.dateformat))
+            
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            i=1
+            for a in storyas:
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        break
+                    i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -19,12 +19,18 @@ import time
 import logging
 import re
 import urllib2
+from urllib import unquote_plus
 import time

-import fanficdownloader.BeautifulSoup as bs
-import fanficdownloader.exceptions as exceptions
+from .. import BeautifulSoup as bs
+from .. import exceptions as exceptions
+from ..htmlcleanup import stripHTML

-from base_adapter import BaseSiteAdapter, utf8FromSoup, makeDate
+from base_adapter import BaseSiteAdapter,  makeDate
+
+ffnetgenres=["Adventure", "Angst", "Crime", "Drama", "Family", "Fantasy", "Friendship", "General",
+             "Horror", "Humor", "Hurt-Comfort", "Mystery", "Parody", "Poetry", "Romance", "Sci-Fi",
+             "Spiritual", "Supernatural", "Suspense", "Tragedy", "Western"]

 class FanFictionNetSiteAdapter(BaseSiteAdapter):

@@ -70,10 +76,11 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):

        url = self.origurl
        logging.debug("URL: "+url)
-        
+
        # use BeautifulSoup HTML parser to make everything easier to find.
        try:
            data = self._fetchUrl(url)
+            #print("\n===================\n%s\n===================\n"%data)
            soup = bs.BeautifulSoup(data)
        except urllib2.HTTPError, e:
            if e.code == 404:
@@ -83,73 +90,125 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
            
        if "Unable to locate story with id of " in data:
            raise exceptions.StoryDoesNotExist(url)
-            
-        if "Chapter not found. Please check to see you are not using an outdated url." in data:
+
+        # some times "Chapter not found...", sometimes "Chapter text not found..."
+        if "not found. Please check to see you are not using an outdated url." in data:
            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  'Chapter not found. Please check to see you are not using an outdated url.'" % url)
-            
+
+        try:
+            # rather nasty way to check for a newer chapter.  ffnet has a
+            # tendency to send out update notices in email before all
+            # their servers are showing the update on the first chapter.
+            try:
+                chapcount = len(soup.find('select', { 'name' : 'chapter' } ).findAll('option'))
+            # get chapter part of url.
+            except:
+                chapcount = 1
+            chapter = url.split('/',)[5]
+            tryurl = "http://%s/s/%s/%d/"%(self.getSiteDomain(),
+                                           self.story.getMetadata('storyId'),
+                                           chapcount+1)
+            print('=Trying newer chapter: %s' % tryurl)
+            newdata = self._fetchUrl(tryurl)
+            if "not found. Please check to see you are not using an outdated url." \
+                    not in newdata:
+                print('=======Found newer chapter: %s' % tryurl)
+                soup = bs.BeautifulSoup(newdata)
+        except:
+            pass
+        
        # Find authorid and URL from... author url.
        a = soup.find('a', href=re.compile(r"^/u/\d+"))
        self.story.setMetadata('authorId',a['href'].split('/')[2])
        self.story.setMetadata('authorUrl','http://'+self.host+a['href'])
        self.story.setMetadata('author',a.string)

-            
-        # start by finding a script towards the bottom that has a
-        # bunch of useful stuff in it.
-            
-        # var storyid = 6577076;
-        # var chapter = 1;
-        # var chapters = 17;
-        # var words = 42787;
-        # var userid = 2645830;
-        # var title = 'The+Invitation';
-        # var title_t = 'The Invitation';
-        # var summary = 'Dudley Dursley would be the first to say he lived a very normal life. But what happens when he gets invited to his cousin Harry Potter\'s wedding? Will Dudley get the courage to apologize for the torture he caused all those years ago? Harry/Ginny story.';
-        # var categoryid = 224;
-        # var cat_title = 'Harry Potter';
-        # var datep = '12-21-10';
-        # var dateu = '04-06-11';
-        # var author = 'U n F a b u l o u s M e';
+        ## Pull some additional data from html.

-        for script in soup.findAll('script', src=None):
-            if not script:
-                continue
-            if not script.string:
-                continue
-            if 'var storyid' in script.string:
-                for line in script.string.split('\n'):
-                    m = re.match(r"^ +var ([^ ]+) = '?(.*?)'?;$",line)
-                    if m == None : continue
-                    var,value = m.groups()
-                    # remove javascript escaping from values.
-                    value = re.sub(r'\\(.)',r'\1',value)
-                    #print var,value
-                    if 'words' in var:
-                        self.story.setMetadata('numWords', value)
-                    if 'title_t' in var:
-                        self.story.setMetadata('title', value)
-                    if 'summary' in var:
-                        self.story.setMetadata('description', value)
-                    if 'datep' in var:
-                        self.story.setMetadata('datePublished',makeDate(value, '%m-%d-%y'))
-                    if 'dateu' in var:
-                        self.story.setMetadata('dateUpdated',makeDate(value, '%m-%d-%y'))
-                    if 'cat_title' in var:
-                        if "Crossover" in value:
-                            value = re.sub(r' Crossover$','',value)
-                            for c in value.split(' and '):
-                                self.story.addToList('category',c)
-                                # Screws up when the category itself
-                                # contains ' and '.  But that's rare
-                                # and the only alternative is to find
-                                # the 'Crossover' category URL and
-                                # parse that page to search for <a>
-                                # with href /crossovers/(name)/(num)/
-				# <a href="/crossovers/Harry_Potter/224/">Harry Potter</a>
-				# <a href="/crossovers/Naruto/1402/">Naruto</a>
-                        else:
-                            self.story.addToList('category',value)
-                break # for script in soup.findAll('script', src=None):
+        ## ffnet shows category two ways
+        ## 1) class(Book, TV, Game,etc) >> category(Harry Potter, Sailor Moon, etc)
+        ## 2) cat1_cat2_Crossover
+        ## For 1, use the second link.
+        ## For 2, fetch the crossover page and pull the two categories from there.
+
+        categories = soup.findAll('a',{'class':'xcontrast_txt'})
+        if len(categories) > 1:
+            self.story.addToList('category',stripHTML(categories[-1]))
+        elif 'Crossover' in categories[0]['href']:
+            caturl = "http://%s%s"%(self.getSiteDomain(),categories[0]['href'])
+            catsoup = bs.BeautifulSoup(self._fetchUrl(caturl))
+            for a in catsoup.findAll('a',href=re.compile(r"^/crossovers/")):
+                self.story.addToList('category',stripHTML(a))
+            
+        a = soup.find('a', href='http://www.fictionratings.com/')
+        rating = a.string
+        if 'Fiction' in rating: # if rating has 'Fiction ', strip that out for consistency with past.
+            rating = rating[8:]
+            
+        self.story.setMetadata('rating',rating)
+
+        # after Rating, the same bit of text containing id:123456 contains
+        # Complete--if completed.
+        gui_table1i = soup.find(id="gui_table1i")
+
+        self.story.setMetadata('title', stripHTML(gui_table1i.find('b'))) # title appears to be only(or at least first) bold tag in gui_table1i
+
+        summarydiv = gui_table1i.find('div',{'style':'margin-top:2px'})
+        if summarydiv:
+            self.setDescription(url,stripHTML(summarydiv))
+            
+        
+        metatext = stripHTML(gui_table1i.find('div', {'style':'color:gray;'})).replace('Hurt/Comfort','Hurt-Comfort')
+        metalist = metatext.split(" - ")
+        #print("metatext:(%s)"%metalist)
+
+        # Rated: Fiction K - English - Words: 158,078 - Published: 02-04-11
+
+        # rating is obtained above more robustly.
+        if metalist[0].startswith('Rated:'):
+            metalist=metalist[1:]
+
+        # next is assumed to be language.
+        self.story.setMetadata('language',metalist[0])
+        metalist=metalist[1:]
+
+        # next might be genre.
+        genrelist = metalist[0].split('/') # Hurt/Comfort already changed above.
+        goodgenres=True
+        for g in genrelist:
+            if g not in ffnetgenres:
+                goodgenres=False
+        if goodgenres:
+            self.story.extendList('genre',genrelist)
+            metalist=metalist[1:]
+
+        while len(metalist) > 0:
+            if  metalist[0].startswith('Reviews') or metalist[0].startswith('Chapters') or metalist[0].startswith('Status'):
+                pass
+            elif  metalist[0].startswith('Updated'):
+                self.story.setMetadata('dateUpdated',makeDate(metalist[0].split(':')[1].strip(), '%m-%d-%y'))
+            elif  metalist[0].startswith('Published'):
+                self.story.setMetadata('datePublished',makeDate(metalist[0].split(':')[1].strip(), '%m-%d-%y'))
+            elif  metalist[0].startswith('Words'):
+                self.story.setMetadata('numWords',metalist[0].split(':')[1].strip())
+                pass
+            else:
+                self.story.extendList('characters',metalist[0].split('&'))
+            metalist=metalist[1:]
+
+        # next might be characters, otherwise Reviews, Updated, Published, Words
+        # if not ( metalist[0].startswith('Reviews') or metalist[0].startswith('Updated') or metalist[0].startswith('Published') or metalist[0].startswith('Words') or metalist[0].startswith('Chapters') ):
+        #     self.story.extendList('characters',metalist[0].split('&'))
+        
+        if 'Status: Complete' in metatext:
+            self.story.setMetadata('status', 'Completed')
+        else:
+            self.story.setMetadata('status', 'In-Progress')
+
+        if self.getConfig('include_images'):
+            img = soup.find('img',{'class':'cimage'})
+            if img:
+                self.story.addImgUrl(self,url,img['src'],self._fetchUrlRaw,cover=True)
            
        # Find the chapter selector 
        select = soup.find('select', { 'name' : 'chapter' } )
@@ -170,35 +229,8 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):

        self.story.setMetadata('numChapters',len(self.chapterUrls))

-        ## Pull some additional data from html.  Find Rating and look around it.
-
-        a = soup.find('a', href='http://www.fictionratings.com/')
-        self.story.setMetadata('rating',a.string)
-
-        # after Rating, the same bit of text containing id:123456 contains
-        # Complete--if completed.
-        if 'Complete' in a.findNext(text=re.compile(r'id:'+self.story.getMetadata('storyId'))):
-            self.story.setMetadata('status', 'Completed')
-        else:
-            self.story.setMetadata('status', 'In-Progress')
-
-        # Parse genre(s) from <meta name="description" content="..."
-        # <meta name="description" content="Chapter 1 of a Harry Potter  - Family/Friendship fanfiction. Dudley Dursley would be the first to say he lived a very normal life. But what happens when he gets invited to his cousin Harry Potter's wedding? Will Dudley get the courage to apologize for the torture he caused all those years ago? Harry/Ginny story..">
-        # <meta name="description" content="A Gundam Wing/AC and Gundam Seed  - Romance/Sci-Fi crossover fanfiction  with characters:  & Kira Y.. Story summary: One-Shoot dividido en dos partes. Kira va en camino a rescatar a Lacus, pero él no es el unico. Dos personajes de diferentes universos Gundams. SEED vs ZERO.">
-        # <meta name="description" content="Chapter 1 of a Alvin and the chipmunks and Alpha and Omega  crossover fanfiction  with characters: Alvin S. & Humphrey. You'll just have to read to find out... No Flames Plesae... and tell me what you want to see by PM'ing me....">
-        # genre is after first -, but before first 'fanfiction'.
-        m = re.match(r"^(?:Chapter \d+ of a|A) (?:.*?)  (?:- (?P<genres>.*?)) (?:crossover )?fanfiction",
-                     soup.find('meta',{'name':'description'})['content'])
-        if m != None:
-            genres=m.group('genres')
-            # Hurt/Comfort is one genre.
-            genres=re.sub('Hurt/Comfort','Hurt-Comfort',genres)
-            for g in genres.split('/'):
-                self.story.addToList('genre',g)
-        
        return

-
    def getChapterText(self, url):
        logging.debug('Getting chapter text from: %s' % url)
        time.sleep(0.5) ## ffnet(and, I assume, fpcom) tends to fail
@@ -212,16 +244,14 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
        sharediv = soup.find('div', {'class' : 'a2a_kit a2a_default_style'})
        if sharediv:
            sharediv.extract()
-        else:
-            logging.debug('share button div not found')
        
-        div = soup.find('div', {'id' : 'storytext'})
+        div = soup.find('div', {'id' : 'storytextp'})
        
        if None == div:
-            logging.debug('div id=storytext not found.  data:%s'%data)
+            logging.debug('div id=storytextp not found.  data:%s'%data)
            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)

-        return utf8FromSoup(div)
+        return self.utf8FromSoup(url,div)

 def getClass():
    return FanFictionNetSiteAdapter
@@ -0,0 +1,154 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return FanFiktionDeAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class FanFiktionDeAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["utf8",
+                       "Windows-1252"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/s/'+self.story.getMetadata('storyId') + '/1')
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','ffde')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%d.%m.%Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.fanfiktion.de'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/s/46ccbef30000616306614050"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/s/")+r"\w+(/\d+)?$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        url = self.url
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'/s/'+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        head = soup.find('div', {'style' : 'width:85%;float:left;'})
+        a = head.find('a')
+        self.story.setMetadata('authorId',a['href'].split('/')[2])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.find('select').findAll('option'):
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/s/'+self.story.getMetadata('storyId')+'/'+chapter['value']))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+        self.story.setMetadata('language','German')
+
+        #find metadata on the story page
+        self.story.setMetadata('datePublished', makeDate(head.text.split('erstellt: ')[1].split('\n')[0], self.dateformat))
+        
+        self.story.setMetadata('dateUpdated', makeDate(head.text.split('letztes Update: ')[1].split('\n')[0], self.dateformat))
+        
+        for genre in head.text.split('&nbsp;&nbsp;&nbsp;')[3].split('/')[0].split(', '):
+            self.story.addToList('genre',genre)
+            
+        if 'fertiggestellt' in head.text:
+            self.story.setMetadata('status', 'Fertiggestellt')
+        else:
+            self.story.setMetadata('status', 'In Arbeit')
+
+        #find metadata on the author's page
+        asoup = bs.BeautifulSoup(self._fetchUrl("http://"+self.getSiteDomain()+"?a=q&a1=v&t=nickdetailsstories&lbi=stories&ar=0&nick="+self.story.getMetadata('authorId')))
+        tr=asoup.findAll('tr')
+        for i in range(1,len(tr)):
+            a = tr[i].find('a')
+            if a['href'] == '/s/'+self.story.getMetadata('storyId'):
+                break
+        self.setDescription(url,a['onmouseover'].split("', '")[1])
+        
+        td = tr[i].findAll('td')
+        self.story.addToList('category',td[1].string)
+        self.story.setMetadata('rating', td[4].string)
+        self.story.setMetadata('numWords', td[5].string)
+ 
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'storytext'})
+        for a in div.findAll('script'):
+            a.extract()
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -0,0 +1,222 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import datetime
+import logging
+import re
+import urllib2
+from .. import translit
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+
+def getClass():
+    return FicBookNetAdapter
+
+
+class FicBookNetAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["utf8",
+                       "Windows-1252"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/readfic/'+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','fbn')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%d %m %Y" 
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.ficbook.net'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/readfic/12345"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/readfic/")+r"\d+"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+        url=self.url
+        logging.debug("URL: "+url)
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+				
+
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+
+        # Now go hunting for all the meta data and the chapter list.
+		
+        table = soup.find('td',{'width':'50%'})
+        
+        ## Title
+        a = soup.find('h1')
+        self.story.setMetadata('title',stripHTML(a))
+        logging.debug("Title: (%s)"%self.story.getMetadata('title'))
+        
+        # Find authorid and URL from... author url.
+        a = table.find('a')
+        self.story.setMetadata('authorId',a.text) # Author's name is unique
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.text)
+        logging.debug("Author: (%s)"%self.story.getMetadata('author'))
+
+        # Find the chapters:
+        chapters = soup.find('div', {'class' : 'part_list'})
+        if chapters != None:
+            chapters=chapters.findAll('a', href=re.compile(r'/readfic/'+self.story.getMetadata('storyId')+"/\d+#part_content$"))
+            self.story.setMetadata('numChapters',len(chapters))
+            for x in range(0,len(chapters)):
+                chapter=chapters[x]
+                churl='http://'+self.host+chapter['href']
+                self.chapterUrls.append((stripHTML(chapter),churl))
+                if x == 0:
+                    pubdate = translit.translit(stripHTML(bs.BeautifulSoup(self._fetchUrl(churl)).find('div', {'class' : 'part_added'}).find('span')))
+                if x == len(chapters)-1:
+                    update = translit.translit(stripHTML(bs.BeautifulSoup(self._fetchUrl(churl)).find('div', {'class' : 'part_added'}).find('span')))
+        else:
+            self.chapterUrls.append((self.story.getMetadata('title'),url))
+            self.story.setMetadata('numChapters',1)
+            pubdate=translit.translit(stripHTML(soup.find('div', {'class' : 'part_added'}).find('span')))
+            update=pubdate
+
+        logging.debug("numChapters: (%s)"%self.story.getMetadata('numChapters'))
+
+        if not ',' in pubdate:
+            pubdate=datetime.date.today().strftime(self.dateformat)
+        if not ',' in update:
+            update=datetime.date.today().strftime(self.dateformat)
+        pubdate=pubdate.split(',')[0]
+        update=update.split(',')[0]
+		
+        fullmon = {"yanvarya":"01", "января":"01",
+           "fievralya":"02", "февраля":"02",
+           "marta":"03", "марта":"03",
+           "aprielya":"04", "апреля":"04",
+           "maya":"05", "мая":"05",
+           "iyunya":"06", "июня":"06",
+           "iyulya":"07", "июля":"07",
+           "avghusta":"08", "августа":"08",
+           "sentyabrya":"09", "сентября":"09",
+           "oktyabrya":"10", "октября":"10",
+           "noyabrya":"11", "ноября":"11",
+           "diekabrya":"12", "декабря":"12" }
+        for (name,num) in fullmon.items():
+            if name in pubdate:
+                pubdate = pubdate.replace(name,num)
+            if name in update:
+                update = update.replace(name,num)
+
+        self.story.setMetadata('dateUpdated', makeDate(update, self.dateformat))
+        self.story.setMetadata('datePublished', makeDate(pubdate, self.dateformat))
+        self.story.setMetadata('language','Russian')
+		
+        pr=soup.find('a', href=re.compile(r'/printfic/\w+'))
+        pr='http://'+self.host+pr['href']
+        pr = bs.BeautifulSoup(self._fetchUrl(pr))
+        pr=pr.findAll('div', {'class' : 'part_text'})
+        i=0
+        for part in pr:
+            i=i+len(stripHTML(part).split(' '))
+        self.story.setMetadata('numWords', str(i))
+		
+        i=0
+        fandoms = table.findAll('a', href=re.compile(r'/fanfiction/\w+'))
+        for fandom in fandoms:
+            self.story.addToList('category',fandom.string)
+            i=i+1
+        if i > 1:
+            self.story.addToList('genre', 'Кроссовер')
+		
+        meta=table.findAll('a', href=re.compile(r'/ratings/'))
+        i=0
+        for m in meta:
+            if i == 0:
+                self.story.setMetadata('rating', m.find('b').text)
+                i=1
+            elif i == 1:
+                if not "," in m.nextSibling:
+                    i=2
+                self.story.addToList('genre', m.find('b').text)
+            elif i == 2:
+                self.story.addToList('warnings', m.find('b').text)
+		
+
+        if table.find('span', {'style' : 'color: green'}):
+            self.story.setMetadata('status', 'Completed')
+        else:
+            self.story.setMetadata('status', 'In Progress')
+		
+
+        tags = table.findAll('b')
+        for tag in tags:
+            label = translit.translit(tag.text)
+            if 'Piersonazhi:' in label or 'Персонажи:' in label:
+                chars=tag.nextSibling.string.split(', ')
+                for char in chars:
+                    self.story.addToList('characters',char)
+                break
+				
+        summary=soup.find('span', {'class' : 'urlize'})
+        self.setDescription(url,summary.text)
+        #self.story.setMetadata('description', summary.text)
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        chapter = soup.find('div', {'class' : 'public_beta'})
+        if chapter == None:
+            chapter = soup.find('div', {'class' : 'public_beta_disabled'})
+
+        if None == chapter:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,chapter)
@@ -21,11 +21,11 @@ import re
 import urllib
 import urllib2

-import fanficdownloader.BeautifulSoup as bs
-from fanficdownloader.htmlcleanup import stripHTML
-import fanficdownloader.exceptions as exceptions
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions

-from base_adapter import BaseSiteAdapter, utf8FromSoup, makeDate
+from base_adapter import BaseSiteAdapter,  makeDate

 class FictionAlleyOrgSiteAdapter(BaseSiteAdapter):

@@ -187,7 +187,8 @@ class FictionAlleyOrgSiteAdapter(BaseSiteAdapter):
            
        for small in storydd.findAll('small'):
            small.extract() ## removes the <small> tags, leaving only the summary.
-        self.story.setMetadata('description',stripHTML(storydd))
+        self.setDescription(url,storydd)
+        #self.story.setMetadata('description',stripHTML(storydd))
        
        return

@@ -203,6 +204,10 @@ class FictionAlleyOrgSiteAdapter(BaseSiteAdapter):
 	# our div with poor html inside the story text.
 	data = data.replace('<!-- headerend -->','<crazytagstringnobodywouldstumbleonaccidently id="storytext">').replace('<!-- footerstart -->','</crazytagstringnobodywouldstumbleonaccidently>')
        
+        # problems with some stories confusing Soup.  This is a nasty
+        # hack, but it works.
+        data = data[data.index("<crazytagstringnobodywouldstumbleonaccidently"):]
+
        soup = bs.BeautifulStoneSoup(data,
                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
        body = soup.findAll('body') ## some stories use a nested body and body
@@ -219,7 +224,7 @@ class FictionAlleyOrgSiteAdapter(BaseSiteAdapter):
        if not data or not text:
            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
    
-        return utf8FromSoup(text)
+        return self.utf8FromSoup(url,text)

 def getClass():
    return FictionAlleyOrgSiteAdapter
@@ -22,11 +22,11 @@ import urllib2
 import time
 import httplib, urllib

-import fanficdownloader.BeautifulSoup as bs
-import fanficdownloader.exceptions as exceptions
-from fanficdownloader.htmlcleanup import stripHTML
+from .. import BeautifulSoup as bs
+from .. import exceptions as exceptions
+from ..htmlcleanup import stripHTML

-from base_adapter import BaseSiteAdapter, utf8FromSoup, makeDate
+from base_adapter import BaseSiteAdapter,  makeDate

 class FicwadComSiteAdapter(BaseSiteAdapter):

@@ -124,7 +124,8 @@ class FicwadComSiteAdapter(BaseSiteAdapter):

        # description
        storydiv = soup.find("div",{"id":"story"})
-        self.story.setMetadata('description', storydiv.find("blockquote",{'class':'summary'}).p.string)
+        self.setDescription(url,storydiv.find("blockquote",{'class':'summary'}).p.string)
+        #self.story.setMetadata('description', storydiv.find("blockquote",{'class':'summary'}).p.string)

        # most of the meta data is here:
        metap = storydiv.find("p",{"class":"meta"})
@@ -152,6 +153,12 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
            for g in m.group(1).split(','):
                self.story.addToList('genre',g)
        
+        m = re.match(r".*?Characters: (.*?) -.*?",metastr)
+        if m:
+            for g in m.group(1).split(','):
+                if g:
+                    self.story.addToList('characters',g)
+        
        m = re.match(r".*?Published: ([0-9/]+?) -.*?",metastr)
        if m:
            self.story.setMetadata('datePublished',makeDate(m.group(1), "%Y/%m/%d"))
@@ -195,9 +202,6 @@ class FicwadComSiteAdapter(BaseSiteAdapter):

    def getChapterText(self, url):
        logging.debug('Getting chapter text from: %s' % url)
-        time.sleep(0.5) ## ffnet tends to fail more if hit too fast.
-                        ## This is in additional to what ever the
-                        ## slow_down_sleep_time setting is.
        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

@@ -206,7 +210,7 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
        if None == span:
            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)

-        return utf8FromSoup(span)
+        return self.utf8FromSoup(url,span)

 def getClass():
    return FicwadComSiteAdapter
@@ -20,12 +20,14 @@ import logging
 import re
 import urllib2
 import cookielib as cl
+from datetime import datetime
+import json

-import fanficdownloader.BeautifulSoup as bs
-from fanficdownloader.htmlcleanup import stripHTML
-import fanficdownloader.exceptions as exceptions
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions

-from base_adapter import BaseSiteAdapter, utf8FromSoup, makeDate
+from base_adapter import BaseSiteAdapter,  makeDate

 def getClass():
    return FimFictionNetSiteAdapter
@@ -35,7 +37,7 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
    def __init__(self, config, url):
        BaseSiteAdapter.__init__(self, config, url)
        self.story.setMetadata('siteabbrev','fimficnet')
-        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
+        self.story.setMetadata('storyId', self.parsedUrl.path.split('/',)[2])
        self._setURL("http://"+self.getSiteDomain()+"/story/"+self.story.getMetadata('storyId')+"/")
        self.is_adult = False
        
@@ -73,6 +75,10 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
            self.opener = urllib2.build_opener(cookieproc)
        
        try:
+            apiResponse = urllib2.urlopen("http://www.fimfiction.net/api/story.php?story=%s" % (self.story.getMetadata("storyId"))).read()
+            apiData = json.loads(apiResponse)
+            
+            # Unfortunately, we still need to load the story index page to parse the characters
            data = self._fetchUrl(self.url)
        except urllib2.HTTPError, e:
            if e.code == 404:
@@ -82,80 +88,81 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
        
        if "Warning: mysql_fetch_array(): supplied argument is not a valid MySQL result resource" in data:
            raise exceptions.StoryDoesNotExist(self.url)
+
+        if "/images/missing_story.png" in data:
+            raise exceptions.StoryDoesNotExist(self.url)
+        
+        if "Invalid story id" in apiData.values():
+            raise exceptions.StoryDoesNotExist(self.url)
        
        if "This story has been marked as having adult content." in data:
            raise exceptions.AdultCheckRequired(self.url)
        
-        soup = bs.BeautifulSoup(data).find("div", {"class":"content_box post_content_box"})
-        
-        title, author = [link.text for link in soup.find("h2").findAll("a")]
-        self.story.setMetadata("title", title)
-        self.story.setMetadata("author", author)
-        self.story.setMetadata("authorId", author) # The author's name will be unique
-        self.story.setMetadata("authorUrl", "http://%s/user/%s" % (self.getSiteDomain(),author))
-        
-        chapterDates = []
+        if self.password:
+            params = {}
+            params['password'] = self.password
+            data = self._postUrl(self.url,params)

-        for chapter in soup.findAll("a", {"class":"chapter_link"}):
-            chapterDates.append(chapter.span.extract().text.strip("()"))
-            self.chapterUrls.append((chapter.text.strip(), "http://"+self.getSiteDomain() + chapter['href']))
+        if "Enter the password the author set for this story to view it." in data:
+            if self.getConfig('fail_on_password'):
+                raise exceptions.FailedToDownload("%s requires story password and fail_on_password is true."%self.url)
+            else:
+                raise exceptions.FailedToLogin(self.url,"Story requires individual password",passwdonly=True)
+         
+        storyMetadata = apiData["story"]    
+            
+        self.story.setMetadata("title", storyMetadata["title"])
+        self.story.setMetadata("author", storyMetadata["author"]["name"])
+        self.story.setMetadata("authorId", storyMetadata["author"]["id"])
+        self.story.setMetadata("authorUrl", "http://%s/user/%s" % (self.getSiteDomain(), storyMetadata["author"]["name"]))
        
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
+        chapters = [{"chapterTitle": chapter["title"], "chapterURL": chapter["link"]} for chapter in storyMetadata["chapters"]]
+        for chapter in chapters:
+            self.chapterUrls.append((chapter["chapterTitle"], chapter["chapterURL"]))
+        self.story.setMetadata("numChapters", len(self.chapterUrls))
        
-        for character in [character_icon['title'] for character_icon in soup.findAll("a", {"class":"character_icon"})]:
-            self.story.addToList("characters", character)
-        for category in [category.text for category in soup.find("div", {"class":"categories"}).findAll("a")]:
-            self.story.addToList("category", category)
-        self.story.addToList("category", "My Little Pony")
-        
-        
-        # The very last list element in the list of chapters contains the status, rating and word count e.g.:
-        #
-        #    <li>
-        #       Incomplete | Rating:
-        #       <span style="color:#c78238;">Teen</span>
-        #       <div class="word_count"><b>5,203</b>words total</div>
-        #    </li>
-        #
-
-        status_bar = soup.findAll('li')[-1]
        # In the case of fimfiction.net, possible statuses are 'Completed', 'Incomplete', 'On Hiatus' and 'Cancelled'
        # For the sake of bringing it in line with the other adapters, 'Incomplete' and 'On Hiatus' become 'In-Progress'
        # and 'Complete' beomes 'Completed'. 'Cancelled' seems an important enough (not to mention more strictly true) 
        # status to leave unchanged.
-        status = status_bar.text.split("|")[0].strip().replace("Incomplete", "In-Progress").replace("On Hiatus", "In-Progress").replace("Complete", "Completed")
-        self.story.setMetadata('status', status)
-        self.story.setMetadata('rating', status_bar.span.text)
-        self.story.setMetadata('numWords', status_bar.div.b.text)
-        
-        description_soup = soup.find("div", {"class":"description"})
-        # Sometimes the description has an expanding element
-        # This removes the ellipsis and the expand button
-        try:
-            description_soup.find('span', {"id":re.compile(r"description_more_elipses_\d+")}).extract() # Web designer can't spell 'ellipsis'
-            description_soup.find('a', {"class":"more"}).extract()
-        except:
-            pass
-        self.story.setMetadata('description', description_soup.text)
-        
-        # Unfortunately, nowhere on the page is the year mentioned. Because we would much rather update the story needlessly
-        # than miss an update, we hardcode the year of creation and update to be 2011.
+        status = storyMetadata["status"].replace("Incomplete", "In-Progress").replace("On Hiatus", "In-Progress").replace("Complete", "Completed")
+        self.story.setMetadata("status", status)
+        self.story.setMetadata("rating", storyMetadata["content_rating_text"])
+            
+        for category in storyMetadata["categories"]:
+            if storyMetadata["categories"][category]:
+                self.story.addToList("genre", category) 

-        # Get the date of creation from the first chapter
-        datePublished_text = chapterDates[0]
-        day, month = datePublished_text.split()
-        day = re.sub(r"[^\d.]+", '', day)
-        datePublished = makeDate("2011"+month+day, "%Y%b%d")
-        self.story.setMetadata("datePublished", datePublished)
-        dateUpdated_soup = bs.BeautifulSoup(data).find("div", {"class":"calendar"})
-        dateUpdated_soup.find('span').extract()
-        dateUpdated = makeDate("2011"+dateUpdated_soup.text, "%Y%b%d")
-        self.story.setMetadata("dateUpdated", dateUpdated)
+        self.story.addToList("category", "My Little Pony")
        
+        self.story.setMetadata("numWords", str(storyMetadata["words"]))
+        
+        
+        # fimfic is the first site with an explicit cover image.
+        if self.getConfig('include_images') and "image" in storyMetadata.keys():
+            coverurl = storyMetadata["image"]
+            if coverurl.startswith('//static.fimfiction.net'): # fix for img urls missing 'http:'
+                coverurl = "http:"+coverurl
+            self.story.addImgUrl(self,self.url,coverurl,self._fetchUrlRaw,cover=True)
+            
+        self.setDescription(self.url, storyMetadata["description"])
+        
+        # Dates are in Unix time
+        # Take the publish date from the first chapter posted
+        rawDatePublished = storyMetadata["chapters"][0]["date_modified"]
+        self.story.setMetadata("datePublished", datetime.fromtimestamp(rawDatePublished))
+        rawDateUpdated = storyMetadata["date_modified"]
+        self.story.setMetadata("dateUpdated", datetime.fromtimestamp(rawDateUpdated))
+        
+        soup = bs.BeautifulSoup(data).find("div", {"class":"story"})
+        for character in [character_icon["title"] for character_icon in soup.findAll("a", {"class":"character_icon"})]:
+            self.story.addToList("characters", character)
+            
+            
    def getChapterText(self, url):
        logging.debug('Getting chapter text from: %s' % url)
        soup = bs.BeautifulSoup(self._fetchUrl(url),selfClosingTags=('br','hr')).find('div', {'id' : 'chapter_container'})
        if soup == None:
            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-        return utf8FromSoup(soup)
-        
+        return self.utf8FromSoup(url,soup)
+        
@@ -21,11 +21,11 @@ import re
 import urllib
 import urllib2

-import fanficdownloader.BeautifulSoup as bs
-from fanficdownloader.htmlcleanup import stripHTML
-import fanficdownloader.exceptions as exceptions
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions

-from base_adapter import BaseSiteAdapter, utf8FromSoup, makeDate
+from base_adapter import BaseSiteAdapter,  makeDate

 class HarryPotterFanFictionComSiteAdapter(BaseSiteAdapter):

@@ -125,7 +125,8 @@ class HarryPotterFanFictionComSiteAdapter(BaseSiteAdapter):

        ## Finding the metadata is a bit of a pain.  Desc is the only thing this color.
        desctable= soup.find('table',{'bgcolor':'#f0e8e8'})
-        self.story.setMetadata('description',stripHTML(desctable))
+        self.setDescription(url,desctable)
+        #self.story.setMetadata('description',stripHTML(desctable))

        ## Finding the metadata is a bit of a pain.  Most of the meta
        ## data is in a center.table without a bgcolor.
@@ -159,6 +160,11 @@ class HarryPotterFanFictionComSiteAdapter(BaseSiteAdapter):
                    for g in m.group(1).split(','):
                        self.story.addToList('genre',g)
        
+                m = re.match(r".*?Characters: (.+?) Genre.*?",metastr)
+                if m:
+                    for g in m.group(1).split(','):
+                        self.story.addToList('characters',g)
+        
                m = re.match(r".*?Warnings: (.+).*?",metastr)
                if m:
                    for w in m.group(1).split(','):
@@ -188,7 +194,7 @@ class HarryPotterFanFictionComSiteAdapter(BaseSiteAdapter):
        if None == div:
            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)

-        return utf8FromSoup(div)
+        return self.utf8FromSoup(url,div)

 def getClass():
    return HarryPotterFanFictionComSiteAdapter
@@ -0,0 +1,234 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+# This function is called by the downloader in all adapter_*.py files
+# in this dir to register the adapter class.  So it needs to be
+# updated to reflect the class below it.  That, plus getSiteDomain()
+# take care of 'Registering'.
+def getClass():
+    return HPFandomNetAdapterAdapter # XXX
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class HPFandomNetAdapterAdapter(BaseSiteAdapter): # XXX
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        # XXX Most sites don't have the /eff part.  Replace all to remove it usually.
+        self._setURL('http://' + self.getSiteDomain() + '/eff/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','hpfdm') # XXX
+
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","Harry Potter") # XXX
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%Y.%m.%d" # XXX
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.hpfandom.net' # XXX
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/eff/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/eff/viewstory.php?sid=")+r"\d+$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        url = self.url
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/eff/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        ## Going to get the rest from the author page.
+        authdata = self._fetchUrl(self.story.getMetadata('authorUrl'))
+        # fix a typo in the site HTML so I can find the Characters list.
+        authdata = authdata.replace('<td width=10%">','<td width="10%">')
+
+        # hpfandom.net only seems to indicate adult-only by javascript on the story/chapter links.
+        if "javascript:if (confirm('Slash/het fiction which incorporates sexual situations to a somewhat graphic degree and some violence. ')) location = 'viewstory.php?sid=%s'"%self.story.getMetadata('storyId') in authdata \
+                and not (self.is_adult or self.getConfig("is_adult")):
+            raise exceptions.AdultCheckRequired(self.url)
+        
+        authsoup = bs.BeautifulSoup(authdata)
+
+        reviewsa = authsoup.find('a', href="reviews.php?sid="+self.story.getMetadata('storyId')+"&a=")
+        # <table><tr><td><p><b><a ...>
+        metablock = reviewsa.findParent("table")
+        #print("metablock:%s"%metablock)
+        
+        ## Title
+        titlea = metablock.find('a', href=re.compile("viewstory.php"))
+        #print("titlea:%s"%titlea)
+        if titlea == None:
+            raise exceptions.FailedToDownload("Story URL (%s) not found on author's page, can't use chapter URLs"%url)
+        self.story.setMetadata('title',stripHTML(titlea))
+        
+        # Find the chapters: !!! hpfandom.net differs from every other
+        # eFiction site--the sid on viewstory for chapters is
+        # *different* for each chapter
+        for chapter in soup.findAll('a', {'href':re.compile(r"viewstory.php\?sid=\d+&i=\d+")}):
+            m = re.match(r'.*?(viewstory.php\?sid=\d+&i=\d+).*?',chapter['href'])
+            # just in case there's tags, like <i> in chapter titles.
+            #print("====chapter===%s"%m.group(1))
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/eff/'+m.group(1)))
+
+        if len(self.chapterUrls) == 0:
+            self.chapterUrls.append((stripHTML(self.story.getMetadata('title')),url))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        summary = metablock.find("td",{"class":"summary"})
+        summary.name='span'
+        self.setDescription(url,summary)
+
+        # words & completed in first row of metablock.
+        firstrow = stripHTML(metablock.find('tr'))
+        # A Mother's Love xx Going Grey 1 (G+) by Kiristeen | Reviews - 18 | Words: 27468 | Completed: Yes
+        m = re.match(r".*?\((?P<rating>[^)]+)\).*?Words: (?P<words>\d+).*?Completed: (?P<status>Yes|No)",firstrow)
+        if m != None:
+            if m.group('rating') != None:
+                self.story.setMetadata('rating', m.group('rating'))
+                
+            if m.group('words') != None:
+                self.story.setMetadata('numWords', m.group('words'))
+
+            if m.group('status') != None:
+                if 'Yes' in m.group('status'):
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+                
+
+        # <tr><td width="10%" valign="top">Chapters:</td><td width="40%" valign="top">4</td>
+        # <td width="10%" valign="top">Published:</td><td width="40%" valign="top">2010.09.29</td></tr>
+        # <tr><td width="10%" valign="top">Completed:</td><td width="40%" valign="top">Yes</td><td width="10%" valign="top">Updated:</td><td width="40%" valign="top">2010.10.03</td></tr>
+        labels = metablock.findAll('td',{'width':'10%'})
+        for td in labels:
+            label = td.string
+            value = td.nextSibling.string
+            #print("\nlabel:%s\nvalue:%s\n"%(label,value))
+
+            if 'Category' in label:
+                cats = td.parent.findAll('a',href=re.compile(r'categories.php'))
+                catstext = [cat.string for cat in cats]
+                for cat in catstext:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                for char in value.split(','):
+                    self.story.addToList('characters',char.strip())
+
+            if 'Genre' in label:
+                for genre in value.split(','):
+                    self.story.addToList('genre',genre.strip())
+
+            if 'Warnings' in label:
+                for warning in value.split(','):
+                    if warning.strip() != 'none':
+                        self.story.addToList('warnings',warning.strip())
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        data = self._fetchUrl(url)
+        # There's no good wrapper around the chapter text. :-/
+        # There are, however, tables with width=100% just above and below the real text.
+        data = re.sub(r'<table width="100%">.*?</table>','<div name="storybody">',
+                      data,count=1,flags=re.DOTALL)
+        
+        data = re.sub(r'<table width="100%">.*?</table>','</div>',
+                      data,count=1,flags=re.DOTALL)
+        
+        soup = bs.BeautifulStoneSoup(data,selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+
+        div = soup.find("div",{'name':'storybody'})
+        #print("\n\ndiv:%s\n\n"%div)
+        
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+        return self.utf8FromSoup(url,div)
@@ -0,0 +1,285 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return IkEternalNetAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class IkEternalNetAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','ike')
+
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","InuYasha")
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%B %d, %Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.ik-eternal.net'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Registered Users Only' in data \
+                or 'There is no such account on our website' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['cookiecheck'] = '1'
+        params['submit'] = 'Submit'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
+        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "Member Account" not in d : #Member Account
+            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&warning=1"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+
+        # The actual text that is used to announce you need to be an
+        # adult varies from site to site.  Again, print data before
+        # the title search to troubleshoot.
+            
+        # Since the warning text can change by warning level, let's
+        # look for the warning pass url.  ksarchive uses
+        # &amp;warning= -- actually, so do other sites.  Must be an
+        # eFiction book.
+            
+        # viewstory.php?sid=1882&amp;warning=4
+        # viewstory.php?sid=1654&amp;ageconsent=ok&amp;warning=5
+        #print data
+        #m = re.search(r"'viewstory.php\?sid=1882(&amp;warning=4)'",data)
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&amp;ageconsent=ok)?&amp;warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid &amp; error in url.
+                addurl = addurl.replace("&amp;","&")
+                url = self.url+'&index=1'+addurl
+                logging.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data,selfClosingTags=('p')) #poor formatting of the paragraphs in the title page
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        asoup = soup.find('div', {'class': 'listbox'})
+        for a in asoup.findAll('p'):
+            a.name='br'
+        labels = asoup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -0,0 +1,308 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+# Search for XXX comments--that's where things are most likely to need changing.
+
+# This function is called by the downloader in all adapter_*.py files
+# in this dir to register the adapter class.  So it needs to be
+# updated to reflect the class below it.  That, plus getSiteDomain()
+# take care of 'Registering'.
+def getClass():
+    return KSArchiveComAdapter # XXX
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class KSArchiveComAdapter(BaseSiteAdapter): # XXX
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','ksa') # XXX
+
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","Star Trek") # XXX
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%b/%d/%Y" # XXX
+            
+    @classmethod
+    def getAcceptDomains(cls):
+        return ['www.ksarchive.com','ksarchive.com']
+
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'ksarchive.com' # XXX
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return "http://(www.)?"+re.escape(self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+
+            # Furthermore, there's a couple sites now with more than
+            # one warning level for different ratings.  And they're
+            # fussy about it.  midnightwhispers has three: 10, 3 & 5.
+            # we'll try 5 first.
+            addurl = "&ageconsent=ok&warning=2" # XXX
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        # The actual text that is used to announce you need to be an
+        # adult varies from site to site.  Again, print data before
+        # the title search to troubleshoot.
+            
+        # Since the warning text can change by warning level, let's
+        # look for the warning pass url.  ksarchive uses
+        # &amp;warning= -- actually, so do other sites.  Must be an
+        # eFiction book.
+            
+        # viewstory.php?sid=1882&amp;warning=4
+        # viewstory.php?sid=1654&amp;ageconsent=ok&amp;warning=5
+        #print data
+        #m = re.search(r"'viewstory.php\?sid=1882(&amp;warning=4)'",data)
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&amp;ageconsent=ok)?&amp;warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid &amp; error in url.
+                addurl = addurl.replace("&amp;","&")
+                url = self.url+'&index=1'+addurl
+                logging.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',stripHTML(a)) # title's inside a <b> tag.
+
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                catstext = [cat.string for cat in cats]
+                for cat in catstext:
+                    if cat.string.strip() in ('Poetry','Essays'):
+                        self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                self.story.addToList('characters','Kirk')
+                self.story.addToList('characters','Spock')
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                charstext = [char.string for char in chars]
+                for char in charstext:
+                    self.story.addToList('characters',char.string)
+
+            ## Not all sites use Genre, but there's no harm to
+            ## leaving it in.  Check to make sure the type_id number
+            ## is correct, though--it's site specific.
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) # XXX
+                genrestext = [genre.string for genre in genres]
+                self.genre = ', '.join(genrestext)
+                for genre in genrestext:
+                    self.story.addToList('genre',genre.string)
+
+            ## In addition to Genre (which is very site specific) KSA
+            ## has 'Story Type', which is much more what most sites
+            ## call genre.
+            if 'Story Type' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=5')) # XXX
+                genrestext = [genre.string for genre in genres]
+                self.genre = ', '.join(genrestext)
+                for genre in genrestext:
+                    self.story.addToList('genre',genre.string)
+
+            ## Not all sites use Warnings, but there's no harm to
+            ## leaving it in.  Check to make sure the type_id number
+            ## is correct, though--it's site specific.
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
+                warningstext = [warning.string for warning in warnings]
+                self.warning = ', '.join(warningstext)
+                for warning in warningstext:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            i=1
+            for a in storyas:
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        break
+                    i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        data = self._fetchUrl(url)
+        soup = bs.BeautifulStoneSoup(data,
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            if "A fatal MySQL error was encountered" in data:
+                raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Database error on the site reported!" % url)
+            else:
+                raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -0,0 +1,252 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+
+def getClass():
+    return LibraryOfMoriaComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class LibraryOfMoriaComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/a/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','lom')
+
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","Lord of the Rings")
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%B %d, %Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.libraryofmoria.com'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/a/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/a/viewstory.php?sid=")+r"\d+$"
+
+
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            addurl = "&ageconsent=ok&warning=3"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&amp;ageconsent=ok)?&amp;warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid &amp; error in url.
+                addurl = addurl.replace("&amp;","&")
+                url = self.url+'&index=1'+addurl
+                logging.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/a/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                catstext = [cat.string for cat in cats]
+                for cat in catstext:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                charstext = [char.string for char in chars]
+                for char in charstext:
+                    self.story.addToList('characters',char.string)
+
+            if 'Type' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warning' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=5'))
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/a/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            i=1
+            for a in storyas:
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        break
+                    i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -0,0 +1,239 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return LumosSycophantHexComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class LumosSycophantHexComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+		
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','lsph')
+
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","Harry Potter")
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%m/%d/%Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'lumos.sycophanthex.com'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=19"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        # The actual text that is used to announce you need to be an
+        # adult varies from site to site.  Again, print data before
+        # the title search to troubleshoot.
+        if "Age Consent Required" in data:
+            raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        pt = soup.find('div', {'id' : 'pagetitle'})
+        a = pt.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.text)
+        
+        # Find authorid and URL from... author url.
+        a = pt.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        rating=pt.text.split('(')[1].split(')')[0]
+        self.story.setMetadata('rating', rating)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+				
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+		
+        labels = soup.findAll('span',{'class':'label'})
+		
+        value = labels[0].previousSibling
+        svalue = ""
+        while value != None:
+            val = value
+            value = value.previousSibling
+        while not defaultGetattr(val,'class') == 'label':
+            svalue += str(val)
+            val = val.nextSibling
+        self.setDescription(url,svalue)
+		
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value.split(' -')[0])
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Complete' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value.split(' -')[0]), self.dateformat))
+            
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -21,11 +21,11 @@ import re
 import urllib
 import urllib2

-import fanficdownloader.BeautifulSoup as bs
-from fanficdownloader.htmlcleanup import stripHTML
-import fanficdownloader.exceptions as exceptions
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions

-from base_adapter import BaseSiteAdapter, utf8FromSoup, makeDate
+from base_adapter import BaseSiteAdapter,  makeDate

 class MediaMinerOrgSiteAdapter(BaseSiteAdapter):

@@ -174,7 +174,8 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
        # Summary: ....
        m = re.match(r".*?Summary: (.*)$",metastr) 
        if m:
-            self.story.setMetadata('description', m.group(1))
+            self.setDescription(url, m.group(1))
+            #self.story.setMetadata('description', m.group(1))

        # completed
        m = re.match(r".*?Status: Completed.*?",metastr)
@@ -210,7 +211,7 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
                del div['style']
                del div['align']
            anchor.name='div'
-            return utf8FromSoup(anchor)
+            return self.utf8FromSoup(url,anchor)
        
        else:
            logging.debug('Using kludgey text find for older mediaminer story.')
@@ -226,7 +227,7 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
                    soup.findAll('table',{'class':'tbbrdr'}):
                tag.extract() # remove tag from soup.
                
-            return utf8FromSoup(soup)
+            return self.utf8FromSoup(url,soup)
        

 def getClass():
@@ -0,0 +1,291 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+# Search for XXX comments--that's where things are most likely to need changing.
+
+# This function is called by the downloader in all adapter_*.py files
+# in this dir to register the adapter class.  So it needs to be
+# updated to reflect the class below it.  That, plus getSiteDomain()
+# take care of 'Registering'.
+def getClass():
+    return MidnightwhispersCaAdapter # XXX
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class MidnightwhispersCaAdapter(BaseSiteAdapter): # XXX
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','mw') # XXX
+
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","Queer as Folk") # XXX
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%B %d, %Y" # XXX
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.midnightwhispers.ca' # XXX
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+
+            # Furthermore, there's a couple sites now with more than
+            # one warning level for different ratings.  And they're
+            # fussy about it.  midnightwhispers has three: 10, 3 & 5.
+            # we'll try 5 first.
+            addurl = "&ageconsent=ok&warning=5" # XXX
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        # The actual text that is used to announce you need to be an
+        # adult varies from site to site.  Again, print data before
+        # the title search to troubleshoot.
+            
+        # Since the warning text can change by warning level, let's
+        # look for the warning pass url.  nfacommunity uses
+        # &amp;warning= -- actually, so do other sites.  Must be an
+        # eFiction book.
+            
+        # viewstory.php?sid=1882&amp;warning=4
+        # viewstory.php?sid=1654&amp;ageconsent=ok&amp;warning=5
+        #print data
+        #m = re.search(r"'viewstory.php\?sid=1882(&amp;warning=4)'",data)
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&amp;ageconsent=ok)?&amp;warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid &amp; error in url.
+                addurl = addurl.replace("&amp;","&")
+                url = self.url+'&index=1'+addurl
+                logging.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',stripHTML(a)) # title's inside a <b> tag.
+
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                catstext = [cat.string for cat in cats]
+                for cat in catstext:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                charstext = [char.string for char in chars]
+                for char in charstext:
+                    self.story.addToList('characters',char.string)
+
+            ## Not all sites use Genre, but there's no harm to
+            ## leaving it in.  Check to make sure the type_id number
+            ## is correct, though--it's site specific.
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
+                genrestext = [genre.string for genre in genres]
+                self.genre = ', '.join(genrestext)
+                for genre in genrestext:
+                    self.story.addToList('genre',genre.string)
+
+            ## Not all sites use Warnings, but there's no harm to
+            ## leaving it in.  Check to make sure the type_id number
+            ## is correct, though--it's site specific.
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
+                warningstext = [warning.string for warning in warnings]
+                self.warning = ', '.join(warningstext)
+                for warning in warningstext:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        break
+                    i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        data = self._fetchUrl(url)
+        soup = bs.BeautifulStoneSoup(data,
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            if "A fatal MySQL error was encountered" in data:
+                raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Database error on the site reported!" % url)
+            else:
+                raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -0,0 +1,333 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+# This function is called by the downloader in all adapter_*.py files
+# in this dir to register the adapter class.  So it needs to be
+# updated to reflect the class below it.  That, plus getSiteDomain()
+# take care of 'Registering'.
+def getClass():
+    return MuggleNetComAdapter # XXX
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class MuggleNetComAdapter(BaseSiteAdapter): # XXX
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','mgln') # XXX
+
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","Harry Potter") # XXX
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%m/%d/%y" # XXX
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'fanfiction.mugglenet.com' # XXX
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if "class='errortext'>Registered Users Only" in data \
+                or 'There is no such account on our website' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['cookiecheck'] = '1'
+        params['submit'] = 'Submit'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login&sid='+self.story.getMetadata('storyId')
+        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "Member Account" not in d : #Member Account
+            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            # http://fanfiction.mugglenet.com/viewstory.php?sid=91079&ageconsent=ok&warning=3
+            addurl = "&ageconsent=ok&warning=3" # XXX  &warning=5
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+            #print("\nurl:%s\ndata:\n%s\n"%(url,data))
+
+        # The actual text that is used to announce you need to be an
+        # adult varies from site to site.  Again, print data before
+        # the title search to troubleshoot.
+            
+        # Since the warning text can change by warning level, let's
+        # look for the warning pass url.  nfacommunity uses
+        # &amp;warning= -- actually, so do other sites.  Must be an
+        # eFiction book.
+            
+        # viewstory.php?sid=1882&amp;warning=4
+        # viewstory.php?sid=1654&amp;ageconsent=ok&amp;warning=5
+        #print data
+        #m = re.search(r"'viewstory.php\?sid=1882(&amp;warning=4)'",data)
+        m = re.search(r"'viewstory.php\?sid=%s((?:&amp;ageconsent=ok)?&amp;warning=\d+)'"%self.story.getMetadata('storyId'),data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid &amp; error in url.
+                addurl = addurl.replace("&amp;","&")
+                url = self.url+'&index=1'+addurl
+                logging.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+
+        # Not good enough-- content can contain a ('), which ends the content prematurely.
+        # metadesc = soup.find('meta',{'name':'description'})
+        # print("removeAllEntities(metadesc['content']):\n%s\n"%removeAllEntities(metadesc['content']))
+        start='<span class="label">Summary: </span>'
+        end='<span class="label">Rated:</span>'
+        summarydata = data[data.index(start)+len(start):data.index(end)]
+        #print("summarydata:\n%s\n"%summarydata)
+        self.setDescription(url,bs.BeautifulSoup(summarydata))
+        
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            # not good enough--poorly formated summary html will break it.
+            # if 'Summary' in label:
+            #     ## Everything until the next span class='label'
+            #     svalue = ""
+            #     while not defaultGetattr(value,'class') == 'label':
+            #         svalue += str(value)
+            #         value = value.nextSibling
+            #     self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                catstext = [cat.string for cat in cats]
+                for cat in catstext:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                charstext = [char.string for char in chars]
+                for char in charstext:
+                    self.story.addToList('characters',char.string)
+
+            ## Not all sites use Genre, but there's no harm to
+            ## leaving it in.  Check to make sure the type_id number
+            ## is correct, though--it's site specific.
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
+                genrestext = [genre.string for genre in genres]
+                self.genre = ', '.join(genrestext)
+                for genre in genrestext:
+                    self.story.addToList('genre',genre.string)
+
+            ## Not all sites use Warnings, but there's no harm to
+            ## leaving it in.  Check to make sure the type_id number
+            ## is correct, though--it's site specific.
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
+                warningstext = [warning.string for warning in warnings]
+                self.warning = ', '.join(warningstext)
+                for warning in warningstext:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            i=1
+            for a in storyas:
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        break
+                    i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -0,0 +1,200 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return NCISFictionComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class NCISFictionComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["iso-8859-1",
+                       "Windows-1252"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL("http://"+self.getSiteDomain()\
+                         +"/chapters.php?stid="+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','ncisfn')
+
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","NCIS")
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%d/%m/%Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.ncisfiction.com'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/story.php?stid=01234 http://"+self.getSiteDomain()+"/chapters.php?stid=1234"
+
+    def getSiteURLPattern(self):
+        return "http://"+self.getSiteDomain()+r'/(chapters|story)?.php\?stid=\d+'
+       
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulStoneSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title and author
+        a = soup.find('div', {'class' : 'main_title'})
+        
+        aut = a.find('a')
+        self.story.setMetadata('authorId',aut['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+aut['href'])
+        self.story.setMetadata('author',aut.string)
+        
+        aut.extract()
+        self.story.setMetadata('title',stripHTML(a)[:len(stripHTML(a))-2])
+
+        # Find the chapters:
+        i=0
+        chapters=soup.findAll('table', {'class' : 'story_table'})
+        for chapter in chapters:
+            ch=chapter.find('a')
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(ch),'http://'+self.host+'/'+ch['href']))
+            if i == 0:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(chapter.find('td')).split('Added: ')[1], self.dateformat))
+            if i == len(chapters)-1:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(chapter.find('td')).split('Added: ')[1], self.dateformat))
+            i=i+1
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        info = soup.find('table', {'class' : 'story_info'})
+        
+        # no convenient way to calculate word count as it is logged differently for stories with and without series
+        
+        labels = info.findAll('tr')
+        for tr in labels:
+            value = tr.find('td')
+            label = tr.find('th').string
+
+            if 'Summary' in label:
+                self.setDescription(url,value)
+
+            if 'Rating' in label:
+                self.story.setMetadata('rating', value.string)
+
+            if 'Category' in label:
+                cats = value.findAll('a')
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = value.findAll('a')
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Genre' in label:
+                genres = value.findAll('a')
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = value.findAll('a')
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Status' in label:
+                if 'not completed' in value.text:
+                    self.story.setMetadata('status', 'In-Progress')
+                else:
+                    self.story.setMetadata('status', 'Completed')
+                    
+        try:
+            # Find Series name from series URL.
+            a = soup.find('div',{'class' : 'sub_header'})
+            series_name = a.find('a').string
+            i = a.text.split('#')[1]
+            self.setSeries(series_name, i)
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'class' : 'story_text'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -0,0 +1,291 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+# Search for XXX comments--that's where things are most likely to need changing.
+
+# This function is called by the downloader in all adapter_*.py files
+# in this dir to register the adapter class.  So it needs to be
+# updated to reflect the class below it.  That, plus getSiteDomain()
+# take care of 'Registering'.
+def getClass():
+    return NfaCommunityComAdapter # XXX
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class NfaCommunityComAdapter(BaseSiteAdapter): # XXX
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','nfa') # XXX
+
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","NCIS") # XXX
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%m/%d/%Y" # XXX
+            
+    @classmethod
+    def getAcceptDomains(cls):
+        return ['www.nfacommunity.com','nfacommunity.com']
+
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'nfacommunity.com' # XXX
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return "http://(www.)?"+re.escape(self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+
+            # Furthermore, there's a couple sites now with more than
+            # one warning level for different ratings.  And they're
+            # fussy about it.  nfacommunity has two: 4 & 5.
+            # we'll try 5 first.
+            addurl = "&ageconsent=ok&warning=5" # XXX
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        # The actual text that is used to announce you need to be an
+        # adult varies from site to site.  Again, print data before
+        # the title search to troubleshoot.
+            
+        # Since the warning text can change by warning level, let's
+        # look for the warning pass url.  nfacommunity uses
+        # &amp;warning= -- actually, so do other sites.  Must be an
+        # eFiction book.
+            
+        # viewstory.php?sid=1882&amp;warning=4
+        # viewstory.php?sid=1654&amp;ageconsent=ok&amp;warning=5
+        #print data
+        #m = re.search(r"'viewstory.php\?sid=1882(&amp;warning=4)'",data)
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&amp;ageconsent=ok)?&amp;warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid &amp; error in url.
+                addurl = addurl.replace("&amp;","&")
+                url = self.url+'&index=1'+addurl
+                logging.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                catstext = [cat.string for cat in cats]
+                for cat in catstext:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                charstext = [char.string for char in chars]
+                for char in charstext:
+                    self.story.addToList('characters',char.string)
+
+            ## Not all sites use Genre, but there's no harm to
+            ## leaving it in.  Check to make sure the type_id number
+            ## is correct, though--it's site specific.
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
+                genrestext = [genre.string for genre in genres]
+                self.genre = ', '.join(genrestext)
+                for genre in genrestext:
+                    self.story.addToList('genre',genre.string)
+
+            ## Not all sites use Warnings, but there's no harm to
+            ## leaving it in.  Check to make sure the type_id number
+            ## is correct, though--it's site specific.
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
+                warningstext = [warning.string for warning in warnings]
+                self.warning = ', '.join(warningstext)
+                for warning in warningstext:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            i=1
+            for a in storyas:
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        break
+                    i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -0,0 +1,265 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return OcclumencySycophantHexComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class OcclumencySycophantHexComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+		
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','osph')
+
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","Harry Potter")
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%m/%d/%Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'occlumency.sycophanthex.com'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'This story contains adult content and/or themes.' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['rememberme'] = '1'
+        params['sid'] = ''
+        params['intent'] = ''
+        params['submit'] = 'Submit'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/user.php'
+        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "Logout" not in d : #Member Account
+            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
+		
+        try:
+            # in case link points somewhere other than the first chapter
+            a = soup.findAll('option')[1]['value']
+            self.story.setMetadata('storyId',a.split('=',)[1])
+            url = 'http://'+self.host+'/'+a
+            soup = bs.BeautifulSoup(self._fetchUrl(url))
+        except:
+            pass
+		
+        for info in asoup.findAll('table', {'class' : 'border'}):
+            a = info.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+            if a != None:
+                self.story.setMetadata('title',a.text)
+                break
+		
+
+        # Find the chapters:
+        chapters=soup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+&i=1$'))
+        if len(chapters) == 0:
+            self.chapterUrls.append((self.story.getMetadata('title'),url))
+        else:
+            for chapter in chapters:
+                # just in case there's tags, like <i> in chapter titles.
+                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d):
+            try:
+                return d.name
+            except:
+                return ""
+				
+        cats = info.findAll('a',href=re.compile('categories.php'))
+        for cat in cats:
+            self.story.addToList('category',cat.string)
+					
+			
+        a = info.find('a', href=re.compile(r'reviews.php\?sid='+self.story.getMetadata('storyId')))
+        val = a.nextSibling
+        svalue = ""
+        while not defaultGetattr(val) == 'br':
+            val = val.nextSibling
+        val = val.nextSibling
+        while not defaultGetattr(val) == 'table':
+            svalue += str(val)
+            val = val.nextSibling
+        self.setDescription(url,svalue)
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = info.findAll('b')
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = stripHTML(labelspan)
+
+            if 'Rating' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word Count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Genres' in label:
+                genres = value.string.split(', ')
+                for genre in genres:
+                    if genre != 'none':
+                        self.story.addToList('genre',genre)
+						
+            if 'Characters' in label:
+                chars = value.string.split(', ')
+                for char in chars:
+                    if char != 'none':
+                        self.story.addToList('characters',char)
+
+            if 'Warnings' in label:
+                warnings = value.string.split(', ')
+                for warning in warnings:
+                    if warning != ' none':
+                        self.story.addToList('warnings',warning)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+		
+        data = self._fetchUrl(url)
+        data = data.replace('<div align="left"', '<div align="left">')
+
+        soup = bs.BeautifulSoup(data, selfClosingTags=('br','hr'))
+        
+        story = soup.find('div', {"align" : "left"})
+
+        if None == story:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,story)
@@ -0,0 +1,271 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return OneDirectionFanfictionComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class OneDirectionFanfictionComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','odf')
+
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","One Direction")
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%m/%d/%y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'onedirectionfanfiction.com'
+        
+    @classmethod
+    def getAcceptDomains(cls):
+        return ['www.onedirectionfanfiction.com','onedirectionfanfiction.com']
+
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://")+"(www\.)?"+re.escape(self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+        
+        ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Registered Users Only' in data \
+                or 'There is no such account on our website' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['cookiecheck'] = '1'
+        params['submit'] = 'Submit'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
+        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "Member Account" not in d : #Member Account
+            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+    
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=4"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+            
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+        
+        # The actual text that is used to announce you need to be an
+        # adult varies from site to site.  Again, print data before
+        # the title search to troubleshoot.
+        if "Age Consent Required" in data:
+            raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=6'))
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -0,0 +1,242 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2, urllib, cookielib
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return PhoenixSongNetAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class PhoenixSongNetAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[3])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/fanfiction/story/' +self.story.getMetadata('storyId')+'/')
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','phs')
+
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","Harry Potter")
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%B %d %Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.phoenixsong.net'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/fanfiction/story/1234/"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/fanfiction/story/")+r"\d+/?$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Please login to continue.' in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+		
+        if self.password:
+            params['txtusername'] = self.username
+            params['txtpassword'] = self.password
+        else:
+            params['txtusername'] = self.getConfig("username")
+            params['txtpassword'] = self.getConfig("password")
+        #params['remember'] = '1'
+        params['login'] = 'Login'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/users/processlogin.php'
+        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['txtusername']))
+        d = self._fetchUrl(loginUrl, params)
+    
+        if 'Please login to continue.' in d : #Member Account
+            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['txtusername']))
+            raise exceptions.FailedToLogin(url,params['txtusername'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+			
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        b = soup.find('div', {'id' : 'nav25'})
+        a = b.find('a', href=re.compile(r'fanfiction/story/'+self.story.getMetadata('storyId')+"/$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.  /fanfiction/stories.php?psid=125
+        a = b.find('a', href=re.compile(r"/fanfiction/stories.php\?psid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        chapters = soup.find('select')
+        if chapters == None:
+            self.chapterUrls.append((self.story.getMetadata('title'),url))
+            for b in soup.findAll('b'):
+                if b.text == "Updated":
+                    date = b.nextSibling.string.split(': ')[1].split(',')
+                    self.story.setMetadata('datePublished', makeDate(date[0]+date[1], self.dateformat))
+                    self.story.setMetadata('dateUpdated', makeDate(date[0]+date[1], self.dateformat))
+        else:
+            i = 0
+            chapters = chapters.findAll('option')
+            for chapter in chapters:
+                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+chapter['value']))
+                if i == 0:
+                    self.story.setMetadata('storyId',chapter['value'].split('/')[3])
+                    head = bs.BeautifulSoup(self._fetchUrl('http://'+self.host+chapter['value'])).findAll('b')
+                    for b in head:
+                        if b.text == "Updated":
+                            date = b.nextSibling.string.split(': ')[1].split(',')
+                            self.story.setMetadata('datePublished', makeDate(date[0]+date[1], self.dateformat))
+                            
+                if  i == (len(chapters)-1):
+                    head = bs.BeautifulSoup(self._fetchUrl('http://'+self.host+chapter['value'])).findAll('b')
+                    for b in head:
+                        if b.text == "Updated":
+                            date = b.nextSibling.string.split(': ')[1].split(',')
+                            self.story.setMetadata('dateUpdated', makeDate(date[0]+date[1], self.dateformat))
+                i = i+1
+				
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))        	
+				
+        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
+        
+        info = asoup.find('a', href=re.compile(r'fanfiction/story/'+self.story.getMetadata('storyId')+"/$"))
+        while info != None:
+            info = info.findNext('div')
+            b = info.find('b')
+            val = b.nextSibling
+            
+            if 'Rating' in b.string:
+                self.story.setMetadata('rating', val.string.split(': ')[1])
+				
+            if 'Words' in b.string:
+                self.story.setMetadata('numWords', val.string.split(': ')[1])
+				
+            if 'Setting' in b.string:
+                self.story.addToList('category', val.string.split(': ')[1])
+				
+            if 'Status' in b.string:
+                if 'Completed' in val:
+                    val = 'Completed'
+                else:
+                    val = 'In-Progress'
+                self.story.setMetadata('status', val)
+            
+            if 'Summary' in b.string:
+                b.extract()
+                info.find('br').extract()
+                self.setDescription(url,info)
+                break
+	
+	
+	# grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+
+        chapter=bs.BeautifulSoup('<div class="story"></div>')
+        for p in soup.findAll('p'):
+            if "This is for problems with the formatting or the layout of the chapter." in stripHTML(p):
+                break
+            chapter.append(p)
+			
+        for a in chapter.findAll('div'):
+            a.extract()
+        for a in chapter.findAll('table'):
+            a.extract()
+        for a in chapter.findAll('script'):
+            a.extract()
+        for a in chapter.findAll('form'):
+            a.extract()
+        for a in chapter.findAll('textarea'):
+            a.extract()
+
+
+        if None == chapter:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,chapter)
@@ -0,0 +1,252 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return PonyFictionArchiveNetAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class PonyFictionArchiveNetAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        logging.info(self.parsedUrl.netloc)
+        # normalized story URL.
+        if "explicit" in self.parsedUrl.netloc:
+            self._setURL('http://explicit.' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+            self.dateformat = "%d/%b/%y"
+        else:
+            self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+            self.dateformat = "%d %b %Y"
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','pffa')
+
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","My Little Pony")
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'ponyfictionarchive.net'
+        
+    @classmethod
+    def getAcceptDomains(cls):
+        return ['www.ponyfictionarchive.net','ponyfictionarchive.net','explicit.ponyfictionarchive.net']
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://")+"(www\.|explicit\.)?"+re.escape(self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&warning=9"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&amp;ageconsent=ok)?&amp;warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid &amp; error in url.
+                addurl = addurl.replace("&amp;","&")
+                url = self.url+'&index=1'+addurl
+                logging.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        genres = soup.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+        for genre in genres:
+            self.story.addToList('genre',genre.string)
+           
+        warnings = soup.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=3'))
+        for warning in warnings:
+            self.story.addToList('warnings',warning.string)
+        
+        status = soup.find('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
+        self.story.setMetadata('status',status.string)
+        
+        section = soup.findAll('span', {'class' : 'General'})[1]            
+        
+        self.story.setMetadata('rating', section.previousSibling.previousSibling.string)
+
+        value = section.nextSibling
+        svalue = ""
+        while not defaultGetattr(value,'class') == 'label':
+            svalue += str(value)
+            value = value.nextSibling
+        self.setDescription(url,svalue)
+        
+        # <span class="label">Rated:</span> NC-17<br /> etc           
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            i=1
+            for a in storyas:
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        break
+                    i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -0,0 +1,278 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+import cookielib as cl
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+# Search for XXX comments--that's where things are most likely to need changing.
+
+# This function is called by the downloader in all adapter_*.py files
+# in this dir to register the adapter class.  So it needs to be
+# updated to reflect the class below it.  That, plus getSiteDomain()
+# take care of 'Registering'.
+def getClass():
+    return PortkeyOrgAdapter # XXX
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class PortkeyOrgAdapter(BaseSiteAdapter): # XXX
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/story/'+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','prtky') # XXX
+
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","Harry Potter") # XXX
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%d/%m/%y" # XXX
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'fanfiction.portkey.org' # XXX
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/story/1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/story/")+r"\d+(/\d+)?$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        url = self.url
+        logging.debug("URL: "+url)
+
+        # portkey screws around with using a different URL to set the
+        # cookie and it's a pain.  So... cheat!
+        if self.is_adult or self.getConfig("is_adult"):
+            cookieproc = urllib2.HTTPCookieProcessor()
+            cookie = cl.Cookie(version=0, name='verify17', value='1',
+                               port=None, port_specified=False,
+                               domain=self.getSiteDomain(), domain_specified=False, domain_initial_dot=False,
+                               path='/', path_specified=True,
+                               secure=False,
+                               expires=time.time()+10000,
+                               discard=False,
+                               comment=None,
+                               comment_url=None,
+                               rest={'HttpOnly': None},
+                               rfc2109=False)
+            cookieproc.cookiejar.set_cookie(cookie)
+            self.opener = urllib2.build_opener(cookieproc)        
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if "You must be over 18 years of age to view it" in data: # XXX 
+            raise exceptions.AdultCheckRequired(self.url)
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        #print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"/profile/\d+"))
+        #print("======a:%s"%a)
+        self.story.setMetadata('authorId',a['href'].split('/')[-1])
+        self.story.setMetadata('authorUrl','http://'+self.host+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        ## Going to get the rest from the author page.
+        authsoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
+        
+        ## Title
+        titlea = authsoup.find('a', href=re.compile(r'/story/'+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',stripHTML(titlea))
+        metablock = titlea.parent
+        
+        # Find the chapters:
+        for chapter in soup.find('select',{'name':'select5'}).findAll('option', {'value':re.compile(r'/story/'+self.story.getMetadata('storyId')+"/\d+$")}):
+            # just in case there's tags, like <i> in chapter titles.
+            chtitle = stripHTML(chapter)
+            if not chtitle:
+                chtitle = "(Untitled Chapter)"
+            self.chapterUrls.append((chtitle,'http://'+self.host+chapter['value']))
+            
+        if len(self.chapterUrls) == 0:
+            self.chapterUrls.append((stripHTML(self.story.getMetadata('title')),url))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+        # <SPAN class="dark-small-bold">Contents:</SPAN> <SPAN class="small-grey">NC17 </SPAN> 
+        # <SPAN class="dark-small-bold">Published: </SPAN><SPAN class="small-grey">12/11/07</SPAN>
+        # <SPAN class="dark-small-bold"><BR>
+        # Description:</SPAN> <SPAN class="small-black">A special book helps Harry tap into the power the Dark Lord knows not.  Of course it’s a book on sex magic and rituals… but Harry’s not complaining.  Spurned on by the ghost of a pervert founder, Harry leads his friends in the hunt for Voldemort’s Horcruxes.
+        # EROTIC COMEDY!  Loads of crude humor and sexual situations!
+        # </SPAN>
+        labels = metablock.findAll('span',{'class':'dark-small-bold'})
+        for labelspan in labels:
+            value = labelspan.findNext('span').string
+            label = stripHTML(labelspan)
+#            print("\nlabel:%s\nlabel:%s\nvalue:%s\n"%(labelspan,label,value))
+
+            if 'Description' in label:
+                self.setDescription(url,value)
+
+            if 'Contents' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Words' in label:
+                self.story.setMetadata('numWords', value)
+
+            # if 'Categories' in label:
+            #     cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+            #     catstext = [cat.string for cat in cats]
+            #     for cat in catstext:
+            #         self.story.addToList('category',cat.string)
+
+            # if 'Characters' in label:
+            #     chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+            #     charstext = [char.string for char in chars]
+            #     for char in charstext:
+            #         self.story.addToList('characters',char.string)
+
+            if 'Genre' in label:
+                # genre is typo'ed on the site--it falls between the
+                # dark-small-bold label and dark-small-bold content
+                # spans.
+                svalue = ""
+                value = labelspan.nextSibling
+                while not defaultGetattr(value,'class') == 'dark-small-bold':
+                    svalue += str(value)
+                    value = value.nextSibling
+                    
+                for genre in svalue.split("/"):
+                    genre = genre.strip()
+                    if genre != 'None':
+                        self.story.addToList('genre',genre)
+
+            ## Not all sites use Warnings, but there's no harm to
+            ## leaving it in.  Check to make sure the type_id number
+            ## is correct, though--it's site specific.
+            # if 'Warnings' in label:
+            #     warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
+            #     warningstext = [warning.string for warning in warnings]
+            #     self.warning = ', '.join(warningstext)
+            #     for warning in warningstext:
+            #         self.story.addToList('warnings',warning.string)
+
+            if 'Status' in label:
+                if 'Completed' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        # try:
+        #     # Find Series name from series URL.
+        #     a = metablock.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+        #     series_name = a.string
+        #     series_url = 'http://'+self.host+'/'+a['href']
+
+        #     # use BeautifulSoup HTML parser to make everything easier to find.
+        #     seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+        #     storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+        #     i=1
+        #     for a in storyas:
+        #         if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+        #             self.setSeries(series_name, i)
+        #             break
+        #         i+=1            
+        # except:
+        #     # I find it hard to care if the series parsing fails
+        #     pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+        data = self._fetchUrl(url)
+        soup = bs.BeautifulStoneSoup(data,
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+
+        #print("soup:%s"%soup)
+        tag = soup.find('td', {'class' : 'story'})
+        if tag == None and "<center><b>Chapter does not exist!</b></center>" in data:
+            print("Chapter is missing at: %s"%url)
+            return  self.utf8FromSoup(url,bs.BeautifulStoneSoup("<div><p><center><b>Chapter does not exist!</b></center></p><p>Chapter is missing at: <a href='%s'>%s</a></p></div>"%(url,url)))
+        tag.name='div' # force to be a div to avoid problems with nook.
+
+        centers = tag.findAll('center')
+        # first two and last two center tags are some script, 'report
+        # story', 'report story' and an ad.
+        centers[0].extract()
+        centers[1].extract()
+        centers[-1].extract()
+        centers[-2].extract()
+
+        if None == tag:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,tag)
@@ -21,11 +21,11 @@ import re
 import urllib
 import urllib2

-import fanficdownloader.BeautifulSoup as bs
-from fanficdownloader.htmlcleanup import stripHTML
-import fanficdownloader.exceptions as exceptions
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions

-from base_adapter import BaseSiteAdapter, utf8FromSoup, makeDate
+from base_adapter import BaseSiteAdapter,  makeDate

 class PotionsAndSnitchesNetSiteAdapter(BaseSiteAdapter):

@@ -61,14 +61,6 @@ class PotionsAndSnitchesNetSiteAdapter(BaseSiteAdapter):
    def getSiteURLPattern(self):
        return re.escape("http://")+r"(www\.)?"+re.escape("potionsandsnitches.net/fanfiction/viewstory.php?sid=")+r"\d+$"

-    def needToLoginCheck(self, data):
-        if 'Registered Users Only' in data \
-                or 'There is no such account on our website' in data \
-                or "That password doesn't match the one in our database" in data:
-          return True
-        else:
-          return False
-
    def extractChapterUrlsAndMetadata(self):

        url = self.url+'&index=1'
@@ -131,7 +123,8 @@ class PotionsAndSnitchesNetSiteAdapter(BaseSiteAdapter):
                while not defaultGetattr(value,'class') == 'listbox':
                    svalue += str(value)
                    value = value.nextSibling
-                self.story.setMetadata('description',stripHTML(svalue))
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))

            if 'Rated' in label:
                self.story.setMetadata('rating', value)
@@ -145,6 +138,16 @@ class PotionsAndSnitchesNetSiteAdapter(BaseSiteAdapter):
                for cat in catstext:
                    self.story.addToList('category',cat.string)

+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                charstext = [char.string for char in chars]
+                for char in charstext:
+                    if char == "!Snape and Harry (required)":
+                        self.story.addToList('characters',"Snape")
+                        self.story.addToList('characters',"Harry")
+                    else:
+                        self.story.addToList('characters',char.string)
+
            if 'Genre' in label:
                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class'))
                genrestext = [genre.string for genre in genres]
@@ -166,7 +169,27 @@ class PotionsAndSnitchesNetSiteAdapter(BaseSiteAdapter):
                #value = value[0:-1]
                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), "%b %d %Y"))

+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/fanfiction/'+a['href']

+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+        
    def getChapterText(self, url):

        logging.debug('Getting chapter text from: %s' % url)
@@ -179,7 +202,7 @@ class PotionsAndSnitchesNetSiteAdapter(BaseSiteAdapter):
        if None == div:
            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
    
-        return utf8FromSoup(div)
+        return self.utf8FromSoup(url,div)

 def getClass():
    return PotionsAndSnitchesNetSiteAdapter
@@ -0,0 +1,215 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return PrisonBreakFicNetAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class PrisonBreakFicNetAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','pbf')
+
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","Prison Break")
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%B %d, %Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.prisonbreakfic.net'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+                
+# summary, rated, word count, categories, characters, genre, warnings, completed, published, updated, seires
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) # XXX
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -0,0 +1,234 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return SamDeanArchiveNuAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class SamDeanArchiveNuAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','sda')
+
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","Supernatural")
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%m/%d/%y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'samdean.archive.nu'
+
+    @classmethod
+    def getAcceptDomains(cls):
+        return ['www.samdean.archive.nu','samdean.archive.nu']
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://")+r"(www\.)?"+re.escape(self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title and author
+        a = soup.find('div', {'id' : 'pagetitle'})
+        
+        aut = a.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',aut['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+aut['href'])
+        self.story.setMetadata('author',aut.string)
+        aut.extract()
+        
+        self.story.setMetadata('title',a.string[:(len(a.string)-3)])
+
+        # Find the chapters:
+        chapters=soup.find('select')
+        if chapters != None:
+            for chapter in chapters.findAll('option'):
+                # just in case there's tags, like <i> in chapter titles. http://samdean.archive.nu/viewstory.php?sid=4317&chapter=2
+                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/viewstory.php?sid='+self.story.getMetadata('storyId')+'&chapter='+chapter['value']))
+        else:
+            self.chapterUrls.append((self.story.getMetadata('title'),url))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+        
+        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
+        
+        for list in asoup.findAll('div', {'class' : re.compile('listbox\s+')}):
+            a = list.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+            if a != None:
+                break
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = list.findAll('span', {'class' : 'classification'})
+        for labelspan in labels:
+            label = labelspan.string
+            value = labelspan.nextSibling
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'classification':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value[:len(value)-2])
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'categories.php\?catid=\d+'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                for char in value.string.split(', '):
+                    if not 'None' in char:
+                        self.story.addToList('characters',char)
+
+            if 'Genre' in label:
+                for genre in value.string.split(', '):
+                    if not 'None' in genre:
+                        self.story.addToList('genre',genre)
+
+            if 'Warnings' in label:
+                for warning in value.string.split(', '):
+                    if not 'None' in warning:
+                        self.story.addToList('warnings',warning)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = list.find('a', href=re.compile(r"series.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -0,0 +1,255 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return SG1HeliopolisComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class SG1HeliopolisComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        self.story.setMetadata('section',self.parsedUrl.path.split('/',)[1])
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/'+self.story.getMetadata('section')+'/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','sghp')
+
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        if 'atlantis' in self.story.getMetadata('section'):
+            self.story.addToList("category","Stargate: Atlantis")
+        else:
+            self.story.addToList("category","Stargate: SG-1")
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%Y.%m.%d"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'sg1-heliopolis.com'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/archive/viewstory.php?sid=1234 http://"+self.getSiteDomain()+"/adult/viewstory.php?sid=1234 http://"+self.getSiteDomain()+"/atlantis/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return r"http://sg1-heliopolis.com/(archive|adult|atlantis)?/viewstory.php\?sid=\d+$"
+
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&warning=4"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&amp;ageconsent=ok)?&amp;warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid &amp; error in url.
+                addurl = addurl.replace("&amp;","&")
+                url = self.url+'&index=1'+addurl
+                logging.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+self.story.getMetadata('section')+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+self.story.getMetadata('section')+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            i=1
+            for a in storyas:
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        break
+                    i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -0,0 +1,247 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+# This function is called by the downloader in all adapter_*.py files
+# in this dir to register the adapter class.  So it needs to be
+# updated to reflect the class below it.  That, plus getSiteDomain()
+# take care of 'Registering'.
+def getClass():
+    return SiyeCoUkAdapter # XXX
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class SiyeCoUkAdapter(BaseSiteAdapter): # XXX
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8",]# 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/siye/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','siye') # XXX
+
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","Harry Potter") # XXX
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%Y.%m.%d" # XXX
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.siye.co.uk' # XXX
+
+    @classmethod
+    def getAcceptDomains(cls):
+        return ['www.siye.co.uk','siye.co.uk']
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/siye/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://")+r"(www\.)?siye\.co\.uk/(siye/)?"+re.escape("viewstory.php?sid=")+r"\d+$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        # Except it doesn't this time. :-/
+        url = self.url #+'&index=1'+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/siye/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # need(or easier) to pull other metadata from the author's list page.
+        authsoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
+
+        ## Title
+        titlea = authsoup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',titlea.string)
+        
+        # Find the chapters (from soup, not authsoup):
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/siye/'+chapter['href']))
+
+        if self.chapterUrls:
+            self.story.setMetadata('numChapters',len(self.chapterUrls))
+        else:
+            self.chapterUrls.append((self.story.getMetadata('title'),url))
+            self.story.setMetadata('numChapters',1)
+
+        # The stuff we can get from the chapter list/one-shot page are
+        # in the first table with 95% width.
+        metatable = soup.find('table',{'width':'95%'})
+
+        # Categories
+        cat_as = metatable.findAll('a', href=re.compile(r'categories.php'))
+        for cat_a in cat_as:
+            self.story.addToList('category',stripHTML(cat_a))
+
+        moremetaparts = stripHTML(metatable).split('\n')
+        for part in moremetaparts:
+            part = part.strip()
+            if part.startswith("Characters:"):
+                part = part[part.find(':')+1:]
+                for item in part.split(','):
+                    if item.strip() == "Harry/Ginny":
+                        self.story.addToList('characters',"Harry")
+                        self.story.addToList('characters',"Ginny")
+                    elif item.strip() not in ("None","All"):
+                        self.story.addToList('characters',item)
+
+            if part.startswith("Genres:"):
+                part = part[part.find(':')+1:]
+                for item in part.split(','):
+                    if item.strip() != "None":
+                        self.story.addToList('genre',item)
+
+            if part.startswith("Warnings:"):
+                part = part[part.find(':')+1:]
+                for item in part.split(','):
+                    if item.strip() != "None":
+                        self.story.addToList('warnings',item)
+
+            if part.startswith("Rating:"):
+                part = part[part.find(':')+1:]
+                self.story.setMetadata('rating',part)
+
+            if part.startswith("Summary:"):
+                part = part[part.find(':')+1:]
+                self.setDescription(url,part)
+                #self.story.setMetadata('description',part)
+                
+        # want to get the next tr of the table.
+        #print("%s"%titlea.parent.parent.findNextSibling('tr'))
+            
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+                
+        # SIYE formats stories in the author list differently when their part of a series.
+        # Look for non-series...
+        divdesc = titlea.parent.parent.find('div',{'class':'desc'})
+        if not divdesc:
+            # ... now look for series.
+            divdesc = titlea.parent.parent.findNextSibling('tr').find('div',{'class':'desc'})
+
+        moremeta = stripHTML(divdesc)
+        #print("moremeta:%s"%moremeta)
+        for part in moremeta.replace(' - ','\n').split('\n'):
+            #print("part:%s"%part)
+            try:
+                (name,value) = part.split(': ')
+            except:
+                # not going to worry about fancier processing for the bits
+                # that don't match.
+                continue
+            name=name.strip()
+            value=value.strip()
+            if name == 'Published':
+                self.story.setMetadata('datePublished', makeDate(value, self.dateformat))
+            if name == 'Updated':
+                self.story.setMetadata('dateUpdated', makeDate(value, self.dateformat))
+            if name == 'Completed':
+                if value == 'Yes':
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+            if name == 'Words':
+                self.story.setMetadata('numWords', value)
+
+        try:
+            # Find Series name from series URL.
+            a = titlea.findPrevious('a', href=re.compile(r"series.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass        
+
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        # soup = bs.BeautifulSoup(self._fetchUrl(url))
+        # BeautifulSoup objects to <p> inside <span>, which
+        # technically isn't allowed.
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+
+        # not the most unique thing in the world, but it appears to be
+        # the best we can do here.
+        story = soup.find('span', {'style' : 'font-size: 100%;'})
+        
+        if None == story:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,story)
@@ -0,0 +1,234 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+
+def getClass():
+    return SquidgeOrgPejaAdapter
+
+## XXX IMPORTANT NOTE!!  This adapter is for squidge.org/peja ONLY!
+## There are lots of other sites and stuff under squidge.org that
+## we're not supporting.  If/When we ever want to support more
+## sections of squidge.org, FFDL will need to be changed more
+## fundamentally to find different adapters under the same domain.
+##
+## For now, I've only implemented the part for ini section names so
+## if/when more adapters under squidge.org come along, existing ini
+## files will still work correctly.
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class SquidgeOrgPejaAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["utf8",
+                       "Windows-1252"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/peja/cgi-bin/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','wwomb')
+        
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%m/%d/%y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.squidge.org'
+    
+    @classmethod # must be @staticmethod, don't remove it.
+    def getConfigSection(cls):
+        # The config section name.  Only override if != site domain.
+        return cls.getSiteDomain()+'/peja'
+    
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/peja/cgi-bin/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/")+"~?"+re.escape("peja/cgi-bin/viewstory.php?sid=")+r"\d+$"        
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        url = self.url
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        data = self._fetchUrl(url)
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        # Find authorid and URL from... author url.
+        author = soup.find('div', {'id':"pagetitle"}).find('a')
+        self.story.setMetadata('authorId',author['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/peja/cgi-bin/'+author['href'])
+        self.story.setMetadata('author',author.string)
+		
+        authorSoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        title = authorSoup.find('a',{'href':'viewstory.php?sid='+self.story.getMetadata('storyId')})
+        self.story.setMetadata('title',title.string)
+        titleblock=title.parent.parent
+        
+        chapterselect=soup.find('select',{'name':'chapter'})
+        if chapterselect:
+            for ch in chapterselect.findAll('option'):
+                self.chapterUrls.append((stripHTML(ch),'http://'+self.host+'/peja/cgi-bin/viewstory.php?sid='+self.story.getMetadata('storyId')+'&chapter='+ch['value']))
+        else:
+            self.chapterUrls.append((title,url))
+		
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="classification">Rated:</span> NC-17<br /> etc
+        labels = titleblock.findAll('span',{'class':'classification'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'classification':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Characters' in label:
+                for char in value.split(','):
+                    self.story.addToList('characters',char.strip())
+
+            if 'Genre' in label:
+                for genre in value.split(','):
+                    if genre.strip() != "None":
+                        self.story.addToList('genre',genre.strip())
+
+            if 'Warnings' in label:
+                for warning in value.split(','):
+                    if warning.strip() != 'None':
+                        self.story.addToList('warnings',warning.strip())
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Fandoms' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'categories.php'))
+                catstext = [cat.string for cat in cats]
+                for cat in catstext:
+                    self.story.addToList('category',cat.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            # http://www.squidge.org/peja/cgi-bin/series.php?seriesid=254
+            a = titleblock.find('a', href=re.compile(r"series.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/peja/cgi-bin/'+a['href']
+    
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+                
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+	
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        chaptext = soup.find('div',{'id':"story"}).find('span')
+
+        if None == chaptext:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,chaptext)
@@ -0,0 +1,231 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return StargateAtlantisOrgAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class StargateAtlantisOrgAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/fanfics/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','stat')
+
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","Stargate: Atlantis")
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%B %d %Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'stargate-atlantis.org'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/fanfics/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/fanfics/viewstory.php?sid=")+r"\d+$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title and author
+        a = soup.find('div', {'id' : 'pagetitle'})
+        
+        aut = a.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',aut['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/fanfics/'+aut['href'])
+        self.story.setMetadata('author',aut.string)
+        aut.extract()
+        
+        self.story.setMetadata('title',a.string[:(len(a.string)-3)])
+
+        # Find the chapters:
+        chapters=soup.findAll('div', {'class' : 'content'})
+        if len(chapters) > 1:
+            for chapter in chapters:
+                # just in case there's tags, like <i> in chapter titles. 
+                link = chapter.find('a')
+                self.chapterUrls.append((stripHTML(link),'http://'+self.host+'/fanfics/'+link['href']))
+        else:
+            self.chapterUrls.append((self.story.getMetadata('title'),url))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+        
+        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
+        
+        for list in asoup.findAll('div', {'class' : re.compile('listbox\s+')}):
+            a = list.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+            if a != None:
+                break
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = list.findAll('span', {'class' : 'classification'})
+        for labelspan in labels:
+            label = labelspan.string
+            value = labelspan.nextSibling
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'tail' and value != None:
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value[:len(value)-2])
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'categories.php\?catid=\d+'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                for char in value.string.split(', '):
+                    if not 'None' in char:
+                        self.story.addToList('characters',char)
+
+            if 'Genre' in label:
+                for genre in value.string.split(', '):
+                    if not 'None' in genre:
+                        self.story.addToList('genre',genre)
+
+            if 'Warnings' in label:
+                for warning in value.string.split(', '):
+                    if not 'None' in warning:
+                        self.story.addToList('warnings',warning)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value.split(' ::')[0]), self.dateformat))
+            
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = list.find('a', href=re.compile(r"series.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/fanfics/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+                                     
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -0,0 +1,152 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return StoriesOfArdaComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class StoriesOfArdaComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/chapterlistview.asp?SID='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','soa')
+
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","Lord of the Rings")
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%m/%d/%Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.storiesofarda.com'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/chapterlistview.asp?SID=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/chapterlistview.asp?SID=")+r"\d+$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title and author
+        a = soup.find('th', {'colspan' : '3'})
+        
+        aut = a.find('a')
+        self.story.setMetadata('authorId',aut['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+aut['href'])
+        self.story.setMetadata('author',aut.string)
+        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
+        
+        a.find('em').extract()
+        self.story.setMetadata('title',a.text)
+
+        # Find the chapters: chapterview.asp?sid=7000&cid=30919
+        chapters=soup.findAll('a', href=re.compile(r'chapterview.asp\?sid='+self.story.getMetadata('storyId')+"&cid=\d+$"))
+        if len(chapters)==1:
+            self.chapterUrls.append((self.story.getMetadata('title'),'http://'+self.host+'/'+chapters[0]['href']))
+        else:
+            for chapter in chapters:
+                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+        
+        summary = soup.find('td', {'colspan' : '3'})
+        self.setDescription(url,summary)
+        
+        # no convenient way to get word count
+
+        for td in asoup.findAll('td', {'colspan' : '3'}):
+            if td.find('a', href=re.compile('chapterlistview.asp\?SID='+self.story.getMetadata('storyId'))) != None:
+                break     
+        td=td.nextSibling.nextSibling
+        self.story.setMetadata('dateUpdated', makeDate(stripHTML(td).split(': ')[1], self.dateformat))
+        tr=td.parent.nextSibling.nextSibling.nextSibling.nextSibling
+        td=tr.findAll('td')
+        self.story.setMetadata('rating', td[0].string.split(': ')[1])
+        self.story.setMetadata('status', td[2].string.split(': ')[1])
+        self.story.setMetadata('datePublished', makeDate(stripHTML(td[4]).split(': ')[1], self.dateformat))
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('table', {'width' : '90%'}).find('td')
+        div.name='div'
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -21,11 +21,11 @@ import re
 import urllib
 import urllib2

-import fanficdownloader.BeautifulSoup as bs
-from fanficdownloader.htmlcleanup import stripHTML
-import fanficdownloader.exceptions as exceptions
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions

-from base_adapter import BaseSiteAdapter, utf8FromSoup, makeDate
+from base_adapter import BaseSiteAdapter,  makeDate

 class TenhawkPresentsComSiteAdapter(BaseSiteAdapter):

@@ -164,7 +164,8 @@ class TenhawkPresentsComSiteAdapter(BaseSiteAdapter):
                while not defaultGetattr(value,'class') == 'label':
                    svalue += str(value)
                    value = value.nextSibling
-                self.story.setMetadata('description',stripHTML(svalue))
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))

            if 'Rated' in label:
                self.story.setMetadata('rating', value)
@@ -178,6 +179,12 @@ class TenhawkPresentsComSiteAdapter(BaseSiteAdapter):
                for cat in catstext:
                    self.story.addToList('category',cat.string)

+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                charstext = [char.string for char in chars]
+                for char in charstext:
+                    self.story.addToList('characters',char.string)
+
            if 'Genre' in label:
                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class'))
                genrestext = [genre.string for genre in genres]
@@ -199,6 +206,26 @@ class TenhawkPresentsComSiteAdapter(BaseSiteAdapter):
                #value = value[0:-1]
                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))

+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            

    def getChapterText(self, url):

@@ -212,7 +239,7 @@ class TenhawkPresentsComSiteAdapter(BaseSiteAdapter):
        if None == span:
            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
    
-        return utf8FromSoup(span)
+        return self.utf8FromSoup(url,span)

 def getClass():
    return TenhawkPresentsComSiteAdapter
@@ -19,10 +19,10 @@ import datetime
 import time
 import logging

-import fanficdownloader.BeautifulSoup as bs
-import fanficdownloader.exceptions as exceptions
+from .. import BeautifulSoup as bs
+from .. import exceptions

-from base_adapter import BaseSiteAdapter, utf8FromSoup, makeDate
+from base_adapter import BaseSiteAdapter,  makeDate

 class TestSiteAdapter(BaseSiteAdapter):

@@ -54,6 +54,12 @@ class TestSiteAdapter(BaseSiteAdapter):
        if self.story.getMetadata('storyId') == '666':
            raise exceptions.StoryDoesNotExist(self.url)

+        if self.story.getMetadata('storyId').startswith('670'):
+            time.sleep(1.0)
+            
+        if self.story.getMetadata('storyId').startswith('671'):
+            time.sleep(1.0)
+            
        if self.getConfig("username"):
            self.username = self.getConfig("username")
        
@@ -61,25 +67,42 @@ class TestSiteAdapter(BaseSiteAdapter):
            raise exceptions.FailedToLogin(self.url,self.username)

        if self.story.getMetadata('storyId') == '664':
-            self.story.setMetadata(u'title',"Test Story Title "+self.crazystring)
+            self.story.setMetadata(u'title',"Test Story Title "+self.story.getMetadata('storyId')+self.crazystring)
+            self.story.setMetadata('author','Test Author aa bare amp(&) quote(&#39;) amp(&amp;)')
        else:
-            self.story.setMetadata(u'title',"Test Story Title")
+            self.story.setMetadata(u'title',"Test Story Title "+self.story.getMetadata('storyId'))
+            self.story.setMetadata('author','Test Author aa')
        self.story.setMetadata('storyUrl',self.url)
        self.story.setMetadata('description',u'Description '+self.crazystring+u''' Done

 Some more longer description.  "I suck at summaries!"  "Better than it sounds!"  "My first fic"
 ''')
-        self.story.setMetadata('datePublished',makeDate("1972-01-31","%Y-%m-%d"))
-        self.story.setMetadata('dateCreated',datetime.datetime.now())
+        self.story.setMetadata('datePublished',makeDate("1975-03-15","%Y-%m-%d"))
        if self.story.getMetadata('storyId') == '669':
            self.story.setMetadata('dateUpdated',datetime.datetime.now())
        else:
-            self.story.setMetadata('dateUpdated',makeDate("1975-01-31","%Y-%m-%d"))
+            self.story.setMetadata('dateUpdated',makeDate("1975-04-15","%Y-%m-%d"))
        self.story.setMetadata('numWords','123456')
-        self.story.setMetadata('status','In-Completed')
+
+        idnum = int(self.story.getMetadata('storyId'))
+        if idnum % 2 == 1:
+            self.story.setMetadata('status','In-Progress')
+        else:
+            self.story.setMetadata('status','Completed')
+
+        # greater than 10, no language or series.
+        if idnum < 10:
+            langs = {
+                0:"English",
+                1:"Russian",
+                2:"French",
+                3:"German",
+                }
+            self.story.setMetadata('language',langs[idnum%len(langs)])
+            self.setSeries('The Great Test',idnum)
+        
        self.story.setMetadata('rating','Tweenie')
        
-        self.story.setMetadata('author','Test Author aa')
        self.story.setMetadata('authorId','98765')
        self.story.setMetadata('authorUrl','http://author/url')

@@ -89,7 +112,8 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
        self.story.addToList('category','Harry Potter')
        self.story.addToList('category','Furbie')
        self.story.addToList('category','Crossover')
-        
+        self.story.addToList('category',u'Puella Magi Madoka Magica/魔法少女まどか★マギカ')
+        self.story.addToList('category',u'Magical Girl Lyrical Nanoha')
        self.story.addToList('genre','Fantasy')
        self.story.addToList('genre','SF')
        self.story.addToList('genre','Noir')
@@ -100,7 +124,7 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
                            ('Chapter 3, Over Cinnabar',self.url+"&chapter=4"),
                            ('Chapter 4',self.url+"&chapter=5"),
                            ('Chapter 5',self.url+"&chapter=6"),
-                            # ('Chapter 6',self.url+"&chapter=6"),
+                            ('Chapter 6',self.url+"&chapter=6"),
                            # ('Chapter 7',self.url+"&chapter=6"),
                            # ('Chapter 8',self.url+"&chapter=6"),
                            # ('Chapter 9',self.url+"&chapter=6"),
@@ -128,14 +152,15 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
        if self.story.getMetadata('storyId') == '667':
            raise exceptions.FailedToDownload("Error downloading Chapter: %s!" % url)

-        if self.story.getMetadata('storyId') == '670':
-            time.sleep(2.0)
+        if self.story.getMetadata('storyId').startswith('670') or \
+                self.story.getMetadata('storyId').startswith('672'):
+            time.sleep(1.0)

        if "chapter=1" in url :
            text=u'''
 <div>
 <h3>Prologue</h3>
-<p>This is a fake adapter for testing purposes.  Different storyId's will give different errors:</p>
+<p>This is a fake adapter for testing purposes.  Different sid's will give different errors:</p>
 <p>http://test1.com?sid=664 - Crazy string title</p>
 <p>http://test1.com?sid=665 - raises AdultCheckRequired</p>
 <p>http://test1.com?sid=666 - raises StoryDoesNotExist</p>
@@ -143,26 +168,30 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
 <p>http://test1.com?sid=668 - raises FailedToLogin unless username='Me'</p>
 <p>http://test1.com?sid=669 - Succeeds with Updated Date=now</p>
 <p>http://test1.com?sid=670 - Succeeds, but sleeps 2sec on each chapter</p>
-<p>And other storyId will succeed with the same output.</p>
+<p>http://test1.com?sid=671 - Succeeds, but sleeps 2sec metadata only</p>
+<p>http://test1.com?sid=672 - Succeeds, quick meta, sleeps 2sec chapters only</p>
+<p>Odd sid's will be In-Progress, evens complete.  sid&lt;10 will be assigned one of four languages and included in a series.</p>
 </div>
 '''
        else:
            text=u'''
 <div>
-<h3>Chapter</h3>
-<p><center>Centered text</center></p>
-<p>Lorem '''+self.crazystring+''' <i>italics</i>, <b>bold</b>, <u>underline</u> consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p>
+<h3>Chapter title from site</h3>
+<p>Lorem '''+self.crazystring+u''' <i>italics</i>, <b>bold</b>, <u>underline</u> consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p>
 br breaks<br><br>
+Puella Magi Madoka Magica/魔法少女まどか★マギカ
+<!-- a href="http://code.google.com/p/fanficdownloader/wiki/FanFictionDownLoaderPluginWithReadingList" title="Tilt-a-Whirl by Jim &amp; Sarah, on Flickr"><img src="http://i.imgur.com/bo8eD.png"></a --><br/>
 br breaks<br><br>
+Don't&#8212e;ver&#8212d;o&#8212;that&#8212a;gain, &#27861; &#xE9;
 <hr>
 horizontal rules
-<hr>
+<hr size=1 noshade>
 <p>Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p>
 <p>Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p>
 </div>
 '''
        soup = bs.BeautifulStoneSoup(text,selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        return utf8FromSoup(soup)
+        return self.utf8FromSoup(url,soup)

 def getClass():
    return TestSiteAdapter
@@ -0,0 +1,216 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return TheAlphaGateComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class TheAlphaGateComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','tag')
+
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","Stargate: SG-1")
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%d %b %Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.thealphagate.com'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -0,0 +1,201 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return TheHexFilesNetAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class TheHexFilesNetAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','thf')
+
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","Harry Potter")
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%Y.%m.%d"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'thehexfiles.net'
+
+    @classmethod
+    def getAcceptDomains(cls):
+        return ['www.thehexfiles.net','thehexfiles.net']
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://")+"(www\.)?"+re.escape(self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
+		
+        try:
+            # in case link points somewhere other than the first chapter
+            a = soup.findAll('option')[1]['value']
+            self.story.setMetadata('storyId',a.split('=',)[1])
+            url = 'http://'+self.host+'/'+a
+            soup = bs.BeautifulSoup(self._fetchUrl(url))
+        except:
+            pass
+		
+        for info in asoup.findAll('table', {'cellspacing' : '4'}):
+            a = info.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+            if a != None:
+                self.story.setMetadata('title',a.string)
+                break
+		
+
+        # Find the chapters:
+        chapters=soup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+&i=1$'))
+        if len(chapters) == 0:
+            self.chapterUrls.append((self.story.getMetadata('title'),url))
+        else:
+            for chapter in chapters:
+                # just in case there's tags, like <i> in chapter titles.
+                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+				
+        cats = info.findAll('a',href=re.compile('categories.php'))
+        for cat in cats:
+            self.story.addToList('category',cat.string)
+			
+        words = info.find(text=re.compile('Words:')).split('|')[1].split(': ')[1]
+        self.story.setMetadata('numWords', words)
+		
+        comp = info.find('span', {'class' : 'completed'}).string.split(': ')[1]
+        if 'Yes' in comp:
+            self.story.setMetadata('status', 'Completed')
+        else:
+            self.story.setMetadata('status', 'In-Progress')
+						
+        summary = info.find('td', {'class' : 'summary'})
+        self.setDescription(url,summary)
+		
+        rating=stripHTML(info.find('td', {'align' : 'left'})).split('(')[1].split(')')[0]
+        self.story.setMetadata('rating', rating)
+
+        labels = info.findAll('td', {'width' : '10%'})
+        values = info.findAll('td', {'width' : '40%'})
+        for i in range(0,len(labels)):
+            value = stripHTML(values[i])
+            label = stripHTML(labels[i])
+
+            if 'Genres' in label:
+                genres = value.split(', ')
+                for genre in genres:
+                    if genre != 'none':
+                        self.story.addToList('genre',genre)
+						
+
+            if 'Warnings' in label:
+                warnings = value.split(', ')
+                for warning in warnings:
+                    if warning != 'none':
+                        self.story.addToList('warnings',warning)
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        for a in soup.findAll('table'):
+            a.extract()
+
+        if None == soup:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,soup)
@@ -0,0 +1,293 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+# This function is called by the downloader in all adapter_*.py files
+# in this dir to register the adapter class.  So it needs to be
+# updated to reflect the class below it.  That, plus getSiteDomain()
+# take care of 'Registering'.
+def getClass():
+    return TheQuidditchPitchOrgAdapter # XXX
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class TheQuidditchPitchOrgAdapter(BaseSiteAdapter): # XXX
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        # XXX Most sites don't have the  part.  Replace all to remove it usually.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','tqdpch') # XXX
+
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","Harry Potter") # XXX
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%m/%d/%Y" # XXX
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'thequidditchpitch.org' # XXX
+
+    @classmethod
+    def getAcceptDomains(cls):
+        return ['www.thequidditchpitch.org','thequidditchpitch.org']
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://")+"(www\.)?"+re.escape(self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Registered Users Only - Not suitable for readers under the age of legal consent in their country.' in data \
+                or 'There is no such account on our website' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['cookiecheck'] = '1'
+        params['submit'] = 'Submit'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
+        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "Member Account" not in d : #Member Account
+            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=4" # XXX
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+
+        # The actual text that is used to announce you need to be an
+        # adult varies from site to site.  Again, print data before
+        # the title search to troubleshoot.
+        if ("Not suitable for readers under the age of legal consent in their country." in data \
+                or "Not suitable for readers under 16 yrs. \r\nStories may contain violence, slight nudity, and/or sexual situations." in data ) \
+                and not (self.is_adult or self.getConfig("is_adult")): # XXX 
+            raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        #print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                catstext = [cat.string for cat in cats]
+                for cat in catstext:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                charstext = [char.string for char in chars]
+                for char in charstext:
+                    self.story.addToList('characters',char.string)
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) # XXX
+                genrestext = [genre.string for genre in genres]
+                self.genre = ', '.join(genrestext)
+                for genre in genrestext:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
+                warningstext = [warning.string for warning in warnings]
+                self.warning = ', '.join(warningstext)
+                for warning in warningstext:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+
+        # span?  Really?  span?  Yeah... I don't think so.
+        div = soup.find('span', {'style' : 'font-size: 100%;'})
+        div.name='div'
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -21,11 +21,11 @@ import re
 import urllib
 import urllib2

-import fanficdownloader.BeautifulSoup as bs
-from fanficdownloader.htmlcleanup import stripHTML
-import fanficdownloader.exceptions as exceptions
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions

-from base_adapter import BaseSiteAdapter, utf8FromSoup, makeDate
+from base_adapter import BaseSiteAdapter,  makeDate

 class TheWritersCoffeeShopComSiteAdapter(BaseSiteAdapter):

@@ -123,6 +123,11 @@ class TheWritersCoffeeShopComSiteAdapter(BaseSiteAdapter):
        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
            
+        # problems with some stories, but only in calibre.  I suspect
+        # issues with different SGML parsers in python.  This is a
+        # nasty hack, but it works.
+        data = data[data.index("<body"):] 
+        
        # use BeautifulSoup HTML parser to make everything easier to find.
        soup = bs.BeautifulSoup(data)

@@ -160,8 +165,14 @@ class TheWritersCoffeeShopComSiteAdapter(BaseSiteAdapter):
                svalue = ""
                while not defaultGetattr(value,'class') == 'label':
                    svalue += str(value)
-                    value = value.nextSibling
-                self.story.setMetadata('description',stripHTML(svalue))
+                    # poor HTML(unclosed <p> for one) can cause run on
+                    # over the next label.
+                    if '<span class="label">' in svalue:
+                        svalue = svalue[0:svalue.find('<span class="label">')]
+                        break
+                    else:
+                        value = value.nextSibling
+                self.setDescription(url,svalue)

            if 'Rated' in label:
                self.story.setMetadata('rating', value)
@@ -175,6 +186,12 @@ class TheWritersCoffeeShopComSiteAdapter(BaseSiteAdapter):
                for cat in catstext:
                    self.story.addToList('category',cat.string)

+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                charstext = [char.string for char in chars]
+                for char in charstext:
+                    self.story.addToList('characters',char.string)
+
            if 'Genre' in label:
                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class'))
                genrestext = [genre.string for genre in genres]
@@ -196,12 +213,37 @@ class TheWritersCoffeeShopComSiteAdapter(BaseSiteAdapter):
                #value = value[0:-1]
                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))

+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/library/'+a['href']

+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
    def getChapterText(self, url):

        logging.debug('Getting chapter text from: %s' % url)

-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+        data = self._fetchUrl(url)
+        # problems with some stories, but only in calibre.  I suspect
+        # issues with different SGML parsers in python.  This is a
+        # nasty hack, but it works.
+        data = data[data.index("<body"):]
+        
+        soup = bs.BeautifulStoneSoup(data,
                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
        
        span = soup.find('div', {'id' : 'story'})
@@ -209,7 +251,7 @@ class TheWritersCoffeeShopComSiteAdapter(BaseSiteAdapter):
        if None == span:
            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
    
-        return utf8FromSoup(span)
+        return self.utf8FromSoup(url,span)

 def getClass():
    return TheWritersCoffeeShopComSiteAdapter
@@ -0,0 +1,258 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+import time
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+        self.story.setMetadata('siteabbrev','tth')
+        self.dateformat = "%d %b %y"
+        self.is_adult=False
+        self.username = None
+        self.password = None
+        # get storyId from url--url validation guarantees query correct
+        m = re.match(self.getSiteURLPattern(),url)
+        if m:
+            self.story.setMetadata('storyId',m.group('id'))
+            logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+            # normalized story URL.
+            self._setURL("http://"+self.getSiteDomain()\
+                         +"/Story-"+self.story.getMetadata('storyId'))
+        else:
+            raise exceptions.InvalidStoryURL(url,
+                                             self.getSiteDomain(),
+                                             self.getSiteExampleURLs())
+
+    @staticmethod
+    def getSiteDomain():
+        return 'www.tthfanfic.org'
+
+    def getSiteExampleURLs(self):
+        return "http://www.tthfanfic.org/Story-5583 http://www.tthfanfic.org/Story-5583/Greywizard+Marked+By+Kane.htm http://www.tthfanfic.org/T-526321777890480578489880055880/Story-26448-15/batzulger+Willow+Rosenberg+and+the+Mind+Riders.htm"
+
+    # http://www.tthfanfic.org/T-526321777848988007890480555880/Story-26448-15/batzulger+Willow+Rosenberg+and+the+Mind+Riders.htm
+    # http://www.tthfanfic.org/Story-5583
+    # http://www.tthfanfic.org/Story-5583/Greywizard+Marked+By+Kane.htm
+    # http://www.tthfanfic.org/story.php?no=26093
+    def getSiteURLPattern(self):
+        return r"http://www.tthfanfic.org(/(T-\d+/)?Story-|/story.php\?no=)(?P<id>\d+)(-\d+)?(/.*)?$"
+
+    # tth won't send you future updates if you aren't 'caught up'
+    # on the story.  Login isn't required for F21, but logging in will
+    # mark stories you've downloaded as 'read' on tth.
+    def performLogin(self):
+        params = {}
+
+        if self.password:
+            params['urealname'] = self.username
+            params['password'] = self.password
+        else:
+            params['urealname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['loginsubmit'] = 'Login'
+
+        if not params['password']:
+            return
+        
+        loginUrl = 'http://' + self.getSiteDomain() + '/login.php'
+        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['urealname']))
+
+        ## need to pull empty login page first to get ctkn and
+        ## password name, which are BUSs
+# <form method='post' action='/login.php' accept-charset="utf-8">
+# <input type='hidden' name='ctkn' value='4bdf761f5bea06bf4477072afcbd0f8d721d1a4f989c09945a9e87afb7a66de1'/>
+# <input type='text' id='urealname' name='urealname' value=''/>
+# <input type='password' id='password' name='6bb3fcd148d148629223690bf19733b8'/>
+# <input type='submit' value='Login' name='loginsubmit'/>
+        soup = bs.BeautifulSoup(self._fetchUrl(loginUrl))
+        params['ctkn']=soup.find('input', {'name':'ctkn'})['value']
+        params[soup.find('input', {'id':'password'})['name']] = params['password']
+        
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "Stories Published" not in d : #Member Account
+            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    def extractChapterUrlsAndMetadata(self):
+        # fetch the chapter.  From that we will get almost all the
+        # metadata and chapter list
+
+        url=self.url
+        logging.debug("URL: "+url)
+
+        # tth won't send you future updates if you aren't 'caught up'
+        # on the story.  Login isn't required for F21, but logging in will
+        # mark stories you've downloaded as 'read' on tth.
+        self.performLogin()
+        
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        try:
+            data = self._fetchUrl(url)
+            soup = bs.BeautifulSoup(data)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(url)
+            else:
+                raise e
+            
+        descurl = url
+            
+        if "<h2>Story Not Found</h2>" in data:
+            raise exceptions.StoryDoesNotExist(url)
+        
+        if "NOTE: This story is rated FR21 which is above your chosen filter level." in data:
+            if self.is_adult or self.getConfig("is_adult"):
+                form = soup.find('form', {'id':'sitemaxratingform'})
+                params={'ctkn':form.find('input', {'name':'ctkn'})['value'],
+                        'sitemaxrating':'5'}
+                logging.info("Attempting to get rating cookie for %s" % url)
+                data = self._postUrl("http://"+self.getSiteDomain()+'/setmaxrating.php',params)
+                # refetch story page.
+                data = self._fetchUrl(url)
+                soup = bs.BeautifulSoup(data)
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+
+        # http://www.tthfanfic.org/AuthorStories-3449/Greywizard.htm
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"^/AuthorStories-\d+"))
+        self.story.setMetadata('authorId',a['href'].split('/')[1].split('-')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+a['href'])
+        self.story.setMetadata('author',stripHTML(a))
+
+        try:
+            # going to pull part of the meta data from author list page.
+            logging.debug("**AUTHOR** URL: "+self.story.getMetadata('authorUrl'))
+            authordata = self._fetchUrl(self.story.getMetadata('authorUrl'))
+            descurl=self.story.getMetadata('authorUrl')
+            authorsoup = bs.BeautifulSoup(authordata)
+            # author can have several pages, scan until we find it.
+            while( not authorsoup.find('a', href=re.compile(r"^/Story-"+self.story.getMetadata('storyId'))) ):
+                nextpage = 'http://'+self.host+authorsoup.find('a', {'class':'arrowf'})['href']
+                logging.debug("**AUTHOR** nextpage URL: "+nextpage)
+                authordata = self._fetchUrl(nextpage)
+                descurl=nextpage
+                authorsoup = bs.BeautifulSoup(authordata)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(url)
+            else:
+                raise e
+
+        storydiv = authorsoup.find('div', {'id':'st'+self.story.getMetadata('storyId'), 'class':re.compile(r"storylistitem")})
+        self.setDescription(descurl,storydiv.find('div',{'class':'storydesc'}))
+        #self.story.setMetadata('description',stripHTML(storydiv.find('div',{'class':'storydesc'})))
+        self.story.setMetadata('title',stripHTML(storydiv.find('a',{'class':'storylink'})))
+
+        verticaltable = soup.find('table', {'class':'verticaltable'})
+
+        BtVS = True
+        BtVSNonX = False
+        for cat in verticaltable.findAll('a', href=re.compile(r"^/Category-")):
+            if cat.string not in ['General', 'Non-BtVS/AtS Stories', 'BtVS/AtS Non-Crossover', 'Non-BtVS Crossovers']:
+                self.story.addToList('category',cat.string)
+            else:
+                if 'Non-BtVS' in cat.string:
+                    BtVS = False
+                if 'BtVS/AtS Non-Crossover' == cat.string:
+                    BtVSNonX = True
+
+        verticaltabletds = verticaltable.findAll('td')
+        self.story.setMetadata('rating', verticaltabletds[2].string)
+        self.story.setMetadata('numWords', verticaltabletds[4].string)
+        
+        # Complete--if completed.
+        if 'Yes' in verticaltabletds[10].string:
+            self.story.setMetadata('status', 'Completed')
+        else:
+            self.story.setMetadata('status', 'In-Progress')
+            
+        self.story.setMetadata('datePublished',makeDate(stripHTML(verticaltabletds[8].string), self.dateformat))
+        self.story.setMetadata('dateUpdated',makeDate(stripHTML(verticaltabletds[9].string), self.dateformat))
+
+        for icon in storydiv.find('span',{'class':'storyicons'}).findAll('img'):
+            if( icon['title'] not in ['Non-Crossover'] ) :
+                self.story.addToList('genre',icon['title'])
+            else:
+                if not BtVSNonX:
+                    BtVS = False # Don't add BtVS if Non-Crossover, unless it's a BtVS/AtS Non-Crossover
+
+        print("BtVS: %s BtVSNonX: %s"%(BtVS,BtVSNonX))
+        if BtVS:
+            self.story.addToList('category','Buffy: The Vampire Slayer')
+            
+        # Find the chapter selector 
+        select = soup.find('select', { 'name' : 'chapnav' } )
+    	 
+        if select is None:
+    	   # no selector found, so it's a one-chapter story.
+    	   self.chapterUrls.append((self.story.getMetadata('title'),url))
+        else:
+            allOptions = select.findAll('option')
+            for o in allOptions:
+                url = "http://"+self.host+o['value']
+                # just in case there's tags, like <i> in chapter titles.
+                self.chapterUrls.append((stripHTML(o),url))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        pseries = soup.find('p', {'style':'margin-top:0px'})
+        m = re.match('This story is No\. (?P<num>\d+) in the series &quot;(?P<series>.+)&quot;\.',
+                     pseries.text)
+        if m:
+            self.setSeries(m.group('series'),m.group('num'))
+        
+        return
+
+
+    def getChapterText(self, url):
+        logging.debug('Getting chapter text from: %s' % url)
+        soup = bs.BeautifulSoup(self._fetchUrl(url))
+
+        div = soup.find('div', {'id' : 'storyinnerbody'})
+        
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+
+        # strip out included chapter title, if present, to avoid doubling up.
+        try:
+            div.find('h3').extract()
+        except:
+            pass
+        return self.utf8FromSoup(url,div)
+
+def getClass():
+    return TwistingTheHellmouthSiteAdapter
+
@@ -21,11 +21,11 @@ import re
 import urllib
 import urllib2

-import fanficdownloader.BeautifulSoup as bs
-from fanficdownloader.htmlcleanup import stripHTML
-import fanficdownloader.exceptions as exceptions
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions

-from base_adapter import BaseSiteAdapter, utf8FromSoup, makeDate
+from base_adapter import BaseSiteAdapter,  makeDate

 class TwilightedNetSiteAdapter(BaseSiteAdapter):

@@ -118,6 +118,12 @@ class TwilightedNetSiteAdapter(BaseSiteAdapter):
        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
            
+        # problems with some stories, but only in calibre.  I suspect
+        # issues with different SGML parsers in python.  This is a
+        # nasty hack, but it works.
+        # twilighted isn't writing <body> ??? wtf?
+        data = "<html><body>"+data[data.index("</head>"):] 
+        
        # use BeautifulSoup HTML parser to make everything easier to find.
        soup = bs.BeautifulSoup(data)

@@ -156,7 +162,7 @@ class TwilightedNetSiteAdapter(BaseSiteAdapter):
                while not defaultGetattr(value,'class') == 'label':
                    svalue += str(value)
                    value = value.nextSibling
-                self.story.setMetadata('description',stripHTML(svalue))
+                self.setDescription(url,svalue)

            if 'Rated' in label:
                self.story.setMetadata('rating', value)
@@ -170,6 +176,12 @@ class TwilightedNetSiteAdapter(BaseSiteAdapter):
                for cat in catstext:
                    self.story.addToList('category',cat.string)

+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                charstext = [char.string for char in chars]
+                for char in charstext:
+                    self.story.addToList('characters',char.string)
+
            ## twilighted.net doesn't use genre.
            # if 'Genre' in label:
            #     genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class'))
@@ -192,12 +204,38 @@ class TwilightedNetSiteAdapter(BaseSiteAdapter):
                #value = value[0:-1]
                self.story.setMetadata('dateUpdated', makeDate(value.strip(), "%B %d, %Y"))

+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+            
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+        
+        except:
+            # I find it hard to care if the series parsing fails
+            pass

    def getChapterText(self, url):

        logging.debug('Getting chapter text from: %s' % url)

-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+        data = self._fetchUrl(url)
+        # problems with some stories, but only in calibre.  I suspect
+        # issues with different SGML parsers in python.  This is a
+        # nasty hack, but it works.
+        # twilighted isn't writing <body> ??? wtf?
+        data = "<html><body>"+data[data.index("</head>"):] 
+        
+        soup = bs.BeautifulStoneSoup(data,
                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
        
        span = soup.find('div', {'id' : 'story'})
@@ -205,7 +243,7 @@ class TwilightedNetSiteAdapter(BaseSiteAdapter):
        if None == span:
            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
    
-        return utf8FromSoup(span)
+        return self.utf8FromSoup(url,span)

 def getClass():
    return TwilightedNetSiteAdapter
@@ -21,11 +21,11 @@ import re
 import urllib
 import urllib2

-import fanficdownloader.BeautifulSoup as bs
-from fanficdownloader.htmlcleanup import stripHTML
-import fanficdownloader.exceptions as exceptions
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions

-from base_adapter import BaseSiteAdapter, utf8FromSoup, makeDate
+from base_adapter import BaseSiteAdapter,  makeDate

 class TwiwriteNetSiteAdapter(BaseSiteAdapter):

@@ -38,6 +38,7 @@ class TwiwriteNetSiteAdapter(BaseSiteAdapter):
                               # iso-8859-1 (and some that claim to be
                               # utf8) are really windows-1252.
        self.story.addToList("category","Twilight")
+        self.is_adult = False
        self.username = "NoneGiven" # if left empty, twiwrite.net doesn't return any message at all.
        self.password = ""
        
@@ -84,7 +85,7 @@ class TwiwriteNetSiteAdapter(BaseSiteAdapter):
        params['submit'] = 'Submit'
    
        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
-        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+        logging.info("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                              params['penname']))
    
        d = self._fetchUrl(loginUrl, params)
@@ -99,7 +100,16 @@ class TwiwriteNetSiteAdapter(BaseSiteAdapter):

    def extractChapterUrlsAndMetadata(self):

-        url = self.url+'&index=1'
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=1" # XXX
+        else:
+            addurl=""
+            
+        url = self.url+'&index=1'+addurl
        logging.debug("URL: "+url)

        try:
@@ -117,7 +127,15 @@ class TwiwriteNetSiteAdapter(BaseSiteAdapter):

        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
+
+        if "Contains Explicit Content for mature adults only! May contain graphic violence, mature sexual situations, and explicit language. Read with caution." in data:
+            raise exceptions.AdultCheckRequired(self.url)
+        
+        # problems with some stories, but only in calibre.  I suspect
+        # issues with different SGML parsers in python.  This is a
+        # nasty hack, but it works.
+        data = data[data.index("<body"):] 
+        
        # use BeautifulSoup HTML parser to make everything easier to find.
        soup = bs.BeautifulSoup(data)

@@ -134,7 +152,7 @@ class TwiwriteNetSiteAdapter(BaseSiteAdapter):
        # Find the chapters:
        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
            # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']))
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))

        self.story.setMetadata('numChapters',len(self.chapterUrls))

@@ -164,7 +182,8 @@ class TwiwriteNetSiteAdapter(BaseSiteAdapter):
                while not defaultGetattr(value,'class') == 'label':
                    svalue += str(value)
                    value = value.nextSibling
-                self.story.setMetadata('description',stripHTML(svalue))
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))

            if 'Rated' in label:
                self.story.setMetadata('rating', value)
@@ -178,6 +197,12 @@ class TwiwriteNetSiteAdapter(BaseSiteAdapter):
                for cat in catstext:
                    self.story.addToList('category',cat.string)

+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                charstext = [char.string for char in chars]
+                for char in charstext:
+                    self.story.addToList('characters',char.string)
+
            if 'Genre' in label:
                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=3'))
                genrestext = [genre.string for genre in genres]
@@ -206,12 +231,37 @@ class TwiwriteNetSiteAdapter(BaseSiteAdapter):
                value = value[0:-1]
                self.story.setMetadata('dateUpdated', makeDate(value.strip(), "%B %d, %Y"))

+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass

    def getChapterText(self, url):

        logging.debug('Getting chapter text from: %s' % url)

-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+        data = self._fetchUrl(url)
+        # problems with some stories, but only in calibre.  I suspect
+        # issues with different SGML parsers in python.  This is a
+        # nasty hack, but it works.
+        data = data[data.index("<body"):]
+        
+        soup = bs.BeautifulStoneSoup(data,
                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
        
        span = soup.find('div', {'id' : 'story'})
@@ -219,7 +269,7 @@ class TwiwriteNetSiteAdapter(BaseSiteAdapter):
        if None == span:
            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
    
-        return utf8FromSoup(span)
+        return self.utf8FromSoup(url,span)

 def getClass():
    return TwiwriteNetSiteAdapter
@@ -0,0 +1,233 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return WalkingThePlankOrgAdapter
+
+class WalkingThePlankOrgAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/archive/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','wtp')
+
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","Harry Potter")
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%b %d, %Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.walkingtheplank.org'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/archive/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/archive/viewstory.php?sid=")+r"\d+$"
+
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&warning=4"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+
+        # The actual text that is used to announce you need to be an
+        # adult varies from site to site.  Again, print data before
+        # the title search to troubleshoot.
+    
+        if "By clicking this link, you acknowledge" in data:
+            raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/archive/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+				
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                catstext = [cat.string for cat in cats]
+                for cat in catstext:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                charstext = [char.string for char in chars]
+                for char in charstext:
+                    self.story.addToList('characters',char.string)
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/archive/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -20,10 +20,10 @@ import logging
 import re
 import urllib2

-import fanficdownloader.BeautifulSoup as bs
-import fanficdownloader.exceptions as exceptions
+from .. import BeautifulSoup as bs
+from .. import exceptions as exceptions

-from base_adapter import BaseSiteAdapter, utf8FromSoup, makeDate
+from base_adapter import BaseSiteAdapter,  makeDate

 class WhoficComSiteAdapter(BaseSiteAdapter):

@@ -120,9 +120,10 @@ class WhoficComSiteAdapter(BaseSiteAdapter):
        # link instead to find the appropriate metadata.
        a = soup.find('a', href=re.compile(r'reviews.php\?sid='+self.story.getMetadata('storyId')))
        metadata = a.findParent('td')
-        metadatachunks = utf8FromSoup(metadata).split('<br />')
+        metadatachunks = self.utf8FromSoup(None,metadata).split('<br />')
        # process metadata for this story.
-        self.story.setMetadata('description', metadatachunks[1])
+        self.setDescription(url,metadatachunks[1])
+        #self.story.setMetadata('description', metadatachunks[1])

        # First line of the stuff with ' - ' separators
        moremeta = metadatachunks[2]
@@ -158,6 +159,15 @@ class WhoficComSiteAdapter(BaseSiteAdapter):
        for g in genre.split(r', '):
            self.story.addToList('genre',g)

+        # line 3 is characters.
+        chars = metadatachunks[3]
+        charsearch="<i>Characters:</i>"
+        if charsearch in chars:
+            chars = chars[metadatachunks[3].index(charsearch)+len(charsearch):]
+            for c in chars.split(','):
+                if c.strip() != u'None':
+                    self.story.addToList('characters',c)
+            
        # the next line is stuff with ' - ' separators *and* names--with tags.
        moremeta = metadatachunks[5]
        moremeta = re.sub(r'<[^>]+>','',moremeta) # strip tags.
@@ -180,6 +190,26 @@ class WhoficComSiteAdapter(BaseSiteAdapter):
            if name == 'Word Count':
                self.story.setMetadata('numWords', value)

+        try:
+            # Find Series name from series URL.
+            a = metadata.find('a', href=re.compile(r"series.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
    def getChapterText(self, url):

        logging.debug('Getting chapter text from: %s' % url)
@@ -195,7 +225,7 @@ class WhoficComSiteAdapter(BaseSiteAdapter):
        if None == span:
            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
    
-        return utf8FromSoup(span)
+        return self.utf8FromSoup(url,span)

 def getClass():
    return WhoficComSiteAdapter
@@ -0,0 +1,227 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+
+def getClass():
+    return WraithBaitComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class WraithBaitComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','wb')
+
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","Stargate")
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%d  %b %Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.wraithbait.com'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=12"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+				
+        if "for adults only" in data:
+            raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        pt = soup.find('div', {'id' : 'pagetitle'})
+        a = pt.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = pt.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        rating=pt.text.split('[')[1].split(']')[0]
+        self.story.setMetadata('rating', rating)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+		
+        info = soup.find('div', {'class' : 'small'})
+		
+        word=info.find(text=re.compile("Word count:")).split(':')
+        self.story.setMetadata('numWords', word[1])
+		
+        cats = info.findAll('a',href=re.compile(r'browse.php\?type=categories&id=\d'))
+        for cat in cats:
+            if "General" != cat.string:
+                self.story.addToList('category',cat.string)
+				
+        chars = info.findAll('a',href=re.compile(r'browse.php\?type=characters&charid=\d'))
+        for char in chars:
+            self.story.addToList('characters',char.string)
+			
+        completed=info.find(text=re.compile("Completed: Yes"))
+        if completed != None:
+            self.story.setMetadata('status', 'Completed')
+        else:
+            self.story.setMetadata('status', 'In-Progress')
+			
+        date=soup.find('div',{'class' : 'bottom'})
+        pd=date.find(text=re.compile("Published:")).string.split(': ')
+        self.story.setMetadata('datePublished', makeDate(stripHTML(pd[1].split(' U')[0]), self.dateformat))
+        self.story.setMetadata('dateUpdated', makeDate(stripHTML(pd[2]), self.dateformat))
+		
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        pub=0
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Genres' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+			
+        info.extract()
+        summary = soup.find('div', {'class' : 'content'})
+        self.setDescription(url,summary)
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
+
@@ -0,0 +1,266 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return YourFanfictionComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class YourFanfictionComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','yff')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%d %b %Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.yourfanfiction.com'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=4"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        # The actual text that is used to announce you need to be an
+        # adult varies from site to site.  Again, print data before
+        # the title search to troubleshoot.
+            
+        # Since the warning text can change by warning level, let's
+        # look for the warning pass url.  ksarchive uses
+        # &amp;warning= -- actually, so do other sites.  Must be an
+        # eFiction book.
+            
+        # viewstory.php?sid=1882&amp;warning=4
+        # viewstory.php?sid=1654&amp;ageconsent=ok&amp;warning=5
+        #print data
+        #m = re.search(r"'viewstory.php\?sid=1882(&amp;warning=4)'",data)
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&amp;ageconsent=ok)?&amp;warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid &amp; error in url.
+                addurl = addurl.replace("&amp;","&")
+                url = self.url+'&index=1'+addurl
+                logging.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=5'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+                    
+            if 'Tags' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=7'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=6'))
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            i=1
+            for a in storyas:
+                print("series a['href']:%s"%a['href'])
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        break
+                    i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -22,6 +22,10 @@ import logging
 import urllib
 import urllib2 as u2
 import urlparse as up
+from functools import partial
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML

 try:
    from google.appengine.api import apiproxy_stub_map
@@ -39,13 +43,14 @@ except:
    pass
    #logging.info("Hook to make default deadline 10.0 NOT installed--not using appengine")

-from fanficdownloader.story import Story
-from fanficdownloader.configurable import Configurable
-from fanficdownloader.htmlcleanup import removeEntities, removeAllEntities, stripHTML
-from fanficdownloader.exceptions import InvalidStoryURL
+from ..story import Story
+from ..gziphttp import GZipProcessor
+from ..configurable import Configurable
+from ..htmlcleanup import removeEntities, removeAllEntities, stripHTML
+from ..exceptions import InvalidStoryURL

 try:
-    import fanficdownloader.chardet as chardet
+    from .. import chardet as chardet
 except ImportError:
    chardet = None

@@ -63,19 +68,27 @@ class BaseSiteAdapter(Configurable):
        return re.match(self.getSiteURLPattern(), self.url)

    def __init__(self, config, url):
+        self.config = config
        Configurable.__init__(self, config)
-        self.addConfigSection(self.getSiteDomain())
-        self.addConfigSection("overrides")
+        self.setSectionOrder(self.getConfigSection())
        
-        self.opener = u2.build_opener(u2.HTTPCookieProcessor())
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        self.opener = u2.build_opener(u2.HTTPCookieProcessor(),GZipProcessor())
        self.storyDone = False
        self.metadataDone = False
        self.story = Story()
-        self.story.setMetadata('site',self.getSiteDomain())
+        self.story.setMetadata('site',self.getConfigSection())
        self.story.setMetadata('dateCreated',datetime.datetime.now())
        self.chapterUrls = [] # tuples of (chapter title,chapter url)
        self.chapterFirst = None
        self.chapterLast = None
+        self.oldchapters = None
+        self.oldimgs = None
+        self.oldcover = None # (data of existing cover html, data of existing cover image)
+        self.calibrebookmark = None
        ## order of preference for decoding.
        self.decode = ["utf8",
                       "Windows-1252"] # 1252 is a superset of
@@ -121,9 +134,10 @@ class BaseSiteAdapter(Configurable):
                        code=detected['encoding']
                    else:
                        continue
+                logging.debug("try code:"+code)
                return data.decode(code)
            except:
-                logging.debug("code failed:"+code)
+                logging.info("code failed:"+code)
                pass
        logging.info("Could not decode story, tried:%s Stripping non-ASCII."%decode)
        return "".join([x for x in data if ord(x) < 128])
@@ -144,6 +158,12 @@ class BaseSiteAdapter(Configurable):
                         headers=headers)
        return self._decode(self.opener.open(req).read())

+    def _fetchUrlRaw(self, url, parameters=None):
+        if parameters != None:
+            return self.opener.open(url,urllib.urlencode(parameters)).read()
+        else:
+            return self.opener.open(url).read()
+    
    # parameters is a dict()
    def _fetchUrl(self, url, parameters=None):
        if self.getConfig('slow_down_sleep_time'):
@@ -153,10 +173,7 @@ class BaseSiteAdapter(Configurable):
        for sleeptime in [0, 0.5, 4, 9]:
            time.sleep(sleeptime)	
            try:
-                if parameters:
-                    return self._decode(self.opener.open(url,urllib.urlencode(parameters)).read())
-                else:
-                    return self._decode(self.opener.open(url).read())
+                return self._decode(self._fetchUrlRaw(url,parameters))
            except Exception, e:
                excpt=e
                logging.warn("Caught an exception reading URL: %s  Exception %s."%(unicode(url),unicode(e)))
@@ -176,15 +193,43 @@ class BaseSiteAdapter(Configurable):
    def getStory(self):
        if not self.storyDone:
            self.getStoryMetadataOnly()
+
            for index, (title,url) in enumerate(self.chapterUrls):
                if (self.chapterFirst!=None and index < self.chapterFirst) or \
                        (self.chapterLast!=None and index > self.chapterLast):
                    self.story.addChapter(removeEntities(title),
                                          None)
                else:
+                    if self.oldchapters and index < len(self.oldchapters):
+                        data = self.utf8FromSoup(None,
+                                                 self.oldchapters[index],
+                                                 partial(cachedfetch,self._fetchUrlRaw,self.oldimgs))
+                    else:
+                        data = self.getChapterText(url)
                    self.story.addChapter(removeEntities(title),
-                                          removeEntities(self.getChapterText(url)))
+                                          removeEntities(data))
            self.storyDone = True
+            
+            # include image, but no cover from story, add default_cover_image cover.
+            if self.getConfig('include_images') and \
+                    not self.story.cover and \
+                    self.getConfig('default_cover_image'):
+                self.story.addImgUrl(self,
+                                     None,
+                                     #self.getConfig('default_cover_image'),
+                                     self.story.formatFileName(self.getConfig('default_cover_image'),
+                                                               self.getConfig('allow_unsafe_filename')),
+                                     self._fetchUrlRaw,
+                                     cover=True)
+
+            # no new cover, set old cover, if there is one.
+            if not self.story.cover and self.oldcover:
+                self.story.oldcover = self.oldcover
+                
+            # cheesy way to carry calibre bookmark file forward across update.
+            if self.calibrebookmark:
+                self.story.calibrebookmark = self.calibrebookmark
+                
        return self.story

    def getStoryMetadataOnly(self):
@@ -200,6 +245,11 @@ class BaseSiteAdapter(Configurable):
        "Needs to be overriden in each adapter class."
        return 'no such domain'
    
+    @classmethod
+    def getConfigSection(cls):
+        "Only needs to be overriden if != site domain."
+        return cls.getSiteDomain()
+    
    ## URL pattern validation is done *after* picking an adaptor based
    ## on domain instead of *as* the adaptor selector so we can offer
    ## the user example(s) for that particular site.
@@ -223,28 +273,109 @@ class BaseSiteAdapter(Configurable):
    def getChapterText(self, url):
        "Needs to be overriden in each adapter class."
        pass
-        
+
+    # Just for series, in case we choose to change how it's stored or represented later.
+    def setSeries(self,name,num):
+        if self.getConfig('collect_series'):
+            self.story.setMetadata('series','%s [%s]'%(name, int(num)))
+
+    def setDescription(self,url,svalue):
+        #print("\n\nsvalue:\n%s\n"%svalue)
+        if self.getConfig('keep_summary_html'):
+            if isinstance(svalue,str) or isinstance(svalue,unicode):
+                svalue = bs.BeautifulSoup(svalue)
+            self.story.setMetadata('description',self.utf8FromSoup(url,svalue))
+        else:
+            self.story.setMetadata('description',stripHTML(svalue))
+        #print("\n\ndescription:\n"+self.story.getMetadata('description')+"\n\n")
+
+    # This gives us a unicode object, not just a string containing bytes.
+    # (I gave soup a unicode string, you'd think it could give it back...)
+    # Now also does a bunch of other common processing for us.
+    def utf8FromSoup(self,url,soup,fetch=None):
+        if not fetch:
+            fetch=self._fetchUrlRaw
+
+        acceptable_attributes = ['href','name']
+        #print("include_images:"+self.getConfig('include_images'))
+        if self.getConfig('include_images'):
+            acceptable_attributes.extend(('src','alt','longdesc'))
+            for img in soup.findAll('img'):
+                # some pre-existing epubs have img tags that had src stripped off.
+                if img.has_key('src'):
+                    img['longdesc']=img['src']
+                    img['src']=self.story.addImgUrl(self,url,img['src'],fetch,
+                                                    coverexclusion=self.getConfig('cover_exclusion_regexp'))
+
+        for attr in soup._getAttrMap().keys():
+            if attr not in acceptable_attributes:
+                del soup[attr] ## strip all tag attributes except href and name
+                
+        for t in soup.findAll(recursive=True):
+            for attr in t._getAttrMap().keys():
+                if attr not in acceptable_attributes:
+                    del t[attr] ## strip all tag attributes except href and name
+
+            # these are not acceptable strict XHTML.  But we do already have 
+	    # CSS classes of the same names defined
+            if t.name in ('u'):
+                t['class']=t.name
+                t.name='span'
+            if t.name in ('center'):
+                t['class']=t.name
+                t.name='div'
+	    # removes paired, but empty tags.
+            if t.string != None and len(t.string.strip()) == 0 :
+                t.extract()
+                
+        retval = soup.__str__('utf8').decode('utf-8')
+
+        if self.getConfig('replace_hr'):
+            # replacing a self-closing tag with a container tag in the
+            # soup is more difficult than it first appears.  So cheat.
+            retval = retval.replace("<hr />","<div class='center'>* * *</div>")
+
+        if self.getConfig('nook_img_fix'):
+            # if the <img> tag doesn't have a div or a p around it,
+            # nook gets confused and displays it on every page after
+            # that under the text for the rest of the chapter.
+            retval = re.sub(r"(?!<(div|p)>)\s*(?P<imgtag><img[^>]+>)\s*(?!</(div|p)>)",
+                            "<div>\g<imgtag></div>",retval)
+            
+        # Don't want body tags in chapter html--writers add them.
+        # This is primarily for epub updates.
+        return re.sub(r"</?body>\r?\n?","",retval)
+
+def cachedfetch(realfetch,cache,url):
+    if url in cache:
+        print("cache hit")
+        return cache[url]
+    else:
+        return realfetch(url)
+    
+fullmon = {"January":"01", "February":"02", "March":"03", "April":"04", "May":"05",
+           "June":"06","July":"07", "August":"08", "September":"09", "October":"10",
+           "November":"11", "December":"12" }
+
 def makeDate(string,format):
+    # Surprise!  Abstracting this turned out to be more useful than
+    # just saving bytes.
+
+    # fudge english month names for people who's locale is set to
+    # non-english.  All our current sites date in english, even if
+    # there's non-english content. -- ficbook.net now makes that a
+    # lie.  It has to do something even more complicated to get
+    # Russian month names correct everywhere.
+    do_abbrev = "%b" in format
+        
+    if "%B" in format or do_abbrev:
+        format = format.replace("%B","%m").replace("%b","%m")
+        for (name,num) in fullmon.items():
+            if do_abbrev:
+                name = name[:3] # first three for abbrev
+            if name in string:
+                string = string.replace(name,num)
+                break
+            
    return datetime.datetime.strptime(string,format)

-acceptable_attributes = ['href','name']
-
-# this gives us a unicode object, not just a string containing bytes.
-# (I gave soup a unicode string, you'd think it could give it back...)
-def utf8FromSoup(soup):
-    for t in soup.findAll(recursive=True):
-        for attr in t._getAttrMap().keys():
-            if attr not in acceptable_attributes:
-                del t[attr] ## strip all tag attributes except href and name
-        # these are not acceptable strict XHTML.  But we do already have 
-	# CSS classes of the same names defined in constants.py
-	if t.name in ('u'):
-            t['class']=t.name
-            t.name='span'
-        if t.name in ('center'):
-            t['class']=t.name
-            t.name='div'
-	# removes paired, but empty tags.
-        if t.string != None and len(t.string.strip()) == 0 :
-            t.extract()
-    return soup.__str__('utf8').decode('utf-8')
@@ -21,16 +21,21 @@ import ConfigParser
 # inherit from Configurable.  The config file(s) uses ini format:
 # [sections] with key:value settings.
 #
-# There's a [defaults] section which is overriden by the writer's
-# section [epub], which is overriden by the adapter's section for each
-# site.
+# writer does [defaults], [www.whofic.com], [epub], [www.whofic.com:epub], [overrides]
+#
+# Until a write is created, the adapter only has [defaults], [www.whofic.com], [overrides]
 #
 # [defaults]
 # titlepage_entries: category,genre, status
-# [epub]
-# titlepage_entries: category,genre, status,datePublished,dateUpdated,dateCreated
 # [www.whofic.com]
 # titlepage_entries: category,genre, status,dateUpdated,rating
+# [epub]
+# titlepage_entries: category,genre, status,datePublished,dateUpdated,dateCreated
+# [www.whofic.com:epub]
+# titlepage_entries: category,genre, status,datePublished
+# [overrides]
+# titlepage_entries: category
+

 class Configurable(object):

@@ -38,6 +43,14 @@ class Configurable(object):
        self.config = config
        self.sectionslist = ['defaults']

+    def setSectionOrder(self,site,fileform=None):
+        self.sectionslist = ['defaults']
+        self.addConfigSection(site)
+        if fileform:
+            self.addConfigSection(fileform)
+            self.addConfigSection(site+":"+fileform)
+        self.addConfigSection("overrides")
+
    def addConfigSection(self,section):
        self.sectionslist.insert(0,section)

@@ -0,0 +1,182 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2012, Jim Miller'
+__docformat__ = 'restructuredtext en'
+
+import re, os, traceback
+from zipfile import ZipFile
+from xml.dom.minidom import parseString
+
+from . import BeautifulSoup as bs
+
+def get_dcsource(inputio):
+    return get_update_data(inputio,getfilecount=False,getsoups=False)[0]
+
+def get_dcsource_chaptercount(inputio):
+    return get_update_data(inputio,getfilecount=True,getsoups=False)[:2] # (source,filecount)
+
+def get_update_data(inputio,
+                    getfilecount=True,
+                    getsoups=True):
+    epub = ZipFile(inputio, 'r')
+
+    ## Find the .opf file.
+    container = epub.read("META-INF/container.xml")
+    containerdom = parseString(container)
+    rootfilenodelist = containerdom.getElementsByTagName("rootfile")
+    rootfilename = rootfilenodelist[0].getAttribute("full-path")
+
+    contentdom = parseString(epub.read(rootfilename))
+    firstmetadom = contentdom.getElementsByTagName("metadata")[0]
+    try:
+        source=firstmetadom.getElementsByTagName("dc:source")[0].firstChild.data.encode("utf-8")
+    except:
+        source=None
+
+    ## Save the path to the .opf file--hrefs inside it are relative to it.
+    relpath = get_path_part(rootfilename)
+            
+    oldcover = None
+    calibrebookmark = None
+    # Looking for pre-existing cover.
+    for item in contentdom.getElementsByTagName("reference"):
+        if item.getAttribute("type") == "cover":
+            # there is a cover (x)html file, save the soup for it.
+            href=relpath+item.getAttribute("href")
+            oldcoverhtmlhref = href
+            oldcoverhtmldata = epub.read(href)
+            oldcoverhtmltype = "application/xhtml+xml"
+            for item in contentdom.getElementsByTagName("item"):
+                if( relpath+item.getAttribute("href") == oldcoverhtmlhref ):
+                    oldcoverhtmltype = item.getAttribute("media-type")
+                    break
+            soup = bs.BeautifulSoup(oldcoverhtmldata.decode("utf-8"))
+            src = None
+            # first img or image tag.
+            imgs = soup.findAll('img')
+            if imgs:
+                src = get_path_part(href)+imgs[0]['src']
+            else:
+                imgs = soup.findAll('image')
+                if imgs:
+                    src=get_path_part(href)+imgs[0]['xlink:href']
+
+            if not src:
+                continue
+            try:
+                # remove all .. and the path part above it, if present.
+                # Mostly for epubs edited by Sigil.
+                src = re.sub(r"([^/]+/\.\./)","",src)
+                print("epubutils: found pre-existing cover image:%s"%src)
+                oldcoverimghref = src
+                oldcoverimgdata = epub.read(src)
+                for item in contentdom.getElementsByTagName("item"):
+                    if( relpath+item.getAttribute("href") == oldcoverimghref ):
+                        oldcoverimgtype = item.getAttribute("media-type")
+                        break
+                oldcover = (oldcoverhtmlhref,oldcoverhtmltype,oldcoverhtmldata,oldcoverimghref,oldcoverimgtype,oldcoverimgdata)
+            except Exception as e:
+                print("Cover Image %s not found"%src)
+                print("Exception: %s"%(unicode(e)))
+                traceback.print_exc()
+
+    filecount = 0
+    soups = [] # list of xhmtl blocks
+    images = {} # dict() longdesc->data
+    if getfilecount:
+        # spin through the manifest--only place there are item tags.
+        for item in contentdom.getElementsByTagName("item"):
+            # First, count the 'chapter' files.  FFDL uses file0000.xhtml,
+            # but can also update epubs downloaded from Twisting the
+            # Hellmouth, which uses chapter0.html.
+            if( item.getAttribute("media-type") == "application/xhtml+xml" ):
+                href=relpath+item.getAttribute("href")
+                #print("---- item href:%s path part: %s"%(href,get_path_part(href)))
+                if re.match(r'.*/(file|chapter)\d+\.x?html',href):
+                    if getsoups:
+                        soup = bs.BeautifulSoup(epub.read(href).decode("utf-8"))
+                        for img in soup.findAll('img'):
+                            newsrc=''
+                            longdesc=''
+                            try:
+                                newsrc=get_path_part(href)+img['src']
+                                # remove all .. and the path part above it, if present.
+                                # Mostly for epubs edited by Sigil.
+                                newsrc = re.sub(r"([^/]+/\.\./)","",newsrc)
+                                longdesc=img['longdesc']
+                                data = epub.read(newsrc)
+                                images[longdesc] = data
+                                img['src'] = img['longdesc']
+                            except Exception as e:
+                                print("Image %s not found!\n(originally:%s)"%(newsrc,longdesc))
+                                print("Exception: %s"%(unicode(e)))
+                                traceback.print_exc()
+                        soup = soup.find('body')
+                        # ffdl epubs have chapter title h3
+                        h3 = soup.find('h3')
+                        if h3:
+                            h3.extract()
+                        # TtH epubs have chapter title h2
+                        h2 = soup.find('h2')
+                        if h2:
+                            h2.extract()
+                            
+                        soups.append(soup)
+                        
+                    filecount+=1
+
+    try:
+        calibrebookmark = epub.read("META-INF/calibre_bookmarks.txt")
+    except:
+        pass
+                    
+    for k in images.keys():
+        print("\tlongdesc:%s\n\tData len:%s\n"%(k,len(images[k])))
+    return (source,filecount,soups,images,oldcover,calibrebookmark)
+
+def get_path_part(n):
+    relpath = os.path.dirname(n)
+    if( len(relpath) > 0 ):
+        relpath=relpath+"/"
+    return relpath
+
+def get_story_url_from_html(inputio,_is_good_url=None):
+
+    #print("get_story_url_from_html called")
+    epub = ZipFile(inputio, 'r')
+
+    ## Find the .opf file.
+    container = epub.read("META-INF/container.xml")
+    containerdom = parseString(container)
+    rootfilenodelist = containerdom.getElementsByTagName("rootfile")
+    rootfilename = rootfilenodelist[0].getAttribute("full-path")
+
+    contentdom = parseString(epub.read(rootfilename))
+    #firstmetadom = contentdom.getElementsByTagName("metadata")[0]
+
+    ## Save the path to the .opf file--hrefs inside it are relative to it.
+    relpath = get_path_part(rootfilename)
+            
+    # spin through the manifest--only place there are item tags.
+    for item in contentdom.getElementsByTagName("item"):
+        # First, count the 'chapter' files.  FFDL uses file0000.xhtml,
+        # but can also update epubs downloaded from Twisting the
+        # Hellmouth, which uses chapter0.html.
+        #print("---- item:%s"%item)
+        if( item.getAttribute("media-type") == "application/xhtml+xml" ):
+            filehref=relpath+item.getAttribute("href")
+            soup = bs.BeautifulSoup(epub.read(filehref).decode("utf-8"))
+            for link in soup.findAll('a',href=re.compile(r'^http.*')):
+                ahref=link['href']
+                #print("href:(%s)"%ahref)
+                # hack for bad ficsaver ffnet URLs.
+                m = re.match(r"^http://www.fanfiction.net/s(?P<id>\d+)//$",ahref)
+                if m != None:
+                    ahref="http://www.fanfiction.net/s/%s/1/"%m.group('id')
+                if _is_good_url == None or _is_good_url(ahref):
+                    return ahref
+    return None
@@ -34,12 +34,16 @@ class InvalidStoryURL(Exception):
        return "Bad Story URL: (%s) for site: (%s) Example: (%s)" % (self.url, self.domain, self.example)

 class FailedToLogin(Exception):
-    def __init__(self,url,username):
+    def __init__(self,url, username, passwdonly=False):
        self.url=url
        self.username=username
+        self.passwdonly=passwdonly
        
    def __str__(self):
-        return "Failed to Login for URL: (%s) with username: (%s)" % (self.url, self.username)
+        if self.passwdonly:
+            return "URL Failed, password required: (%s) " % (self.url)
+        else:
+            return "Failed to Login for URL: (%s) with username: (%s)" % (self.url, self.username)

 class AdultCheckRequired(Exception):
    def __init__(self,url):
@@ -59,6 +63,7 @@ class UnknownSite(Exception):
    def __init__(self,url,supported_sites_list):
        self.url=url
        self.supported_sites_list=supported_sites_list
+        self.supported_sites_list.sort()

    def __str__(self):
        return "Unknown Site(%s).  Supported sites: (%s)" % (self.url, ", ".join(self.supported_sites_list))
@@ -0,0 +1,82 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import re
+import urlparse
+import urllib2 as u2
+import ConfigParser
+
+from BeautifulSoup import BeautifulSoup 
+from gziphttp import GZipProcessor
+
+import adapters
+
+def get_urls_from_page(url):
+    
+    opener = u2.build_opener(u2.HTTPCookieProcessor(),GZipProcessor())
+    soup = BeautifulSoup(opener.open(url).read())
+    
+    normalized = set() # normalized url
+    retlist = [] # orig urls.
+    config = ConfigParser.SafeConfigParser()
+    
+    for a in soup.findAll('a'):
+        if a.has_key('href'):
+            href = form_url(url,a['href'])
+            # lots of eFiction sites use similar 'are you old enough' javascript links.
+            if 'javascript' in a['href'] and 'viewstory.php' in a['href']:
+                m = re.search(r"'(?P<sid>(view)?story\.php\?(sid|psid|no|story|stid)=\d+)",a['href'])
+                if m != None:
+                    href = form_url(url,m.group('sid'))
+            try:
+                href = href.replace('&index=1','')
+                adapter = adapters.getAdapter(config,href,"EPUB")
+                if adapter.story.getMetadata('storyUrl') not in normalized:
+                    normalized.add(adapter.story.getMetadata('storyUrl'))
+                    retlist.append(href)
+            except:
+                pass
+
+    return retlist
+
+def form_url(parenturl,url):
+     url = url.strip() # ran across an image with a space in the
+                       # src. Browser handled it, so we'd better, too.
+ 
+     if "//" in url or parenturl == None:
+         returl = url
+     else:
+         parsedUrl = urlparse.urlparse(parenturl)
+         if url.startswith("/") :
+             returl = urlparse.urlunparse(
+                 (parsedUrl.scheme,
+                  parsedUrl.netloc,
+                  url,
+                  '','',''))
+         else:
+             toppath=""
+             if parsedUrl.path.endswith("/"):
+                 toppath = parsedUrl.path
+             else:
+                 toppath = parsedUrl.path[:parsedUrl.path.rindex('/')]
+             returl = urlparse.urlunparse(
+                 (parsedUrl.scheme,
+                  parsedUrl.netloc,
+                  toppath + '/' + url,
+                  '','',''))
+     return returl
+       
@@ -0,0 +1,38 @@
+## Borrowed from http://techknack.net/python-urllib2-handlers/
+
+import urllib2
+from gzip import GzipFile
+from StringIO import StringIO
+
+class GZipProcessor(urllib2.BaseHandler):
+    """A handler to add gzip capabilities to urllib2 requests
+    """
+    def http_request(self, req):
+        req.add_header("Accept-Encoding", "gzip")
+        return req
+    https_request = http_request
+
+    def http_response(self, req, resp):
+        #print("Content-Encoding:%s"%resp.headers.get("Content-Encoding"))
+        if resp.headers.get("Content-Encoding") == "gzip":
+            gz = GzipFile(
+                        fileobj=StringIO(resp.read()),
+                        mode="r"
+                      )
+#            resp.read = gz.read
+#            resp.readlines = gz.readlines
+#            resp.readline = gz.readline
+#            resp.next = gz.next
+            old_resp = resp
+            resp = urllib2.addinfourl(gz, old_resp.headers, old_resp.url, old_resp.code)
+            resp.msg = old_resp.msg
+        return resp
+    https_response = http_response
+
+# brave new world - 1:30 w/o, 1:10 with?  40 chapters, so 20s from sleeps.
+# with gzip, no sleep: 47.469
+# w/o gzip, no sleep: 47.736
+
+# I Am What I Am 67 chapters
+# w/o gzip: 57.168
+# w/ gzip: 40.692
@@ -103,9 +103,10 @@ def onlywhite(line):
            return c is ' '
    return line

-def optwrap(text):
+def optwrap(text,wrap_width=BODY_WIDTH):
    """Wrap all paragraphs in the provided text."""
-    if not BODY_WIDTH:
+    
+    if not wrap_width:
        return text
    
    assert wrap, "Requires Python 2.3."
@@ -114,7 +115,7 @@ def optwrap(text):
    for para in text.split("\n"):
        if len(para) > 0:
            if para[0] is not ' ' and para[0] is not '-' and para[0] is not '*':
-                for line in wrap(para, BODY_WIDTH):
+                for line in wrap(para, wrap_width):
                    result += line + "\n"
                result += "\n"
                newlines = 2
@@ -423,8 +424,8 @@ def html2text_file(html, out=wrapwrite, baseurl=''):
    h.feed("")
    return h.close()

-def html2text(html, baseurl=''):
-    return optwrap(html2text_file(html, None, baseurl))
+def html2text(html, baseurl='', wrap_width=BODY_WIDTH):
+    return optwrap(html2text_file(html, None, baseurl),wrap_width)

 if __name__ == "__main__":
    baseurl = ''
@@ -19,15 +19,31 @@ import re

 def _unirepl(match):
    "Return the unicode string for a decimal number"
-    if match.group(1)=='x':
+    if match.group(1).startswith('x'):
        radix=16
+        s = match.group(1)[1:]
    else:
        radix=10
-    value = int(match.group(2), radix )
-    return unichr(value)
+        s = match.group(1)
+    try:
+        value = int(s, radix)
+        retval = "%s%s"%(unichr(value),match.group(2))
+    except:
+        # This way, at least if there's more of entities out there
+        # that fail, it doesn't blow the entire download.
+        print "Numeric entity translation failed, skipping: &#x%s%s"%(match.group(1),match.group(2))
+        retval = ""
+    return retval

 def _replaceNumberEntities(data):
-    p = re.compile(r'&#(x?)(\d+);')
+    # The same brokenish entity parsing in SGMLParser that inserts ';'
+    # after non-entities will also insert ';' incorrectly after number
+    # entities, including part of the next word if it's a-z.
+    # "Don't&#8212ever&#8212do&#8212that&#8212again," becomes
+    # "Don't&#8212e;ver&#8212d;o&#8212;that&#8212a;gain,"
+    # Also need to allow for 5 digit decimal entities &#27861;
+    # Last expression didn't allow for 2 digit hex correctly: &#xE9;
+    p = re.compile(r'&#(x[0-9a-fA-F]{,4}|[0-9]{,5})([0-9a-fA-F]*?);')
    return p.sub(_unirepl, data)

 def _replaceNotEntities(data):
@@ -50,9 +66,6 @@ def removeAllEntities(text):
    return removeEntities(text).replace('&lt;', '<').replace('&gt;', '>').replace('&amp;', '&')

 def removeEntities(text):
-
-    # replace numeric versions of [&<>] with named versions,
-    # then replace named versions with actual characters,
    
    if text is None:
        return ""
@@ -67,6 +80,8 @@ def removeEntities(text):
        except UnicodeEncodeError, e:
            t = text
    text = t 
+    # replace numeric versions of [&<>] with named versions,
+    # then replace named versions with actual characters,
    text = re.sub(r'&#0*38;','&amp;',text)
    text = re.sub(r'&#0*60;','&lt;',text)
    text = re.sub(r'&#0*62;','&gt;',text)
@@ -15,9 +15,176 @@
 # limitations under the License.
 #

-import os
+import os, re
+import urlparse
+import string
+from math import floor

-from htmlcleanup import conditionalRemoveEntities
+import exceptions
+from htmlcleanup import conditionalRemoveEntities, removeAllEntities
+
+# Create convert_image method depending on which graphics lib we can
+# load.  Preferred: calibre, PIL, none
+try:
+    from calibre.utils.magick import Image
+
+    def convert_image(url,data,sizes,grayscale):
+        export = False
+        img = Image()
+        img.load(data)
+        
+        owidth, oheight = img.size
+        nwidth, nheight = sizes
+        scaled, nwidth, nheight = fit_image(owidth, oheight, nwidth, nheight)
+        if scaled:
+            img.size = (nwidth, nheight)
+            export = True
+            
+        if grayscale and img.type != "GrayscaleType":
+            img.type = "GrayscaleType"
+            export = True
+
+        if normalize_format_name(img.format) != "jpg":
+            export = True
+
+        if export:
+            return (img.export('JPG'),'jpg','image/jpeg')
+        else:
+            print("image used unchanged")
+            return (data,'jpg','image/jpeg')
+        
+except:
+
+    # No calibre routines, try for PIL for CLI.
+    try:
+        import Image
+        from StringIO import StringIO
+        def convert_image(url,data,sizes,grayscale):
+            
+            export = False
+            img = Image.open(StringIO(data))
+            
+            owidth, oheight = img.size
+            nwidth, nheight = sizes
+            scaled, nwidth, nheight = fit_image(owidth, oheight, nwidth, nheight)
+            if scaled:
+                img = img.resize((nwidth, nheight),Image.ANTIALIAS)
+                export = True
+                
+            if grayscale and img.mode != "L":
+                img = img.convert("L")
+                export = True
+
+            if normalize_format_name(img.format) != "jpg":
+                if img.mode == "P":
+                    # convert pallete gifs to RGB so jpg save doesn't fail.
+                    img = img.convert("RGB")
+                export = True
+
+            if export:
+                outsio = StringIO()
+                img.save(outsio,'JPEG')
+                return (outsio.getvalue(),'jpg','image/jpeg')
+            else:
+                print("image used unchanged")
+                return (data,'jpg','image/jpeg')
+        
+    except:
+
+        # No calibre or PIL, simple pass through with mimetype.
+        imagetypes = {
+            'jpg':'image/jpeg',
+            'jpeg':'image/jpeg',
+            'png':'image/png',
+            'gif':'image/gif',
+            'svg':'image/svg+xml',
+            }
+
+        def convert_image(url,data,sizes,grayscale):
+            ext=url[url.rfind('.')+1:].lower()
+            return (data,ext,imagetypes[ext])
+        
+def normalize_format_name(fmt):
+    if fmt:
+        fmt = fmt.lower()
+        if fmt == 'jpeg':
+            fmt = 'jpg'
+    return fmt
+
+def fit_image(width, height, pwidth, pheight):
+    '''
+    Fit image in box of width pwidth and height pheight.
+    @param width: Width of image
+    @param height: Height of image
+    @param pwidth: Width of box
+    @param pheight: Height of box
+    @return: scaled, new_width, new_height. scaled is True iff new_width and/or new_height is different from width or height.
+    '''
+    scaled = height > pheight or width > pwidth
+    if height > pheight:
+        corrf = pheight/float(height)
+        width, height = floor(corrf*width), pheight
+    if width > pwidth:
+        corrf = pwidth/float(width)
+        width, height = pwidth, floor(corrf*height)
+    if height > pheight:
+        corrf = pheight/float(height)
+        width, height = floor(corrf*width), pheight
+
+    return scaled, int(width), int(height)
+
+try:
+    # doesn't really matter what, just checking for appengine.
+    from google.appengine.api import apiproxy_stub_map
+
+    is_appengine = True
+except:
+    is_appengine = False
+
+
+# The list comes from ffnet, the only multi-language site we support
+# at the time of writing.  Values are taken largely from pycountry,
+# but with some corrections and guesses.
+langs = {
+    "English":"en",
+    "Spanish":"es",
+    "French":"fr",
+    "German":"de",
+    "Chinese":"zh",
+    "Japanese":"ja",
+    "Dutch":"nl",
+    "Portuguese":"pt",
+    "Russian":"ru",
+    "Italian":"it",
+    "Bulgarian":"bg",
+    "Polish":"pl",
+    "Hungarian":"hu",
+    "Hebrew":"he",
+    "Arabic":"ar",
+    "Swedish":"sv",
+    "Norwegian":"no",
+    "Danish":"da",
+    "Finnish":"fi",
+    "Filipino":"fil",
+    "Esperanto":"eo",
+    "Hindi":"hi",
+    "Punjabi":"pa",
+    "Farsi":"fa",
+    "Greek":"el",
+    "Romanian":"ro",
+    "Albanian":"sq",
+    "Serbian":"sr",
+    "Turkish":"tr",
+    "Czech":"cs",
+    "Indonesian":"id",
+    "Croatian":"hr",
+    "Catalan":"ca",
+    "Latin":"la",
+    "Korean":"ko",
+    "Vietnamese":"vi",
+    "Thai":"th",
+    "Devanagari":"hi",
+    }

 class Story:
    
@@ -25,20 +192,39 @@ class Story:
        try:
            self.metadata = {'version':os.environ['CURRENT_VERSION_ID']}
        except:
-            self.metadata = {'version':'4.0'}
+            self.metadata = {'version':'4.4'}
+        self.replacements = []
        self.chapters = [] # chapters will be tuples of (title,html)
+        self.imgurls = []
+        self.imgtuples = []
        self.listables = {} # some items (extratags, category, warnings & genres) are also kept as lists.
+        self.cover=None # *href* of new cover image--need to create html.
+        self.oldcover=None # (oldcoverhtmlhref,oldcoverhtmltype,oldcoverhtmldata,oldcoverimghref,oldcoverimgtype,oldcoverimgdata)
+        self.calibrebookmark=None # cheesy way to carry calibre bookmark file forward across update.

    def setMetadata(self, key, value):
+        ## still keeps &lt; &lt; and &amp;
        self.metadata[key]=conditionalRemoveEntities(value)
+        if key == "language":
+            try:
+                self.metadata['langcode'] = langs[self.metadata[key]]
+            except:
+                self.metadata['langcode'] = 'en'

    def getMetadataRaw(self,key):
        if self.metadata.has_key(key):
            return self.metadata[key]
+
+    def doReplacments(self,value):
+        for (p,v) in self.replacements:
+            if (isinstance(value,str) or isinstance(value,unicode)) and re.match(p,value):
+                value = re.sub(p,v,value)                
+        return value
        
-    def getMetadata(self, key):
+    def getMetadata(self, key, removeallentities=False, doreplacements=True):
+        value = None
        if self.getLists().has_key(key):
-            return ', '.join(self.getList(key))
+            value = ', '.join(self.getList(key))
        if self.metadata.has_key(key):
            value = self.metadata[key]
            if value:
@@ -48,24 +234,52 @@ class Story:
                    value = value.strftime("%Y-%m-%d %H:%M:%S")
                if key == "datePublished" or key == "dateUpdated":
                 value = value.strftime("%Y-%m-%d")
+
+        if doreplacements:
+            value=self.doReplacments(value)
+        if removeallentities and value != None:
+            return removeAllEntities(value)
+        else:
            return value

+    def getAllMetadata(self, removeallentities=False, doreplacements=True):
+        '''
+        All single value *and* list value metadata as strings.
+        '''
+        allmetadata = {}
+        for k in self.metadata.keys():
+            allmetadata[k] = self.getMetadata(k, removeallentities, doreplacements)
+        for l in self.listables.keys():
+            allmetadata[l] = self.getMetadata(l, removeallentities, doreplacements)
+
+        return allmetadata
+
+    # just for less clutter in adapters.
+    def extendList(self,listname,l):
+        for v in l:
+            self.addToList(listname,v)
+    
    def addToList(self,listname,value):
        if value==None:
            return
+        value = conditionalRemoveEntities(value)
        if not self.listables.has_key(listname):
            self.listables[listname]=[]
        # prevent duplicates.
        if not value in self.listables[listname]:
-            self.listables[listname].append(conditionalRemoveEntities(value))
+            self.listables[listname].append(value)

    def getList(self,listname):
        if not self.listables.has_key(listname):
            return []
-        return self.listables[listname]
+        return filter( lambda x : x!=None and x!='' ,
+                       map(self.doReplacments,self.listables[listname]) )

    def getLists(self):
-        return self.listables
+        lsts = {}
+        for ln in self.listables.keys():
+            lsts[ln] = self.getList(ln)
+        return lsts
    
    def addChapter(self, title, html):
        self.chapters.append( (title,html) )
@@ -73,10 +287,132 @@ class Story:
    def getChapters(self):
        "Chapters will be tuples of (title,html)"
        return self.chapters
+
+    def formatFileName(self,template,allowunsafefilename=True):
+        values = origvalues = self.getAllMetadata()
+        # fall back default:
+        if not template:
+            template="${title}-${siteabbrev}_${storyId}${formatext}"
+
+        if not allowunsafefilename:
+            values={}
+            pattern = re.compile(r"[^a-zA-Z0-9_\. \[\]\(\)&'-]+")
+            for k in origvalues.keys():
+                values[k]=re.sub(pattern,'_', removeAllEntities(self.getMetadata(k)))
+
+        return string.Template(template).substitute(values).encode('utf8')
+
+    # pass fetch in from adapter in case we need the cookies collected
+    # as well as it's a base_story class method.
+    def addImgUrl(self,configurable,parenturl,url,fetch,cover=False,coverexclusion=None):
+
+        # otherwise it saves the image in the epub even though it
+        # isn't used anywhere.
+        if cover and configurable.getConfig('never_make_cover'):
+            return
+        
+        url = url.strip() # ran across an image with a space in the
+                          # src. Browser handled it, so we'd better, too.
+
+        # appengine (web version) isn't allowed to do images--just
+        # gets too big too fast and breaks things.
+        if is_appengine:
+            return
+        
+        if url.startswith("http") or url.startswith("file") or parenturl == None:
+            imgurl = url
+        else:
+            parsedUrl = urlparse.urlparse(parenturl)
+            if url.startswith("/") :
+                imgurl = urlparse.urlunparse(
+                    (parsedUrl.scheme,
+                     parsedUrl.netloc,
+                     url,
+                     '','',''))
+            else:
+                toppath=""
+                if parsedUrl.path.endswith("/"):
+                    toppath = parsedUrl.path
+                else:
+                    toppath = parsedUrl.path[:parsedUrl.path.rindex('/')]
+                imgurl = urlparse.urlunparse(
+                    (parsedUrl.scheme,
+                     parsedUrl.netloc,
+                     toppath + '/' + url,
+                     '','',''))
+                #print("\n===========\nparsedUrl.path:%s\ntoppath:%s\nimgurl:%s\n\n"%(parsedUrl.path,toppath,imgurl))
+
+        prefix='ffdl'
+        if imgurl not in self.imgurls:
+            parsedUrl = urlparse.urlparse(imgurl)
+            try:
+                sizes = [ int(x) for x in configurable.getConfigList('image_max_size') ]
+            except Exception, e:
+                raise exceptions.FailedToDownload("Failed to parse image_max_size from personal.ini:%s\nException: %s"%(configurable.getConfigList('image_max_size'),e))
+            try:
+                (data,ext,mime) = convert_image(imgurl,
+                                                fetch(imgurl),
+                                                sizes,
+                                                configurable.getConfig('grayscale_images'))
+            except Exception, e:
+                print("Failed to load or convert image, skipping:\n%s\nException: %s"%(imgurl,e))
+                return "failedtoload"
+            
+            # explicit cover, make the first image.
+            if cover and not configurable.getConfig('never_make_cover'):
+                if len(self.imgtuples) > 0 and 'cover' in self.imgtuples[0]['newsrc']:
+                    # remove existing cover, if there is one.
+                    del self.imgurls[0]
+                    del self.imgtuples[0]
+                self.imgurls.insert(0,imgurl)
+                newsrc = "images/cover.%s"%ext
+                self.cover=newsrc
+                self.imgtuples.insert(0,{'newsrc':newsrc,'mime':mime,'data':data})
+            else:
+                self.imgurls.append(imgurl)
+                # First image, copy not link because calibre will replace with it's cover.
+                # Only if: No cover already AND
+                #          make_firstimage_cover AND
+                #          NOT never_make_cover AND
+                #          either no coverexclusion OR coverexclusion doesn't match
+                if self.cover == None and \
+                        configurable.getConfig('make_firstimage_cover') and \
+                        not configurable.getConfig('never_make_cover') and \
+                        (not coverexclusion or not re.search(coverexclusion,imgurl)):
+                    newsrc = "images/cover.%s"%ext
+                    self.cover=newsrc
+                    self.imgtuples.append({'newsrc':newsrc,'mime':mime,'data':data})
+                    self.imgurls.append(imgurl)
+            
+                newsrc = "images/%s-%s.%s"%(
+                    prefix,
+                    self.imgurls.index(imgurl),
+                    ext)
+                self.imgtuples.append({'newsrc':newsrc,'mime':mime,'data':data})
+                
+            print("\nimgurl:%s\nnewsrc:%s\nimage size:%d\n"%(imgurl,newsrc,len(data)))
+        else:
+            newsrc = self.imgtuples[self.imgurls.index(imgurl)]['newsrc']
+            
+        #print("===============\n%s\nimg url:%s\n============"%(newsrc,self.imgurls[-1]))
+        
+        return newsrc
+
+    def getImgUrls(self):
+        retlist = []
+        for i, url in enumerate(self.imgurls):
+            #parsedUrl = urlparse.urlparse(url)
+            retlist.append(self.imgtuples[i])
+        return retlist
    
    def __str__(self):
        return "Metadata: " +str(self.metadata) + "\nListables: " +str(self.listables) #+ "\nChapters: "+str(self.chapters)

+    def setReplace(self,replace):
+        for line in replace.splitlines():
+            if "=>" in line:
+                self.replacements.append(map( lambda x: x.strip(), line.split("=>") ))
+    
 def commaGroups(s):
    groups = []
    while s and s[-1].isdigit():
@@ -0,0 +1,57 @@
+#-*-coding:utf-8-*-
+# Code taken from http://python.su/forum/viewtopic.php?pid=66946
+import unicodedata
+def is_syllable(letter):
+    syllables = ("A", "E", "I", "O", "U", "a", "e", "i", "o", "u")
+    if letter in syllables:
+        return True
+    return False
+def is_consonant(letter):
+    return not is_syllable(letter)
+def romanize(letter):
+    try:
+        str(letter)
+    except UnicodeEncodeError:
+        pass
+    else:
+        return str(letter)
+    unid = unicodedata.name(letter)
+    exceptions = {"NUMERO SIGN": "No", "LEFT-POINTING DOUBLE ANGLE QUOTATION MARK": "\"", "RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK": "\"", "DASH": "-"}
+    for name_contains in exceptions:
+        if unid.find(name_contains)!=-1:
+            return exceptions[name_contains]
+    assert(unid.startswith("CYRILLIC"))# Not ready to romanize anything but cyrillics
+    transformation_pairs = {"CYRILLIC CAPITAL LETTER ": str.capitalize, "CYRILLIC SMALL LETTER ": str.lower}
+    func = str.lower
+    for name_contains in transformation_pairs:
+        if unid.find(name_contains)!=-1:
+            func = transformation_pairs[name_contains]
+            unid = unid.replace(name_contains, "")
+    cyrillic_exceptions = {"YERU": "y", "SHORT I": "y", "HARD SIGN": "\'", "SOFT SIGN": "\'", "BYELORUSSIAN-UKRAINIAN I": "i", "GHE WITH UPTURN": "g", "UKRAINIAN IE": "ie", "YU": "yu", "YA": "ya"}
+    for name_contains in cyrillic_exceptions:
+        if unid.find(name_contains)!=-1:
+            return cyrillic_exceptions[name_contains]
+    if all(map(is_syllable, unid)):
+        return func(unid)
+    else:
+        return func(filter(is_consonant, unid))
+def translit(text):
+    output = ""
+    for letter in text:
+        output += romanize(letter)
+    return output
+#def main():
+    #text = u"русск.: Любя, съешь щипцы, — вздохнёт мэр, — кайф жгуч."
+    #print translit(text)
+    #text = u"укр.: Гей, хлопці, не вспію - на ґанку ваша файна їжа знищується бурундучком."
+    #print translit(text)
+    #text = u"болг.: Ах, чудна българска земьо, полюшквай цъфтящи жита."
+    #print translit(text)
+    #text = u"серб.: Неуредне ноћне даме досађивале су Џеку К."
+    #print translit(text)
+    #russk.: Lyubya, s'iesh' shchiptsy, - vzdohniot mer, - kayf zhghuch.
+    #ukr.: Ghiey, hloptsi, nie vspiyu - na ganku vasha fayna yzha znishchuiet'sya burunduchkom.
+    #bolgh.: Ah, chudna b'lgharska ziem'o, polyushkvay ts'ftyashchi zhita.
+    #sierb.: Nieuriednie notshnie damie dosadjivalie su Dzhieku K.
+if __name__=="__main__":
+    main()
@@ -18,7 +18,7 @@
 ## This could (should?) use a dynamic loader like adapters, but for
 ## now, it's static, since there's so few of them.

-from fanficdownloader.exceptions import FailedToDownload
+from ..exceptions import FailedToDownload

 from writer_html import HTMLWriter
 from writer_txt  import TextWriter
@@ -18,14 +18,13 @@
 import re
 import os.path
 import datetime
-import string
 import StringIO
 import zipfile
 from zipfile import ZipFile, ZIP_DEFLATED
 import logging

-from fanficdownloader.configurable import Configurable
-from fanficdownloader.htmlcleanup import removeEntities, removeAllEntities, stripHTML
+from ..configurable import Configurable
+from ..htmlcleanup import removeEntities, removeAllEntities, stripHTML

 class BaseStoryWriter(Configurable):

@@ -39,17 +38,19 @@ class BaseStoryWriter(Configurable):

    def __init__(self, config, adapter):
        Configurable.__init__(self, config)
-        self.addConfigSection(adapter.getSiteDomain())
-        self.addConfigSection(self.getFormatName())
-        self.addConfigSection(adapter.getSiteDomain()+":"+self.getFormatName())
-        self.addConfigSection("overrides")
+        self.setSectionOrder(adapter.getConfigSection(),self.getFormatName())
        
        self.adapter = adapter
        self.story = adapter.getStoryMetadataOnly() # only cache the metadata initially.
+        
+        self.story.setReplace(self.getConfig('replace_metadata'))
+        
        self.validEntries = [
            'category',
            'genre',
+            'language',
            'characters',
+            'series',
            'status',
            'datePublished',
            'dateUpdated',
@@ -76,7 +77,9 @@ class BaseStoryWriter(Configurable):
        self.titleLabels = {
            'category':'Category',
            'genre':'Genre',
+            'language':'Language',
            'status':'Status',
+            'series':'Series',
            'characters':'Characters',
            'datePublished':'Published',
            'dateUpdated':'Updated',
@@ -97,10 +100,16 @@ class BaseStoryWriter(Configurable):
            'formatname':'File Format',
            'formatext':'File Extension',
            'siteabbrev':'Site Abbrev',
-            'version':'FFD Version'
+            'version':'FFDL Version'
            }
        self.story.setMetadata('formatname',self.getFormatName())
        self.story.setMetadata('formatext',self.getFormatExt())
+        
+        for tag in self.getConfigList("extratags"):
+            self.story.addToList("extratags",tag)
+
+    def getMetadata(self,key):
+        return stripHTML(self.story.getMetadata(key))

    def getOutputFileName(self):
        if self.getConfig('zip_output'):
@@ -109,29 +118,15 @@ class BaseStoryWriter(Configurable):
            return self.getBaseFileName()

    def getBaseFileName(self):
-        return self.formatFileName(self.getConfig('output_filename'))
+        return self.story.formatFileName(self.getConfig('output_filename'),self.getConfig('allow_unsafe_filename'))
    
    def getZipFileName(self):
-        return self.formatFileName(self.getConfig('zip_filename'))
-
-    def formatFileName(self,template):
-        values = self.story.metadata
-        # fall back default:
-        if not template:
-            template="${title}-${siteabbrev}_${storyId}${formatext}"
-
-        if not self.getConfig('allow_unsafe_filename'):
-            values={}
-            pattern = re.compile(r"[^a-zA-Z0-9_\. \[\]\(\)&'-]+")
-            for k in self.story.metadata.keys():
-                values[k]=re.sub(pattern,'_', removeAllEntities(self.story.getMetadata(k)))
-
-        return string.Template(template).substitute(values).encode('utf8')
+        return self.story.formatFileName(self.getConfig('zip_filename'),self.getConfig('allow_unsafe_filename'))

    def _write(self, out, text):
        out.write(text.encode('utf8'))

-    def writeTitlePage(self, out, START, ENTRY, END, WIDE_ENTRY=None):
+    def writeTitlePage(self, out, START, ENTRY, END, WIDE_ENTRY=None, NO_TITLE_ENTRY=None):
        """
        Write the title page, but only include entries that there's
        metadata for.  START, ENTRY and END are expected to already by
@@ -158,6 +153,12 @@ class BaseStoryWriter(Configurable):
                            label=self.getConfig(entry+"_label")
                        else:
                            label=self.titleLabels[entry]
+
+                        # If the label for the title entry is empty, use the
+                        # 'no title' option if there is one.  
+                        if label == "" and NO_TITLE_ENTRY:
+                           TEMPLATE= NO_TITLE_ENTRY
+                           
                        self._write(out,TEMPLATE.substitute({'label':label,
                                                             'value':self.story.getMetadata(entry)}))

@@ -180,16 +181,21 @@ class BaseStoryWriter(Configurable):
            self._write(out,END.substitute(self.story.metadata))

    # if no outstream is given, write to file.
-    def writeStory(self,outstream=None,metaonly=False):
-        for tag in self.getConfigList("extratags"):
-            self.story.addToList("extratags",tag)
+    def writeStory(self,outstream=None, metaonly=False, outfilename=None, forceOverwrite=False):

        self.metaonly = metaonly
-        outfilename=self.getOutputFileName()
+        if outfilename == None:
+            outfilename=self.getOutputFileName()

+        # minor cheat, tucking css into metadata.
+        if self.getConfig("output_css"):
+            self.story.metadata["output_css"] = self.getConfig("output_css")
+        else:
+            self.story.metadata["output_css"] = ''
+            
        if not outstream:
            close=True
-            logging.debug("Save directly to file: %s" % outfilename)
+            logging.info("Save directly to file: %s" % outfilename)
            if self.getConfig('make_directories'):
                path=""
                dirs = os.path.dirname(outfilename).split('/')
@@ -199,7 +205,7 @@ class BaseStoryWriter(Configurable):
                        os.mkdir(path) ## os.makedirs() doesn't work in 2.5.2?

            ## Check for output file date vs updated date here
-            if not self.getConfig('always_overwrite'):
+            if not (self.getConfig('always_overwrite') or forceOverwrite):
                if os.path.exists(outfilename):
                    ## date() truncs off time, which files have, but sites don't report.
                    lastupdated=self.story.getMetadataRaw('dateUpdated').date()
@@ -242,6 +248,29 @@ class BaseStoryWriter(Configurable):
        if close:
            outstream.close()

+    def getTags(self):
+        # set to avoid duplicates subject tags.
+        subjectset = set()
+        
+        if self.story.getMetadataRaw('dateUpdated'):
+            # Last Update tags for Bill.
+            self.story.addToList('lastupdate',self.story.getMetadataRaw('dateUpdated').strftime("Last Update Year/Month: %Y/%m"))
+            self.story.addToList('lastupdate',self.story.getMetadataRaw('dateUpdated').strftime("Last Update: %Y/%m/%d"))
+        
+        for entry in self.validEntries:
+            if entry in self.getConfigList("include_subject_tags") and \
+                    entry not in self.story.getLists() and \
+                    self.story.getMetadata(entry):
+                subjectset.add(self.getMetadata(entry))
+                
+        # listables all go into dc:subject tags, but only if they are configured.
+        for (name,lst) in self.story.getLists().iteritems():
+            if name in self.getConfigList("include_subject_tags"):
+                for tag in lst:
+                    subjectset.add(tag)
+
+        return list(subjectset)
+            
    def writeStoryImpl(self, out):
        "Must be overriden by sub classes."
        pass
@@ -20,13 +20,14 @@ import string
 import StringIO
 import zipfile
 from zipfile import ZipFile, ZIP_STORED, ZIP_DEFLATED
+import urllib

 ## XML isn't as forgiving as HTML, so rather than generate as strings,
 ## use DOM to generate the XML files.
 from xml.dom.minidom import parse, parseString, getDOMImplementation

 from base_writer import *
-from fanficdownloader.htmlcleanup import stripHTML
+from ..htmlcleanup import stripHTML

 class EpubWriter(BaseStoryWriter):

@@ -41,29 +42,7 @@ class EpubWriter(BaseStoryWriter):
    def __init__(self, config, story):
        BaseStoryWriter.__init__(self, config, story)

-        self.EPUB_CSS='''body { margin-left: 2%; margin-right: 2%; margin-top: 2%; margin-bottom: 2%; text-align: justify; }
-pre { font-size: x-small; }
-sml { font-size: small; }
-h1 { text-align: center; }
-h2 { text-align: center; }
-h3 { text-align: center; }
-h4 { text-align: center; }
-h5 { text-align: center; }
-h6 { text-align: center; }
-.CI {
-    text-align:center;
-    margin-top:0px;
-    margin-bottom:0px;
-    padding:0px;
-    }
-.center   {text-align: center;}
-.cover    {text-align: center;}
-.full     {width: 100%; }
-.quarter  {width: 25%; }
-.smcap    {font-variant: small-caps;}
-.u        {text-decoration: underline;}
-.bold     {font-weight: bold;}
-'''
+        self.EPUB_CSS = string.Template('''${output_css}''')
        
        self.EPUB_TITLE_PAGE_START = string.Template('''<?xml version="1.0" encoding="UTF-8"?>
 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
@@ -79,6 +58,10 @@ h6 { text-align: center; }

        self.EPUB_TITLE_ENTRY = string.Template('''
 <b>${label}:</b> ${value}<br />
+''')
+
+        self.EPUB_NO_TITLE_ENTRY = string.Template('''
+${value}<br />
 ''')

        self.EPUB_TITLE_PAGE_END = string.Template('''
@@ -106,6 +89,10 @@ h6 { text-align: center; }

        self.EPUB_TABLE_TITLE_WIDE_ENTRY = string.Template('''
 <tr><td colspan="2"><b>${label}:</b> ${value}</td></tr>
+''')
+
+        self.EPUB_TABLE_NO_TITLE_ENTRY = string.Template('''
+<tr><td colspan="2">${label}${value}</td></tr>
 ''')

        self.EPUB_TABLE_TITLE_PAGE_END = string.Template('''
@@ -153,9 +140,6 @@ h6 { text-align: center; }
 </html>
 ''')

-    def getMetadata(self,key):
-        return stripHTML(self.story.getMetadata(key))
-
    def writeStoryImpl(self, out):

        ## Python 2.5 ZipFile is rather more primative than later
@@ -228,7 +212,10 @@ h6 { text-align: center; }

        metadata.appendChild(newTag(contentdom,"dc:contributor",text="fanficdownloader [http://fanficdownloader.googlecode.com]",attrs={"opf:role":"bkp"}))
        metadata.appendChild(newTag(contentdom,"dc:rights",text=""))
-        metadata.appendChild(newTag(contentdom,"dc:language",text="en"))
+        if self.story.getMetadata('langcode') != None:
+            metadata.appendChild(newTag(contentdom,"dc:language",text=self.story.getMetadata('langcode')))
+        else:
+            metadata.appendChild(newTag(contentdom,"dc:language",text='en'))

        #  published, created, updated, calibre
        #  Leave calling self.story.getMetadataRaw directly in case date format changes.
@@ -249,27 +236,12 @@ h6 { text-align: center; }
            metadata.appendChild(newTag(contentdom,"meta",
                                        attrs={"name":"calibre:timestamp",
                                               "content":self.story.getMetadataRaw('dateUpdated').strftime("%Y-%m-%dT%H:%M:%S")}))
-            # Last Update tags for Bill.
-            self.story.addToList('lastupdate',self.story.getMetadataRaw('dateUpdated').strftime("Last Update Year/Month: %Y/%m"))
-            self.story.addToList('lastupdate',self.story.getMetadataRaw('dateUpdated').strftime("Last Update: %Y/%m/%d"))
        
        if self.getMetadata('description'):
            metadata.appendChild(newTag(contentdom,"dc:description",text=
                                        self.getMetadata('description')))

-        # set to avoid duplicates subject tags.
-        subjectset = set()
-        for entry in self.validEntries:
-            if entry in self.getConfigList("include_subject_tags") and \
-                    entry not in self.story.getLists() and \
-                    self.story.getMetadata(entry):
-                subjectset.add(self.getMetadata(entry))
-        # listables all go into dc:subject tags, but only if they are configured.
-        for (name,lst) in self.story.getLists().iteritems():
-            if name in self.getConfigList("include_subject_tags"):
-                for tag in lst:
-                    subjectset.add(tag)
-        for subject in subjectset:
+        for subject in self.getTags():
            metadata.appendChild(newTag(contentdom,"dc:subject",text=subject))

                    
@@ -289,7 +261,82 @@ h6 { text-align: center; }
        itemrefs = [] # list of strings -- idrefs from .opfs' spines
        items.append(("ncx","toc.ncx","application/x-dtbncx+xml",None)) ## we'll generate the toc.ncx file,
                                                                   ## but it needs to be in the items manifest.
+
+        guide = None
+        coverIO = None
+                
+        imgid = "image0000"
+        if not self.story.cover and self.story.oldcover:
+            print("writer_epub: no new cover, has old cover, write image.")
+            (oldcoverhtmlhref,
+             oldcoverhtmltype,
+             oldcoverhtmldata,
+             oldcoverimghref,
+             oldcoverimgtype,
+             oldcoverimgdata) = self.story.oldcover
+            outputepub.writestr(oldcoverhtmlhref,oldcoverhtmldata)
+            outputepub.writestr(oldcoverimghref,oldcoverimgdata)
+            
+            imgid = "image0"
+            items.append((imgid,
+                          oldcoverimghref,
+                          oldcoverimgtype,
+                          None))
+            items.append(("cover",oldcoverhtmlhref,oldcoverhtmltype,None))
+            itemrefs.append("cover")
+            metadata.appendChild(newTag(contentdom,"meta",{"content":"image0",
+                                                           "name":"cover"}))
+            guide = newTag(contentdom,"guide")
+            guide.appendChild(newTag(contentdom,"reference",attrs={"type":"cover",
+                                                                   "title":"Cover",
+                                                                   "href":oldcoverhtmlhref}))
+            
+            
+
+        if self.getConfig('include_images'):
+            imgcount=0
+            for imgmap in self.story.getImgUrls():
+                imgfile = "OEBPS/"+imgmap['newsrc']
+                outputepub.writestr(imgfile,imgmap['data'])
+                items.append(("image%04d"%imgcount,
+                              imgfile,
+                              imgmap['mime'],
+                              None))
+                imgcount+=1
+
+        
        items.append(("style","OEBPS/stylesheet.css","text/css",None))
+
+        if self.story.cover:
+            # Note that the id of the cover xhmtl *must* be 'cover'
+            # for it to work on Nook.
+            items.append(("cover","OEBPS/cover.xhtml","application/xhtml+xml",None))
+            itemrefs.append("cover")
+            # 
+            # <meta name="cover" content="cover.jpg"/>
+            metadata.appendChild(newTag(contentdom,"meta",{"content":"image0000",
+                                                           "name":"cover"}))
+            # cover stuff for later:
+            # at end of <package>:
+            # <guide>
+            # <reference type="cover" title="Cover" href="Text/cover.xhtml"/>
+            # </guide>
+            guide = newTag(contentdom,"guide")
+            guide.appendChild(newTag(contentdom,"reference",attrs={"type":"cover",
+                                                       "title":"Cover",
+                                                       "href":"OEBPS/cover.xhtml"}))
+            
+            coverIO = StringIO.StringIO()
+            coverIO.write('''
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en"><head><title>Cover</title><style type="text/css" title="override_css">
+@page {padding: 0pt; margin:0pt}
+body { text-align: center; padding:0pt; margin: 0pt; }
+div { margin: 0pt; padding: 0pt; }
+</style></head><body><div>
+<img src="%s" alt="cover"/>
+</div></body></html>
+'''%self.story.cover)
+            
        if self.getConfig("include_titlepage"):
            items.append(("title_page","OEBPS/title_page.xhtml","application/xhtml+xml","Title Page"))
            itemrefs.append("title_page")
@@ -320,8 +367,18 @@ h6 { text-align: center; }
            spine.appendChild(newTag(contentdom,"itemref",
                                     attrs={"idref":itemref,
                                            "linear":"yes"}))
+        # guide only exists if there's a cover.
+        if guide:
+            package.appendChild(guide)
+            
        # write content.opf to zip.
-        outputepub.writestr("content.opf",contentdom.toxml(encoding='utf-8'))
+        contentxml = contentdom.toxml(encoding='utf-8')
+        
+        # tweak for brain damaged Nook STR.  Nook insists on name before content.
+        contentxml = contentxml.replace('<meta content="%s" name="cover"/>'%imgid,
+                                        '<meta name="cover" content="%s"/>'%imgid)
+        outputepub.writestr("content.opf",contentxml)
+
        contentdom.unlink()
        del contentdom

@@ -357,7 +414,7 @@ h6 { text-align: center; }
        index=0
        for item in items:
            (id,href,type,title)=item
-            # only items to be skipped, toc.ncx, stylesheet.css, should have no title.
+            # only items to be skipped, cover.xhtml, images, toc.ncx, stylesheet.css, should have no title.
            if title :
                navPoint = newTag(tocncxdom,"navPoint",
                                  attrs={'id':id,
@@ -370,32 +427,39 @@ h6 { text-align: center; }
                navPoint.appendChild(newTag(tocncxdom,"content",attrs={"src":href}))
                index=index+1
        
-        # write toc.ncs to zip file
+        # write toc.ncx to zip file
        outputepub.writestr("toc.ncx",tocncxdom.toxml(encoding='utf-8'))
        tocncxdom.unlink()
        del tocncxdom

        # write stylesheet.css file.
-        outputepub.writestr("OEBPS/stylesheet.css",self.EPUB_CSS)
+        outputepub.writestr("OEBPS/stylesheet.css",self.EPUB_CSS.substitute(self.story.metadata)) 

        # write title page.
        if self.getConfig("titlepage_use_table"):
            TITLE_PAGE_START  = self.EPUB_TABLE_TITLE_PAGE_START
            TITLE_ENTRY       = self.EPUB_TABLE_TITLE_ENTRY
            WIDE_TITLE_ENTRY  = self.EPUB_TABLE_TITLE_WIDE_ENTRY
+            NO_TITLE_ENTRY    = self.EPUB_TABLE_NO_TITLE_ENTRY
            TITLE_PAGE_END    = self.EPUB_TABLE_TITLE_PAGE_END
        else:
            TITLE_PAGE_START  = self.EPUB_TITLE_PAGE_START
            TITLE_ENTRY       = self.EPUB_TITLE_ENTRY
            WIDE_TITLE_ENTRY  = self.EPUB_TITLE_ENTRY # same, only wide in tables.
+            NO_TITLE_ENTRY    = self.EPUB_NO_TITLE_ENTRY
            TITLE_PAGE_END    = self.EPUB_TITLE_PAGE_END
-        
+
+        if coverIO:
+            outputepub.writestr("OEBPS/cover.xhtml",coverIO.getvalue())
+            coverIO.close()
+            
        titlepageIO = StringIO.StringIO()
        self.writeTitlePage(out=titlepageIO,
                            START=TITLE_PAGE_START,
                            ENTRY=TITLE_ENTRY,
                            WIDE_ENTRY=WIDE_TITLE_ENTRY,
-                            END=TITLE_PAGE_END)
+                            END=TITLE_PAGE_END,
+                            NO_TITLE_ENTRY=NO_TITLE_ENTRY)
        if titlepageIO.getvalue(): # will be false if no title page.
            outputepub.writestr("OEBPS/title_page.xhtml",titlepageIO.getvalue())
        titlepageIO.close()
@@ -421,7 +485,10 @@ h6 { text-align: center; }
                fullhtml = fullhtml.replace('</p>','</p>\n').replace('<br />','<br />\n')
                outputepub.writestr("OEBPS/file%04d.xhtml"%(index+1),fullhtml.encode('utf-8'))
                del fullhtml
- 
+
+        if self.story.calibrebookmark:
+            outputepub.writestr("META-INF/calibre_bookmarks.txt",self.story.calibrebookmark)
+
 	# declares all the files created by Windows.  otherwise, when
        # it runs in appengine, windows unzips the files as 000 perms.
        for zf in outputepub.filelist:
@@ -439,4 +506,4 @@ def newTag(dom,name,attrs=None,text=None):
    if( text is not None ):
        tag.appendChild(dom.createTextNode(text))
    return tag
-    
+
@@ -39,19 +39,7 @@ class HTMLWriter(BaseStoryWriter):
 <head>
 <title>${title} by ${author}</title>
 <style type="text/css">
-.CI {
-    text-align:center;
-    margin-top:0px;
-    margin-bottom:0px;
-    padding:0px;
-    }
-.center   {text-align: center;}
-.cover    {text-align: center;}
-.full     {width: 100%; }
-.quarter  {width: 25%; }
-.smcap    {font-variant: small-caps;}
-.u        {text-decoration: underline;}
-.bold     {font-weight: bold;}
+${output_css}
 </style>
 </head>
 <body>
@@ -20,8 +20,8 @@ import string
 import StringIO

 from base_writer import *
-from fanficdownloader.htmlcleanup import stripHTML
-from fanficdownloader.mobi import Converter
+from ..htmlcleanup import stripHTML
+from ..mobi import Converter

 class MobiWriter(BaseStoryWriter):

@@ -41,7 +41,6 @@ class MobiWriter(BaseStoryWriter):
 <html xmlns="http://www.w3.org/1999/xhtml">
 <head>
 <title>${title} by ${author}</title>
-<link href="stylesheet.css" type="text/css" charset="UTF-8" rel="stylesheet"/>
 </head>
 <body>
 <h3><a href="${storyUrl}">${title}</a> by <a href="${authorUrl}">${author}</a></h3>
@@ -50,6 +49,10 @@ class MobiWriter(BaseStoryWriter):

        self.MOBI_TITLE_ENTRY = string.Template('''
 <b>${label}:</b> ${value}<br />
+''')
+
+        self.MOBI_NO_TITLE_ENTRY = string.Template('''
+${value}<br />
 ''')

        self.MOBI_TITLE_PAGE_END = string.Template('''
@@ -64,7 +67,6 @@ class MobiWriter(BaseStoryWriter):
 <html xmlns="http://www.w3.org/1999/xhtml">
 <head>
 <title>${title} by ${author}</title>
-<link href="stylesheet.css" type="text/css" charset="UTF-8" rel="stylesheet"/>
 </head>
 <body>
 <h3><a href="${storyUrl}">${title}</a> by <a href="${authorUrl}">${author}</a></h3>
@@ -77,6 +79,10 @@ class MobiWriter(BaseStoryWriter):

        self.MOBI_TABLE_TITLE_WIDE_ENTRY = string.Template('''
 <tr><td colspan="2"><b>${label}:</b> ${value}</td></tr>
+''')
+
+        self.MOBI_TABLE_NO_TITLE_WIDE_ENTRY = string.Template('''
+<tr><td colspan="2">${value}</td></tr>
 ''')

        self.MOBI_TABLE_TITLE_PAGE_END = string.Template('''
@@ -91,7 +97,6 @@ class MobiWriter(BaseStoryWriter):
 <html xmlns="http://www.w3.org/1999/xhtml">
 <head>
 <title>${title} by ${author}</title>
-<link href="stylesheet.css" type="text/css" charset="UTF-8" rel="stylesheet"/>
 </head>
 <body>
 <div>
@@ -113,7 +118,6 @@ class MobiWriter(BaseStoryWriter):
 <html xmlns="http://www.w3.org/1999/xhtml">
 <head>
 <title>${chapter}</title>
-<link href="stylesheet.css" type="text/css" charset="UTF-8" rel="stylesheet"/>
 </head>
 <body>
 <h3>${chapter}</h3>
@@ -124,9 +128,6 @@ class MobiWriter(BaseStoryWriter):
 </html>
 ''')

-    def getMetadata(self,key):
-        return stripHTML(self.story.getMetadata(key))
-
    def writeStoryImpl(self, out):

        files = []
@@ -136,11 +137,13 @@ class MobiWriter(BaseStoryWriter):
            TITLE_PAGE_START  = self.MOBI_TABLE_TITLE_PAGE_START
            TITLE_ENTRY       = self.MOBI_TABLE_TITLE_ENTRY
            WIDE_TITLE_ENTRY  = self.MOBI_TABLE_TITLE_WIDE_ENTRY
+            NO_TITLE_ENTRY    = self.MOBI_TABLE_NO_TITLE_ENTRY
            TITLE_PAGE_END    = self.MOBI_TABLE_TITLE_PAGE_END
        else:
            TITLE_PAGE_START  = self.MOBI_TITLE_PAGE_START
            TITLE_ENTRY       = self.MOBI_TITLE_ENTRY
            WIDE_TITLE_ENTRY  = self.MOBI_TITLE_ENTRY # same, only wide in tables.
+            NO_TITLE_ENTRY    = self.MOBI_NO_TITLE_ENTRY
            TITLE_PAGE_END    = self.MOBI_TITLE_PAGE_END
        
        titlepageIO = StringIO.StringIO()
@@ -148,7 +151,8 @@ class MobiWriter(BaseStoryWriter):
                            START=TITLE_PAGE_START,
                            ENTRY=TITLE_ENTRY,
                            WIDE_ENTRY=WIDE_TITLE_ENTRY,
-                            END=TITLE_PAGE_END)
+                            END=TITLE_PAGE_END,
+                            NO_TITLE_ENTRY=NO_TITLE_ENTRY)
        if titlepageIO.getvalue(): # will be false if no title page.
            files.append(titlepageIO.getvalue())
        titlepageIO.close()
@@ -21,7 +21,7 @@ from textwrap import wrap

 from base_writer import *

-from fanficdownloader.html2text import html2text, BODY_WIDTH
+from ..html2text import html2text

 ## In BaseStoryWriter, we define _write to encode <unicode> objects
 ## back into <string> for true output.  But txt needs to write the
@@ -106,6 +106,12 @@ End file.

    def writeStoryImpl(self, out):

+        self.wrap_width = self.getConfig('wrap_width')
+        if self.wrap_width == '' or self.wrap_width == '0':
+            self.wrap_width = None
+        else:
+            self.wrap_width = int(self.wrap_width)
+        
        wrapout = KludgeStringIO()
        
        wrapout.write(self.TEXT_FILE_START.substitute(self.story.metadata))
@@ -131,15 +137,19 @@ End file.
            if html:
                logging.debug('Writing chapter text for: %s' % title)
                self._write(out,self.lineends(self.wraplines(removeAllEntities(self.TEXT_CHAPTER_START.substitute({'chapter':title, 'index':index+1})))))
-                self._write(out,self.lineends(html2text(html)))
+                self._write(out,self.lineends(html2text(html,wrap_width=self.wrap_width)))

        self._write(out,self.lineends(self.wraplines(self.TEXT_FILE_END.substitute(self.story.metadata))))

    def wraplines(self, text):
+        
+        if not self.wrap_width:
+            return text
+        
        result=''
        for para in text.split("\n"):
            first=True
-            for line in wrap(para, BODY_WIDTH):
+            for line in wrap(para, self.wrap_width):
                if first:
                    first=False
                else:
@@ -13,8 +13,23 @@
 # limitations under the License.
 #

+import pickle, copy
 from google.appengine.ext import db

+class ObjectProperty(db.Property):
+	data_type = db.Blob
+
+	def get_value_for_datastore(self, model_instance):
+		value = self.__get__(model_instance, model_instance.__class__)
+		pickled_val = pickle.dumps(value,protocol=pickle.HIGHEST_PROTOCOL)
+		if value is not None: return db.Blob(pickled_val)
+
+	def make_value_from_datastore(self, value):
+		if value is not None: return pickle.loads(value)
+
+	def default_value(self):
+		return copy.copy(self.default)
+
 class DownloadMeta(db.Model):
 	user = db.UserProperty()
 	url = db.StringProperty()
@@ -37,3 +52,12 @@ class DownloadData(db.Model):
 class UserConfig(db.Model):
 	user = db.UserProperty()
 	config = db.BlobProperty()
+
+class SavedMeta(db.Model):
+	url = db.StringProperty()
+	title = db.StringProperty()
+	author = db.StringProperty()
+	date = db.DateTimeProperty(auto_now_add=True)
+	count = db.IntegerProperty()
+	meta = ObjectProperty()
+	
@@ -4,7 +4,7 @@
 		<link href="css/index.css" rel="stylesheet" type="text/css">
 		<link type="text/css" href="http://jqueryui.com/latest/themes/base/ui.all.css" rel="stylesheet" />
 	 
-		<title>Fanfiction Downloader (fanfiction.net, fictionalley, ficwad to epub and HTML)</title>
+		<title>FanFictionDownLoader (fanfiction.net, fictionalley, ficwad to epub and HTML)</title>
 		<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
 		<script src="/js/jquery-1.3.2.js"></script>
 		<script src="/js/fdownloader.js"></script>
@@ -16,7 +16,7 @@
 	<body>
 		<div id='main'>
 			<h1>
-				<a href="/" style="text-decoration: none; color: black;">FanFiction Downloader</a>
+				<a href="/" style="text-decoration: none; color: black;">FanFictionDownLoader</a>
 			</h1>
 				
 <!--			<form action="/fdown" method="post"> -->
@@ -91,8 +91,8 @@
 					<img src="http://code.google.com/appengine/images/appengine-silver-120x30.gif"
 					alt="Powered by Google App Engine" />
 					<br/><br/>
-					FanfictionLoader is a web front-end to <A href="http://code.google.com/p/fanficdownloader/">fanficdownloader</a><br/>
-					Copyright &copy; <a href="http://twitter.com/sigizmund">Roman Kirillov</a>
+	This is a web front-end to <A href="http://code.google.com/p/fanficdownloader/">FanFictionDownLoader</a><br/>
+	Copyright &copy; Fanficdownloader team
 				</div>
 <!--			</form> -->
 		</div>
@@ -2,7 +2,7 @@
 <html>
  <head>
    <link href="/css/index.css" rel="stylesheet" type="text/css">
-    <title>Fanfiction Downloader - read fanfiction from twilighted.net, fanfiction.net, fictionpress.com, fictionalley.org, ficwad.com, potionsandsnitches.net, harrypotterfanfiction.com, mediaminer.org on Kindle, Nook, Sony Reader, iPad, iPhone, Android, Aldiko, Stanza</title>
+    <title>FanFictionDownLoader - read fanfiction from twilighted.net, fanfiction.net, fictionpress.com, fictionalley.org, ficwad.com, potionsandsnitches.net, harrypotterfanfiction.com, mediaminer.org on Kindle, Nook, Sony Reader, iPad, iPhone, Android, Aldiko, Stanza</title>
    <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
    <meta name="google-site-verification" content="kCFc-G4bka_pJN6Rv8CapPBcwmq0hbAUZPkKWqRsAYU" />
    <script type="text/javascript">
@@ -26,7 +26,7 @@
  <body>
    <div id='main'>
      <h1>
-	<a href="/" style="text-decoration: none; color: black;">FanFiction Downloader</a> <g:plusone size="medium"></g:plusone>
+	<a href="/" style="text-decoration: none; color: black;">FanFictionDownLoader</a> <g:plusone size="medium"></g:plusone>
      </h1>

      <div style="text-align: center">
@@ -50,38 +50,32 @@
      <form action="/fdown" method="post">
 	<div id='urlbox'>
 	  <div id='greeting'>
-	    <p>Hi, {{ nickname }}! This is a fan fiction downloader, which makes reading stories from various websites
+	    <p>Hi, {{ nickname }}! This is FanFictionDownLoader, which makes reading stories from various websites
 	      much easier. </p>
 	  </div>
 	  <!-- put announcements here, h3 is a good title size. -->
-	  <h3>Please Switch to <a href="http://fanfictiondownloader.appspot.com/">New Version</a></h3>
+	  <h3>fanfiction.net Fixed</h3>
 	  <p>
-	    We have a new, more efficient, version of the system up now.  Please start using the 
-	    <a href="http://fanfictiondownloader.appspot.com/">New
-	      Version here</a>.  If for some reason, the new version
-	      doesn't work for you, please let us know on
-	      the <a href="http://groups.google.com/group/fanfic-downloader">Fanfiction
-	      Downloader Google Group</a>.
+
+	    Fixes are in place now for the latest round of fanfiction.net changes (as of Jun 28, 2012).
+	    
 	  </p>
-	  <h3>New Google Quotas</h3>
+	  <h3>New Sites</h3>
 	  <p>
-	    Google has changed their quota limits for free
-	    applications using their AppEngine system, like this one.
-	    We expect that there will be times when the
-	    system exceeds it's permitted processing quota.
+	    
+	    New sites www.fanfiktion.de, ponyfictionarchive.net and sg1-heliopolis.com added.
+	    <br />Thanks, Ida!
+	    
 	  </p>
 	  <p>
-	    You also have the option of running the downloader on your
-	    own computer if you have Python available.
-	    <a href="http://code.google.com/p/fanficdownloader/downloads/list">Download here.</a>
-	  </p>
+	    Questions?  Check out our 
+	    <a href="http://code.google.com/p/fanficdownloader/wiki/FanFictionDownloaderFAQs">FAQs</a>.
+	    </p>
 	  <p>
 	    If you have any problems with this application, please
 	    report them in
-	    the <a href="http://groups.google.com/group/fanfic-downloader">Fanfiction
-	      Downloader Google Group</a>.  The
-	    <a href="http://4-0-6.fanfictionloader.appspot.com">Previous
-	      Version</a> is also available for you to use if necessary.
+	    the <a href="http://groups.google.com/group/fanfic-downloader">FanFictionDownLoader Google Group</a>.  The
+	    <a href="http://4-4-16.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
 	  </p>
 	  <div id='error'>
 	    {{ error_message }}
@@ -111,21 +105,50 @@
 	  <p>
 	    Or see your personal list of <a href="/recent">previously downloaded fanfics</a>.
 	  </p>
+	  <p>
+	    See a list of downloaded fanfics by all users by <a href="/allrecent">most popular</a> or <a href="/allrecent?bydate=1">most recent</a>.
+	  </p>
 	</div>	
      </form>
      {% else %}
      <div id='urlbox'>
 	<div id='greeting'>
 	  <p>
-	    This is a fan fiction downloader, which makes reading stories from various websites much easier. Before you
-	    can start downloading fanfics, you need to login, so downloader can remember your fanfics and store them.
+	    This is a FanFictionDownLoader, which makes reading stories from various websites much easier. Before you
+	    can start downloading fanfics, you need to login, so FanFictionDownLoader can remember your fanfics and store them.
 	  </p>
 	  <p><a href="{{ login_url }}">Login using Google account</a></p>
 	</div>
      </div>
      {% endif %}
      
+      <div id='typebox'>
+	  <p>
+	    <b>FanFictionDownLoader calibre Plugin</b>
+	    <br /><br />
+	    
+	    There's now a version of this downloader that runs
+	    entirely inside the
+	    popular <a href="http://calibre-ebook.com/">calibre</a>
+	    ebook management package as a plugin.
+	    
+	    <br /><br />
+	    
+	    Once you have calibre installed and running, inside
+	    calibre, you can go to 'Get plugins to enhance calibre' or
+	    'Get new plugins' and
+	    install <a href="http://www.mobileread.com/forums/showthread.php?t=163261">FanFictionDownLoader</a>.
+	    
+	  </p>
+      </div>
      <div id='helpbox'>
+	<h3>Supported sites:</h3>
+	<p>
+	  There's a 
+ 	  <a href="http://code.google.com/p/fanficdownloader/wiki/FanFictionDownloaderSupportedsites">Supported
+ 	  Sites</a> page in our wiki.  If you have a site you'd like
+ 	  to see supported, please check there first.
+	</p>
 	<dl>
 	  <dt>fictionalley.org</dt>
 	  <dd>
@@ -201,10 +224,10 @@
 	    Use the URL of the story's chapter list, such as
 	    <br /><a href="http://fanfiction.tenhawkpresents.com/viewstory.php?sid=294">http://fanfiction.tenhawkpresents.com/viewstory.php?sid=294</a>.
 	  </dd>
-	  <dt>fanfic.castletv.net</dt>
+	  <dt>castlefans.org</dt>
 	  <dd>
 	    Use the URL of the story's chapter list, such as
-	    <br /><a href="http://fanfic.castletv.net/viewstory.php?sid=123">http://fanfic.castletv.net/viewstory.php?sid=123</a>.
+	    <br /><a href="http://castlefans.org/fanfic/viewstory.php?sid=123">http://castlefans.org/fanfic/viewstory.php?sid=123</a>.
 	  </dd>
 	  <dt>fimfiction.net</dt>
 	  <dd>
@@ -213,17 +236,228 @@
 	    <br /> or the URL of any chapter, such as
 	    <br /><a href="http://www.fimfiction.com/story/123/1/">http://www.fimfiction.com/story/123/1/</a>.
 	  </dd>
-	</dl>	
+	  <dt>tthfanfic.org</dt>
+	  <dd>
+	    Use the URL of any story, with or without chapter, title and notice, such as
+	    <br /><a href="http://www.tthfanfic.org/Story-5583">http://www.tthfanfic.org/Story-5583</a>
+	    <br /><a href="http://www.tthfanfic.org/Story-5583/Greywizard+Marked+By+Kane.htm">http://www.tthfanfic.org/Story-5583/Greywizard+Marked+By+Kane.htm</a>.
+	    <br /><a href="http://www.tthfanfic.org/T-99999999/Story-26448-15/batzulger+Willow+Rosenberg+and+the+Mind+Riders.htm">http://www.tthfanfic.org/T-99999999/Story-26448-15/batzulger+Willow+Rosenberg+and+the+Mind+Riders.htm</a>.
+	  </dd>
+	  <dt>www.siye.co.uk</dt>
+	  <dd>
+	    Use the URL of the story's chapter list, such as
+	    <br /><a href="http://www.siye.co.uk/siye/viewstory.php?sid=123">http://www.siye.co.uk/siye/viewstory.php?sid=123</a>.
+	  </dd>
+	  <dt>archiveofourown.org</dt>
+	  <dd>
+	    Use the URL of the story, or one of it's chapters, such as
+	    <br /><a href="http://archiveofourown.org/works/76366">http://archiveofourown.org/works/76366</a>.
+	    <br /><a href="http://archiveofourown.org/works/76366/chapters/101584">http://archiveofourown.org/works/76366/chapters/101584</a>.
+	  </dd>
+	  <dt>ficbook.net(Russian)</dt>
+	  <dd>
+	    Use the URL of the story, or one of it's chapters, such as
+	    <br /><a href="http://ficbook.net/readfic/93626">http://ficbook.net/readfic/93626</a>.
+	    <br /><a href="http://ficbook.net/readfic/93626/246417#part_content">http://ficbook.net/readfic/93626/246417#part_content</a>.
+	  </dd>
+	  <dt>fanfiction.mugglenet.com</dt>
+	  <dd>
+	    Use the URL of the story's chapter list, such as
+	    <br /><a href="http://fanfiction.mugglenet.com/viewstory.php?sid=123">http://fanfiction.mugglenet.com/viewstory.php?sid=123</a>.
+	  </dd>
+	  <dt>www.hpfandom.net</dt>
+	  <dd>
+	    Use the URL of the story's chapter list, such as
+	    <br /><a href="http://www.hpfandom.net/eff/viewstory.php?sid=123">http://www.hpfandom.net/eff/viewstory.php?sid=123</a>.
+	  </dd>
+	  <dt>thequidditchpitch.org</dt>
+	  <dd>
+	    Use the URL of the story's chapter list, such as
+	    <br /><a href="http://thequidditchpitch.org/viewstory.php?sid=123">http://thequidditchpitch.org/viewstory.php?sid=123</a>.
+	  </dd>
+	  <dt>fanfiction.portkey.org</dt>
+	  <dd>
+	    Use the URL of the story's chapter list, such as
+	    <br /><a href="http://fanfiction.portkey.org/story/123">http://fanfiction.portkey.org/story/123</a>.
+	  </dd>
+	  <dt>nfacommunity.com</dt>
+	  <dd>
+	    Use the URL of the story's chapter list, such as
+	    <br /><a href="http://nfacommunity.com/viewstory.php?sid=1654">http://nfacommunity.com/viewstory.php?sid=1654</a>.
+	  </dd>
+	  <dt>www.midnightwhispers.ca</dt>
+	  <dd>
+	    Use the URL of the story's chapter list, such as
+	    <br /><a href="http://www.midnightwhispers.ca/viewstory.php?sid=1124">http://www.midnightwhispers.ca/viewstory.php?sid=1124</a>.
+	  </dd>
+	  <dt>ksarchive.com</dt>
+	  <dd>
+	    Use the URL of the story's chapter list, such as
+	    <br /><a href="http://ksarchive.com/viewstory.php?sid=1124">http://ksarchive.com/viewstory.php?sid=1124</a>.
+	  </dd>
+	  <dt>archive.skyehawke.com</dt>
+	  <dd>
+	    Use the URL of the story's summary, such as
+	    <br /><a href="http://archive.skyehawke.com/story.php?no=17466">http://archive.skyehawke.com/story.php?no=17466</a>.
+	  </dd>
+	  <dt>www.libraryofmoria.com</dt>
+	  <dd>
+	    Use the URL of the story's summary, such as
+	    <br /><a href="http://www.libraryofmoria.com/a/viewstory.php?sid=434">http://www.libraryofmoria.com/a/viewstory.php?sid=434</a>.
+	  </dd>
+	  <dt>www.wraithbait.com</dt>
+	  <dd>
+	    Use the URL of the story's summary, such as
+	    <br /><a href="http://www.wraithbait.com/viewstory.php?sid=14305">http://www.wraithbait.com/viewstory.php?sid=14305</a>.
+	  </dd>
+	  <dt>www.squidge.org/peja (WWOMB)</dt>
+	  <dd>
+	    Use the URL of the story's summary, such as
+	    <br /><a href="http://www.squidge.org/peja/cgi-bin/viewstory.php?sid=1234">http://www.squidge.org/peja/cgi-bin/viewstory.php?sid=1234</a>.<br />
+	    This is <i>only</i> for squidge.org/peja, not other parts of squidge.org.
+	  </dd>
+	  <dt>www.checkmated.com</dt>
+	  <dd>
+	    Use the URL of the story's first chapter, such as
+	    <br /><a href="http://www.checkmated.com/story.php?story=10898">http://www.checkmated.com/story.php?story=10898</a>.
+	  </dd>

-	
-	A few additional things to know, which will make your life substantially easier:
+	  <dt>ashwinder.sycophanthex.com</dt>
+	  <dd>
+	    Use the URL of the story's chapter list, such as
+	    <br /><a href="http://ashwinder.sycophanthex.com/viewstory.php?sid=1234">http://ashwinder.sycophanthex.com/viewstory.php?sid=1234</a>
+	  </dd>
+	  <dt>chaos.sycophanthex.com</dt>
+	  <dd>
+	    Use the URL of the story's chapter list, such as
+	    <br /><a href="http://chaos.sycophanthex.com/viewstory.php?sid=1234">http://chaos.sycophanthex.com/viewstory.php?sid=1234</a>
+	  </dd>
+	  <dt>erosnsappho.sycophanthex.com</dt>
+	  <dd>
+	    Use the URL of the story's chapter list, such as
+	    <br /><a href="http://erosnsappho.sycophanthex.com/viewstory.php?sid=1234">http://erosnsappho.sycophanthex.com/viewstory.php?sid=1234</a>
+	  </dd>
+	  <dt>lumos.sycophanthex.com</dt>
+	  <dd>
+	    Use the URL of the story's chapter list, such as
+	    <br /><a href="http://lumos.sycophanthex.com/viewstory.php?sid=1234">http://lumos.sycophanthex.com/viewstory.php?sid=1234</a>
+	  </dd>
+	  <dt>occlumency.sycophanthex.com</dt>
+	  <dd>
+	    Use the URL of the story's chapter list, such as
+	    <br /><a href="http://occlumency.sycophanthex.com/viewstory.php?sid=1234">http://erosnsappho.sycophanthex.com/viewstory.php?sid=1234</a>
+	  </dd>
+	  <dt>dramione.org</dt>
+	  <dd>
+	    Use the URL of the story's chapter list, such as
+	    <br /><a href="http://dramione.org/viewstory.php?sid=1234">http://dramione.org/viewstory.php?sid=1234</a>
+	  </dd>
+	  <dt>www.phoenixsong.net</dt>
+	  <dd>
+	    Use the URL of any story chapter, such as
+	    <br /><a href="http://www.phoenixsong.net/fanfiction/story/1234/">http://www.phoenixsong.net/fanfiction/story/1234/</a>
+	  </dd>
+	  <dt>www.walkingtheplank.org</dt>
+	  <dd>
+	    Use the URL of the story's first chapter, such as
+	    <br /><a href="http://www.walkingtheplank.org/archive/viewstory.php?sid=1234">http://www.walkingtheplank.org/archive/viewstory.php?sid=1234</a>
+	  </dd>
+	  <dt>thehexfiles.net</dt>
+	  <dd>
+	    Use the URL of the story's chapter list, such as
+	    <br /><a href="http://thehexfiles.net/viewstory.php?sid=1234">http://thehexfiles.net/viewstory.php?sid=1234</a>
+	  </dd>
+	  <dt>www.dokuga.com</dt>
+	  <dd>
+	    Use the URL of any story chapter, such as
+	    <br /><a href="http://www.dokuga.com/fanfiction/story/1234/1">http://www.dokuga.com/fanfiction/story/1234/1</a> or
+	    <br /><a href="http://www.dokuga.com/spark/story/1234/1">http://www.dokuga.com/spark/story/1234/1</a>
+	  </dd>
+	  <dt>www.ik-eternal.net</dt>
+	  <dd>
+	    Use the URL of the story's chapter list, such as
+	    <br /><a href="http://www.ik-eternal.net/viewstory.php?sid=1234">http://www.ik-eternal.net/viewstory.php?sid=1234</a>
+	  </dd>
+	  <dt>onedirectionfanfiction.com</dt>
+	  <dd>
+	    Use the URL of the story's first chapter, such as
+	    <br /><a href="http://onedirectionfanfiction.com/viewstory.php?sid=1234">http://onedirectionfanfiction.com/viewstory.php?sid=1234</a>
+	  </dd>
+	  <dt>www.prisonbreakfic.net</dt>
+	  <dd>
+	    Use the URL of the story's first chapter, such as
+	    <br /><a href="http://www.prisonbreakfic.net/viewstory.php?sid=1234">http://www.prisonbreakfic.net/viewstory.php?sid=1234</a>
+	  </dd>
+	  <dt>www.storiesofarda.com</dt>
+	  <dd>
+	    Use the URL of the story's chapter list, such as
+	    <br /><a href="http://www.storiesofarda.com/chapterlistview.asp?SID=7000">http://www.storiesofarda.com/chapterlistview.asp?SID=7000</a>
+	  </dd>
+	  <dt>samdean.archive.nu</dt>
+	  <dd>
+	    Use the URL of the story's first chapter, such as
+	    <br /><a href="http://samdean.archive.nu/viewstory.php?sid=1234">http://samdean.archive.nu/viewstory.php?sid=1234</a>
+	  </dd>
+	  <dt>www.yourfanfiction.com</dt>
+	  <dd>
+	    Use the URL of the story's first chapter, such as
+	    <br /><a href="http://www.yourfanfiction.com/viewstory.php?sid=1234">http://www.yourfanfiction.com/viewstory.php?sid=1234</a>
+	  </dd>
+	  <dt>www.destinysgateway.com</dt>
+	  <dd>
+	    Use the URL of the story's first chapter, such as
+	    <br /><a href="http://www.destinysgateway.com/viewstory.php?sid=1234">http://www.destinysgateway.com/viewstory.php?sid=1234</a>
+	  </dd>
+	  <dt>www.thealphagate.com</dt>
+	  <dd>
+	    Use the URL of the story's first chapter, such as
+	    <br /><a href="http://www.thealphagate.com/viewstory.php?sid=1234">http://www.thealphagate.com/viewstory.php?sid=1234</a>
+	  </dd>
+	  <dt>stargate-atlantis.org</dt>
+	  <dd>
+	    Use the URL of the story's first chapter, such as
+	    <br /><a href="http://stargate-atlantis.org/fanfics/viewstory.php?sid=1234">http://stargate-atlantis.org/fanfics/viewstory.php?sid=1234</a>
+	  </dd>
+	  <dt>www.ncisfiction.com</dt>
+	  <dd>
+	    Use the URL of the story's first chapter, such as
+	    <br /><a href="http://www.ncisfiction.com/story.php?stid=1234">http://www.ncisfiction.com/story.php?stid=1234</a>
+	    Or use the URL of the story's chapter list, such as
+	    <br /><a href="http://www.ncisfiction.com/chapters.php?stid=1234">http://www.ncisfiction.com/chapters.php?stid=1234</a>
+	  </dd>
+	  <dt>www.fanfiktion.de</dt>
+	  <dd>
+	    Use the URL of the story's first chapter, such as
+	    <br /><a href="http://www.fanfiktion.de/s/46ccbef30000616306614050">http://www.fanfiktion.de/s/46ccbef30000616306614050</a>
+	  </dd>
+	  <dt>ponyfictionarchive.net</dt>
+	  <dd>
+	    Use the URL of the story's first chapter, such as
+	    <br /><a href="http://ponyfictionarchive.net/viewstory.php?sid=1234">http://ponyfictionarchive.net/viewstory.php?sid=1234</a>
+	    <br />Or:
+	    <br /><a href="http://explicit.ponyfictionarchive.net/viewstory.php?sid=1234">http://explicit.ponyfictionarchive.net/viewstory.php?sid=1234</a>
+	  </dd>
+	  <dt>sg1-heliopolis.com</dt>
+	  <dd>
+	    Use the URL of the story's first chapter, such as
+	    <br /><a href="http://sg1-heliopolis.com/archive/viewstory.php?sid=1234">http://sg1-heliopolis.com/archive/viewstory.php?sid=1234</a>
+	    <br />Or:
+	    <br /><a href="http://sg1-heliopolis.com/adult/viewstory.php?sid=1234">http://sg1-heliopolis.com/adult/viewstory.php?sid=1234</a>
+	    <br />Or:
+	    <br /><a href="http://sg1-heliopolis.com/atlantis/viewstory.php?sid=1234">http://sg1-heliopolis.com/atlantis/viewstory.php?sid=1234</a>
+	  </dd>
+	</dl>
+	<p>
+	  A few additional things to know, which will make your life substantially easier:
+	</p>
 	<ol>
 	  <li>
-	    First thing to know: I do not use your Google login and password. In fact, all I know about it is your ID &ndash; password
+	    First thing to know: We do not use your Google login and password. In fact, all we know about it is your ID &ndash; password
 	    is being verified by Google and is absolutely, totally unknown to anyone but you.
 	  </li>
 	  <li>
-	    Small <a href="http://www.sigizmund.com/reading-fanfiction-off-line-in-stanza-and-oth">post written by me</a>
+	    
+	    Small <a href="http://www.sigizmund.com/reading-fanfiction-off-line-in-stanza-and-oth">post written by Roman</a>
 	    &mdash; how to read fiction in Stanza or any other ebook reader.
 	  </li>
 	  <li>
@@ -239,7 +473,7 @@
 	  </li>
 	  <li>
 	    If you think that something that should work in fact doesn't, post a message to
-	    our <a href="http://groups.google.com/group/fanfic-downloader">Google Group</a>. I also encourage you to join it so
+	    our <a href="http://groups.google.com/group/fanfic-downloader">Google Group</a>. we also encourage you to join it so
 	    you will find out about latest updates and fixes as soon as possible
 	  </li>
 	</ol>
@@ -249,8 +483,8 @@
 	<img src="http://code.google.com/appengine/images/appengine-silver-120x30.gif"
 	     alt="Powered by Google App Engine" />
 	<br/><br/>
-	FanfictionLoader is a web front-end to <A href="http://code.google.com/p/fanficdownloader/">fanficdownloader</a><br/>
-	Copyright &copy; Fanficdownloader team
+	This is a web front-end to <A href="http://code.google.com/p/fanficdownloader/">FanFictionDownLoader</a><br/>
+	Copyright &copy; FanFictionDownLoader team
      </div>

      <div style="margin-top: 1em; text-align: center'">
@@ -1,6 +1,6 @@
 indexes:

-# AUTOGENERATED
+# notAUTOGENERATED

 # This index.yaml is automatically updated whenever the dev_appserver
 # detects that a new type of query is run.  If you want to manage the
@@ -21,13 +21,8 @@ indexes:
  - name: date
    direction: desc

- kind: DownloadedFanfic
+- kind: SavedMeta
  properties:
-  - name: cleared
-  - name: date
-
- kind: DownloadedFanfic
-  properties:
-  - name: user
+  - name: count
  - name: date
    direction: desc
@@ -2,7 +2,7 @@
 <html>
  <head>
    <link href="/css/index.css" rel="stylesheet" type="text/css">
-    <title>Login Needed Fanfiction Downloader</title>
+    <title>Login Needed FanFictionDownLoader</title>
    <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
    <meta name="google-site-verification" content="kCFc-G4bka_pJN6Rv8CapPBcwmq0hbAUZPkKWqRsAYU" />
    <script type="text/javascript">
@@ -22,7 +22,7 @@
  <body>
    <div id='main'>
      <h1>
-	<a href="/" style="text-decoration: none; color: black;">FanFiction Downloader</a>
+	<a href="/" style="text-decoration: none; color: black;">FanFictionDownLoader</a>
      </h1>

      <div style="text-align: center">
@@ -88,8 +88,8 @@
 	<img src="http://code.google.com/appengine/images/appengine-silver-120x30.gif"
 	     alt="Powered by Google App Engine" />
 	<br/><br/>
-	FanfictionLoader is a web front-end to <A href="http://code.google.com/p/fanficdownloader/">fanficdownloader</a><br/>
-	Copyright &copy; Fanficdownloader team
+	This is a web front-end to <A href="http://code.google.com/p/fanficdownloader/">FanFictionDownLoader</a><br/>
+	Copyright &copy; FanFictionDownLoader team
      </div>

      <div style="margin-top: 1em; text-align: center'">
@@ -41,23 +41,24 @@ import ConfigParser
 ## Console page first, you will get a django version mismatch error when you
 ## to go hit one of the application pages.  Just change a file again, and
 ## make sure to hit an app page before the SDK page to clear it.
-os.environ['DJANGO_SETTINGS_MODULE'] = 'settings'
-from google.appengine.dist import use_library
-use_library('django', '1.2')
+#os.environ['DJANGO_SETTINGS_MODULE'] = 'settings'
+#from google.appengine.dist import use_library
+#use_library('django', '1.2')

 from google.appengine.ext import db
 from google.appengine.api import taskqueue
 from google.appengine.api import users
-from google.appengine.ext import webapp
+#from google.appengine.ext import webapp
+import webapp2
 from google.appengine.ext.webapp import template
-from google.appengine.ext.webapp import util
+#from google.appengine.ext.webapp2 import util
 from google.appengine.runtime import DeadlineExceededError

 from ffstorage import *

 from fanficdownloader import adapters, writers, exceptions

-class UserConfigServer(webapp.RequestHandler):
+class UserConfigServer(webapp2.RequestHandler):
    def getUserConfig(self,user):
        config = ConfigParser.SafeConfigParser()

@@ -73,7 +74,7 @@ class UserConfigServer(webapp.RequestHandler):

        return config

-class MainHandler(webapp.RequestHandler):
+class MainHandler(webapp2.RequestHandler):
    def get(self):
        user = users.get_current_user()
        if user:
@@ -160,7 +161,7 @@ class EditConfigServer(UserConfigServer):
            self.response.out.write(template.render(path, template_values))


-class FileServer(webapp.RequestHandler):
+class FileServer(webapp2.RequestHandler):

    def get(self):
        fileId = self.request.get('id')
@@ -221,7 +222,7 @@ class FileServer(webapp.RequestHandler):
            path = os.path.join(os.path.dirname(__file__), 'status.html')
            self.response.out.write(template.render(path, template_values))

-class FileStatusServer(webapp.RequestHandler):
+class FileStatusServer(webapp2.RequestHandler):
    def get(self):
        user = users.get_current_user()
        if not user:
@@ -257,7 +258,7 @@ class FileStatusServer(webapp.RequestHandler):
        path = os.path.join(os.path.dirname(__file__), 'status.html')
        self.response.out.write(template.render(path, template_values))

-class ClearRecentServer(webapp.RequestHandler):
+class ClearRecentServer(webapp2.RequestHandler):
    def get(self):
        user = users.get_current_user()
        if not user:
@@ -282,7 +283,7 @@ class ClearRecentServer(webapp.RequestHandler):
        logging.info('Deleted %d instances download.' % num)
        self.redirect("/?error=recentcleared")

-class RecentFilesServer(webapp.RequestHandler):
+class RecentFilesServer(webapp2.RequestHandler):
    def get(self):
        user = users.get_current_user()
        if not user:
@@ -302,6 +303,39 @@ class RecentFilesServer(webapp.RequestHandler):
        path = os.path.join(os.path.dirname(__file__), 'recent.html')
        self.response.out.write(template.render(path, template_values))

+class AllRecentFilesServer(webapp2.RequestHandler):
+    def get(self):
+        user = users.get_current_user()
+        if not user:
+            self.redirect(users.create_login_url(self.request.uri))
+            return
+
+        q = SavedMeta.all()
+        if self.request.get('bydate'):
+            q.order('-date')
+        else:
+            q.order('-count')
+            
+        fics = q.fetch(200)
+        logging.info("Recent fetched %d downloads for user %s."%(len(fics),user.nickname()))
+
+        sendslugs = []
+
+        for fic in fics:
+            ficslug = FicSlug(fic)
+            sendslugs.append(ficslug)
+                
+        template_values = dict(fics = sendslugs, nickname = user.nickname())
+        path = os.path.join(os.path.dirname(__file__), 'allrecent.html')
+        self.response.out.write(template.render(path, template_values))
+
+class FicSlug():
+    def __init__(self,savedmeta):
+        self.url = savedmeta.url
+        self.count = savedmeta.count
+        for k, v in savedmeta.meta.iteritems():
+            setattr(self,k,v)
+        
 class FanfictionDownloader(UserConfigServer):
    def get(self):
        self.post()
@@ -338,7 +372,7 @@ class FanfictionDownloader(UserConfigServer):
                self.redirect("/?error=custom&errtext=%s"%urlEscape("There's an error in your User Configuration: "+str(e)))
                return

-            adapter = adapters.getAdapter(config,url)
+            adapter = adapters.getAdapter(config,url,format)
            logging.info('Created an adaper: %s' % adapter)

            if len(login) > 1:
@@ -384,7 +418,7 @@ class FanfictionDownloader(UserConfigServer):
            template_values = dict(nickname = user.nickname(),
                                   url = url,
                                   format = format,
-                                   site = adapter.getSiteDomain(),
+                                   site = adapter.getConfigSection(),
                                   fic = download,
                                   is_login=is_login,
                                   )
@@ -441,7 +475,7 @@ class FanfictionDownloaderTask(UserConfigServer):

        try:
            config = self.getUserConfig(user)
-            adapter = adapters.getAdapter(config,url)
+            adapter = adapters.getAdapter(config,url,format)

            logging.info('Created an adapter: %s' % adapter)

@@ -463,6 +497,8 @@ class FanfictionDownloaderTask(UserConfigServer):
            download.url = adapter.getStory().getMetadata('storyUrl')
            download.put()

+            allmeta = adapter.getStory().getAllMetadata(removeallentities=True,doreplacements=False)
+
            outbuffer = StringIO()
            writer.writeStory(outbuffer)
            data = outbuffer.getvalue()
@@ -494,6 +530,21 @@ class FanfictionDownloaderTask(UserConfigServer):
            download.completed=True
            download.put()

+            smetal = SavedMeta.all().filter('url =', allmeta['storyUrl'] ).fetch(1)
+            if smetal and smetal[0]:
+                smeta = smetal[0]
+                smeta.count += 1
+            else:
+                smeta=SavedMeta()
+                smeta.count = 1
+
+            smeta.url = allmeta['storyUrl']
+            smeta.title = allmeta['title']
+            smeta.author = allmeta['author']
+            smeta.meta = allmeta
+            smeta.date = datetime.datetime.now()
+            smeta.put()
+            
            logging.info("Download finished OK")
            del data

@@ -556,20 +607,15 @@ def urlEscape(data):
    p = re.compile(r'([^\w])')
    return p.sub(toPercentDecimal, data.encode("utf-8"))

-def main():
-  application = webapp.WSGIApplication([('/', MainHandler),
-                                        ('/fdowntask', FanfictionDownloaderTask),
-                                        ('/fdown', FanfictionDownloader),
-                                        (r'/file.*', FileServer),
-                                        ('/status', FileStatusServer),
-                                        ('/recent', RecentFilesServer),
-                                        ('/editconfig', EditConfigServer),
-                                        ('/clearrecent', ClearRecentServer),
-                                        ],
-                                       debug=False)
-  util.run_wsgi_app(application)
-
-
-if __name__ == '__main__':
-    logging.getLogger().setLevel(logging.DEBUG)
-    main()
+logging.getLogger().setLevel(logging.DEBUG)
+app = webapp2.WSGIApplication([('/', MainHandler),
+                               ('/fdowntask', FanfictionDownloaderTask),
+                               ('/fdown', FanfictionDownloader),
+                               (r'/file.*', FileServer),
+                               ('/status', FileStatusServer),
+                               ('/allrecent', AllRecentFilesServer),
+                               ('/recent', RecentFilesServer),
+                               ('/editconfig', EditConfigServer),
+                               ('/clearrecent', ClearRecentServer),
+                               ],
+                              debug=False)
@@ -0,0 +1,38 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+# epubmerge.py 1.0
+
+# Copyright 2011, Jim Miller
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#   http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+from glob import glob
+
+from makezip import createZipFile
+
+if __name__=="__main__":
+    filename="FanFictionDownLoader.zip"
+    exclude=['*.pyc','*~','*.xcf','*[0-9].png']
+    # from top dir. 'w' for overwrite
+    createZipFile(filename,"w",
+                  ['plugin-defaults.ini','plugin-example.ini','epubmerge.py','fanficdownloader'],
+                  exclude=exclude)
+    #from calibre-plugin dir. 'a' for append
+    os.chdir('calibre-plugin')
+    files=['about.txt','images',]
+    files.extend(glob('*.py'))
+    files.extend(glob('plugin-import-name-*.txt'))
+    createZipFile("../"+filename,"a",
+                  files,exclude=exclude)
@@ -0,0 +1,54 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+# epubmerge.py 1.0
+
+# Copyright 2011, Jim Miller
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#   http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os, zipfile, sys
+from glob import glob
+
+def addFolderToZip(myZipFile,folder,exclude=[]):
+    folder = folder.encode('ascii') #convert path to ascii for ZipFile Method
+    excludelist=[]
+    for ex in exclude:
+        excludelist.extend(glob(folder+"/"+ex))
+    for file in glob(folder+"/*"):
+        if file in excludelist:
+            continue
+        if os.path.isfile(file):
+            #print file
+            myZipFile.write(file, file, zipfile.ZIP_DEFLATED)
+        elif os.path.isdir(file):
+            addFolderToZip(myZipFile,file,exclude=exclude)
+
+def createZipFile(filename,mode,files,exclude=[]):
+    myZipFile = zipfile.ZipFile( filename, mode ) # Open the zip file for writing
+    excludelist=[]
+    for ex in exclude:
+        excludelist.extend(glob(ex))
+    for file in files:
+        if file in excludelist:
+            continue
+        file = file.encode('ascii') #convert path to ascii for ZipFile Method
+        if os.path.isfile(file):
+            (filepath, filename) = os.path.split(file)
+            #print file
+            myZipFile.write( file, filename, zipfile.ZIP_DEFLATED )
+        if os.path.isdir(file):
+            addFolderToZip(myZipFile,file,exclude=exclude)
+    myZipFile.close()
+    return (1,filename)
+
@@ -0,0 +1,662 @@
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+[defaults]
+
+## [defaults] section applies to all formats and sites but may be
+## overridden at several levels
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
+## All available titlepage_entries and the label used for them:
+## <entryname>_label:<label>
+## Labels may be customized.
+title_label:Title
+storyUrl_label:Story URL
+description_label:Summary
+author_label:Author
+authorUrl_label:Author URL
+## epub, txt, html
+formatname_label:File Format
+## .epub, .txt, .html
+formatext_label:File Extension
+## Category and Genre have overlap, depending on the site.
+## Sometimes Harry Potter is a category and Fantasy a genre. (fanfiction.net)
+## Sometimes Fantasy is category *and* a genre (fictionpress.com)
+## Sometimes there are multiple categories and/or genres.
+category_label:Category
+genre_label:Genre
+language_label:Language
+characters_label:Characters
+series_label:Series
+## Completed/In-Progress
+status_label:Status
+## Dates story first published, last updated, and downloaded(last with time).
+datePublished_label:Published
+dateUpdated_label:Updated
+dateCreated_label:Packaged
+## Rating depends on the site.  Some use K,T,M,etc, and some PG,R,NC-17
+rating_label:Rating
+## Also depends on the site.  
+warnings_label:Warnings
+numChapters_label:Chapters
+numWords_label:Words
+## www.fanfiction.net, fictionalley.com, etc.
+site_label:Publisher
+## ffnet, fpcom, etc.
+siteabbrev_label:Site Abbrev
+## The site's unique story/author identifier.  Usually a number.
+storyId_label:Story ID
+authorId_label:Author ID
+## Primarily to put specific values in dc:subject tags for epub.  Will
+## show up in Calibre as tags.  Also carried into mobi when converted.
+extratags_label:Extra Tags
+## The version of fanficdownloader
+##
+version_label:FFDL Version
+
+## items to include in the title page
+## Empty entries will *not* appear, even if in the list.
+## All current formats already include title and author.
+titlepage_entries: series,category,genre,language,characters,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,description
+
+## Try to collect series name and number of this story in series.
+## Some sites (ab)use 'series' for reading lists and personal
+## collections.  This lets us turn it on and off by site without
+## keeping a lengthy titlepage_entries per site and prevents it
+## updating in the plugin.
+collect_series: true
+
+## include title page as first page.
+include_titlepage: true
+
+## include a TOC page before the story text
+include_tocpage: true
+
+## website encoding(s) In theory, each website reports the character
+## encoding they use for each page.  In practice, some sites report it
+## incorrectly.  Each adapter has a default list, usually "utf8,
+## Windows-1252" or "Windows-1252, utf8", but this will let you
+## explicitly set the encoding and order if you need to.  The special
+## value 'auto' will call chardet and use the encoding it reports if
+## it has +90% confidence.  'auto' is not reliable.
+#website_encodings: auto, utf8, Windows-1252
+
+## entries to make epub subjects and calibre tags
+## lastupdate creates two tags: "Last Update Year/Month: %Y/%m" and "Last Update: %Y/%m/%d"
+include_subject_tags: extratags, genre, category, characters, status
+
+## extra tags (comma separated) to include, primarily for epub.
+extratags: FanFiction
+
+## number of seconds to sleep between calls to the story site.  May by
+## useful if pulling large numbers of stories or if the site is slow.
+#slow_down_sleep_time:0.5
+
+## Use regular expressions to find and replace (or remove) metadata.
+## For example, you could change Sci-Fi=>SF, remove *-Centered tags,
+## etc.  See http://docs.python.org/library/re.html (look for re.sub)
+## for regexp details.
+## Make sure to keep at least one space at the start of each line and
+## to escape % to %%, if used.
+#replace_metadata:
+# Sci-Fi=>SF
+# Puella Magi Madoka Magica.* => Madoka
+# Comedy=>Humor
+# Crossover: (.*)=>\1
+# (.*)Great(.*)=>\1Moderate\2
+# .*-Centered=>
+
+## Some readers don't show horizontal rule (<hr />) tags correctly.
+## This replaces them all with a centered '* * *'.  (Note centering
+## doesn't work on some devices either.)
+#replace_hr: false
+
+## If you have the Generate Cover plugin installed, you can use the
+## generate_cover_settings parameter to intelligently decide which GC
+## setting to run.  There are three parts 1) a template of which
+## metadata part(s) to look at, 2) a regular expression to match the
+## template, and 3) the name of the GC setting to use, which must
+## match exactly.  Use this parameter in [defaults], or by site eg,
+## [www.ficwad.com]
+## Make sure to keep at least one space at the start of each line and
+## to escape % to %%, if used.
+## template => regexp to match => GC Setting to use.
+#generate_cover_settings:
+# ${category} => Buffy:? [tT]he Vampire Slayer => BuffyCover
+# ${category} => Star Trek => StarTrekCover
+
+## Each output format has a section that overrides [defaults]
+[html]
+
+## output background color--only used by html and epub (and ignored in
+## epub by many readers).  Included below in output_css--will be
+## ignored if not in output_css.
+background_color: ffffff
+
+## Allow customization of CSS.  Make sure to keep at least one space
+## at the start of each line and to escape % to %%.  Also need
+## background_color to be in the same section, if included in CSS.
+output_css:
+ body { background-color: #%(background_color)s; }
+ .CI {
+     text-align:center;
+     margin-top:0px;
+     margin-bottom:0px;
+     padding:0px;
+     }
+ .center   {text-align: center;}
+ .cover    {text-align: center;}
+ .full     {width: 100%%; }
+ .quarter  {width: 25%%; }
+ .smcap    {font-variant: small-caps;}
+ .u        {text-decoration: underline;}
+ .bold     {font-weight: bold;}
+
+[txt]
+## Add URLs since there aren't links.
+titlepage_entries: series,category,genre,language,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,storyUrl, authorUrl, description
+
+## Width to word wrap text output.  0 indicates no wrapping.
+wrap_width: 78
+
+## use \r\n for line endings, the windows convention.  text output only.
+windows_eol: true
+
+[epub]
+
+## epub carries the TOC in metadata.
+## mobi generated from epub will have a TOC at the end.
+include_tocpage: false
+
+## epub->mobi conversions typically don't like tables.
+titlepage_use_table: false
+
+## When using tables, make these span both columns.
+wide_titlepage_entries: description, storyUrl, author URL
+
+## output background color--only used by html and epub (and ignored in
+## epub by many readers).  Included below in output_css--will be
+## ignored if not in output_css.
+background_color: ffffff
+
+## Allow customization of CSS.  Make sure to keep at least one space
+## at the start of each line and to escape % to %%.  Also need
+## background_color to be in the same section, if included in CSS.
+output_css: 
+ body { background-color: #%(background_color)s;
+        text-align: justify;
+        margin: 2%%; }
+ pre { font-size: x-small; }
+ sml { font-size: small; }
+ h1 { text-align: center; }
+ h2 { text-align: center; }
+ h3 { text-align: center; }
+ h4 { text-align: center; }
+ h5 { text-align: center; }
+ h6 { text-align: center; }
+ .CI {
+     text-align:center;
+     margin-top:0px;
+     margin-bottom:0px;
+     padding:0px;
+     }
+ .center   {text-align: center;}
+ .cover    {text-align: center;}
+ .full     {width: 100%%; }
+ .quarter  {width: 25%%; }
+ .smcap    {font-variant: small-caps;}
+ .u        {text-decoration: underline;}
+ .bold     {font-weight: bold;}
+
+## include images from img tags in the body and summary of
+## stories.  Images will be converted to jpg for size if possible.
+#include_images:false
+
+## If not set, the summary will have all html stripped for safety.
+## Both this and include_images must be true to get images in the
+## summary.
+#keep_summary_html:false
+
+## If set, the first image found will be made the cover image.  If
+## keep_summary_html is true, any images in summary will be before any
+## in chapters.
+#make_firstimage_cover: false
+
+## If set, the epub will never have a cover, even include_images is on
+## and the site has specific cover images.
+#never_make_cover: false
+
+## If set, and there isn't already a cover image from the adapter or
+## from make_firstimage_cover, this image will be made the cover.
+## It can be either a 'file:' or 'http:' url.
+## Note that if you enable make_firstimage_cover in [epub], but want
+## to use default_cover_image for a specific site, use the site:format
+## section, for example: [www.ficwad.com:epub]
+## default_cover_image is a python string Template string with
+## ${title}, ${author} etc, same as titlepage_entries. Unless
+## allow_unsafe_filename is true, invalid filename chars will be
+## removed from metadata fields
+#default_cover_image:file:///C:/Users/username/Desktop/nook/images/icon.png
+#default_cover_image:file:///C:/Users/username/Desktop/nook/images/${title}/icon.png
+#default_cover_image:http://www.somesite.com/someimage.gif
+
+## some sites include images that we don't ever want becoming the
+## cover image.  This lets you exclude them.
+#cover_exclusion_regexp:/stories/999/images/.*?_trophy.png
+
+## Resize images down to width, height, preserving aspect ratio.
+## Nook size, with margin.
+image_max_size: 580, 725
+
+## Change image to grayscale, if graphics library allows, to save
+## space.
+#grayscale_images: false
+
+## if the <img> tag doesn't have a div or a p around it, nook gets
+## confused and displays it on every page after that under the text
+## for the rest of the chapter.  I doubt adding a div around the img
+## will break any other readers, but in case it does, the fix can be
+## turned off.
+nook_img_fix:true
+
+[mobi]
+## mobi TOC cannot be turned off right now.
+#include_tocpage: true
+
+## Each site has a section that overrides [defaults] *and* the format
+## sections test1.com specifically is not a real story site.  Instead,
+## it is a fake site for testing configuration and output.  It uses
+## URLs like: http://test1.com?sid=12345
+[test1.com]
+extratags: FanFiction,Testing
+
+## If necessary, you can define [<site>:<format>] sections to
+## customize the formats differently for the same site.  Overrides
+## defaults, format and site.
+[test1.com:txt]
+extratags: FanFiction,Testing,Text
+
+[test1.com:html]
+extratags: FanFiction,Testing,HTML
+
+[archive.skyehawke.com]
+
+[ashwinder.sycophanthex.com]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+[castlefans.org]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
+[chaos.sycophanthex.com]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+## some sites include images that we don't ever want becoming the
+## cover image.  This lets you exclude them.
+cover_exclusion_regexp:/images/.*?ribbon.gif
+
+[dramione.org]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
+[erosnsappho.sycophanthex.com]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+## some sites include images that we don't ever want becoming the
+## cover image.  This lets you exclude them.
+cover_exclusion_regexp:/images/.*?ribbon.gif
+
+[fanfiction.mugglenet.com]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
+[fanfiction.portkey.org]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+[fanfiction.tenhawkpresents.com]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+[lumos.sycophanthex.com]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+[nfacommunity.com]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+[occlumency.sycophanthex.com]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+[onedirectionfanfiction.com]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
+[ponyfictionarchive.net]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+[samdean.archive.nu]
+
+[sg1-heliopolis.com]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+[stargate-atlantis.org]
+
+[thehexfiles.net]
+
+[thequidditchpitch.org]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
+[www.adastrafanfic.com]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+[www.thealphagate.com]
+
+[www.archiveofourown.org]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+[www.checkmated.com]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+[www.destinysgateway.com]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+[www.dokuga.com]
+
+[www.fanfiction.net]
+## fanfiction.net's 'cover' images are really just tiny thumbnails.
+## Comment this out or change it to false to use them anyway.
+never_make_cover: true
+
+[www.fanfiktion.de]
+
+[www.ficbook.net]
+
+[www.fictionalley.org]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+[www.fictionpress.com]
+## Clear FanFiction from defaults, fictionpress.com is original fiction.
+extratags:
+
+[www.ficwad.com]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+[www.fimfiction.net]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+## fimfiction.net stories can be locked requiring individual
+## passwords.  If fail_on_password is set, the downloader will fail
+## when a password is required rather than prompting every time.
+#fail_on_password: false
+
+[www.harrypotterfanfiction.com]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+[www.hpfandom.net]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+[www.ik-eternal.net]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
+[www.libraryofmoria.com]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+[www.mediaminer.org]
+
+[www.midnightwhispers.ca]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+## some sites include images that we don't ever want becoming the
+## cover image.  This lets you exclude them.
+cover_exclusion_regexp:/stories/999/images/.*?_trophy.png
+
+[www.ncisfiction.com]
+
+[www.phoenixsong.net]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+[www.potionsandsnitches.net]
+
+[www.prisonbreakfic.net]
+
+[www.siye.co.uk]
+
+[www.squidge.org/peja]
+# www.squidge.org/peja calls it Fandom <shrug>
+category_label:Fandom
+# Remove numWords -- www.squidge.org/peja word counts are inaccurate
+titlepage_entries: series,category,genre,language,characters,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,site,description
+
+[www.squidge.org/peja:txt]
+## Add URLs since there aren't links.
+# Remove numWords -- www.squidge.org/peja word counts are inaccurate
+titlepage_entries: series,category,genre,language,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,site,storyUrl, authorUrl, description
+
+[www.storiesofarda.com]
+
+[www.thewriterscoffeeshop.com]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
+## thewriterscoffeeshop.com (ab)uses series as personal reading lists.
+collect_series: false
+
+[www.tthfanfic.org]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+## tth is a little unusual--it doesn't require user/pass, but the site
+## keeps track of which chapters you've read and won't send another
+## update until it thinks you're up to date.  This way, on download,
+## it thinks you're up to date.
+#username:YourName
+#password:yourpassword
+
+[www.twilighted.net]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## twilighted.net (ab)uses series as personal reading lists.
+collect_series: false
+
+[www.twiwrite.net]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## twiwrite.net (ab)uses series as personal reading lists.
+collect_series: false
+
+[www.walkingtheplank.org]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+[www.whofic.com]
+
+[www.wraithbait.com]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+[www.yourfanfiction.com]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+[overrides]
+## It may sometimes be useful to override all of the specific format,
+## site and site:format sections in your private configuration.  For
+## example, this extratags param here would override all of the
+## extratags params in all other sections.  Only commandline options
+## beat overrides.
+#extratags:fanficdownloader
@@ -0,0 +1,97 @@
+## This is an example of what your personal configuration might look
+## like.
+
+[defaults]
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
+## include images from img tags in the body and summary of
+## stories.  Images will be converted to jpg for size if possible.
+#include_images:true
+
+## If not set, the summary will have all html stripped for safety.
+## Both this and include_images must be true to get images in the
+## summary.
+#keep_summary_html:true
+
+## If set, the first image found will be made the cover image.  If
+## keep_summary_html is true, any images in summary will be before any
+## in chapters.
+#make_firstimage_cover:true
+
+## Resize images down to width, height, preserving aspect ratio.
+## Nook size, with margin.
+#image_max_size: 580, 725
+
+## Change image to grayscale, if graphics library allows, to save
+## space.
+#grayscale_images: false
+
+## Try to collect series name and number of this story in series.
+## Some sites (ab)use 'series' for reading lists and personal
+## collections.  This lets us turn it on and off by site without
+## keeping a lengthy titlepage_entries per site and prevents it
+## updating in the plugin.
+## Turn off in [defaults] or [overrides] to prevent all sites from
+## updating series column.
+## default is true
+#collect_series: false
+
+## Most common, I expect will be using this to save username/passwords
+## for different sites.
+[www.twilighted.net]
+#username:YourPenname
+#password:YourPassword
+## default is false
+#collect_series: true
+
+[www.ficwad.com]
+#username:YourUsername
+#password:YourPassword
+
+[www.twiwrite.net]
+#username:YourName
+#password:yourpassword
+## default is false
+#collect_series: true
+
+[www.adastrafanfic.com]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.
+#is_adult:true
+
+[www.thewriterscoffeeshop.com]
+#username:YourName
+#password:yourpassword
+#is_adult:true
+## default is false
+#collect_series: true
+
+[www.fictionalley.org]
+#is_adult:true
+
+[www.harrypotterfanfiction.com]
+#is_adult:true
+
+[www.fimfiction.net]
+#is_adult:true
+#fail_on_password: false
+
+[www.tthfanfic.org]
+#is_adult:true
+## tth is a little unusual--it doesn't require user/pass, but the site
+## keeps track of which chapters you've read and won't send another
+## update until it thinks you're up to date.  This way, on download,
+## it thinks you're up to date.
+#username:YourName
+#password:yourpassword
+
+
+## This section will override anything in the system defaults or other
+## sections here.
+[overrides]
+## default varies by site.  Set true here to force all sites to
+## collect series.
+#collect_series: true
@@ -10,5 +10,10 @@ python downloader.py http://www.fanfiction.net/s/5192986/1/A_Fox_in_Tokyo

 Do 'python downloader.py -h' for more options.

-This tool uses Python 2.5.2, but should work with newer versions of Python.
+This tool uses Python 2.7, but should work with newer versions of Python.

+For more information, see:
+
+http://code.google.com/p/fanficdownloader/wiki/FanFictionDownloaderSupportedsites
+
+http://code.google.com/p/fanficdownloader/wiki/FanFictionDownloaderFAQs
--- a/Show More
+++ b/Show More