Extra metadata feature(refactoring), loosen url check on AO3, fix auth on castlefans.org.

Remove extra '[' from squidgeorgpeja Rating when no stars.
Added cover images for dramione.org, thanks ilovejedd.
2026-09-15 11:36:30 +08:00 · 2012-09-21 12:55:26 -05:00 · 2012-09-17 12:29:12 -05:00 · 2012-09-14 17:43:48 -05:00 · 2012-09-14 17:08:22 -05:00 · 2012-09-14 20:17:53 +00:00
90 changed files with 4852 additions and 630 deletions
@@ -1,6 +1,6 @@
 # ffd-retief-hrd fanfictiondownloader
 application: fanfictiondownloader
-version: 4-4-22
+version: 4-4-26
 runtime: python27
 api_version: 1
 threadsafe: true
@@ -27,7 +27,7 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
    description         = 'UI plugin to download FanFiction stories from various sites.'
    supported_platforms = ['windows', 'osx', 'linux']
    author              = 'Jim Miller'
-    version             = (1, 6, 3)
+    version             = (1, 6, 10)
    minimum_calibre_version = (0, 8, 57)

    #: This field defines the GUI plugin class that contains all the code
@@ -4,7 +4,7 @@
 '<a href="http://www.mobileread.com/forums/showthread.php?t=134856">Reading List</a>',
 '<a href="http://www.mobileread.com/forums/showthread.php?t=126727">Extract ISBN</a>' and
 '<a href="http://www.mobileread.com/forums/showthread.php?t=134000">Count Pages</a>'
-plugins.</p>
+plugins.  bbcodeutils code contributed by Pau Sanchez.</p>

 <p>
 Calibre officially distributes plugins from the mobileread.com forum site.
@@ -64,6 +64,7 @@ default_prefs['countpagesstats'] = []
 default_prefs['errorcol'] = ''
 default_prefs['custom_cols'] = {}
 default_prefs['custom_cols_newonly'] = {}
+default_prefs['allow_custcol_from_ini'] = True

 default_prefs['std_cols_newonly'] = {}

@@ -258,7 +259,7 @@ class ConfigWidget(QWidget):
        # error column
        prefs['errorcol'] = unicode(self.cust_columns_tab.errorcol.itemData(self.cust_columns_tab.errorcol.currentIndex()).toString())

-        # cust cols
+        # cust cols tab
        colsmap = {}
        for (col,combo) in self.cust_columns_tab.custcol_dropdowns.iteritems():
            val = unicode(combo.itemData(combo.currentIndex()).toString())
@@ -272,6 +273,8 @@ class ConfigWidget(QWidget):
            colsnewonly[col] = checkbox.isChecked()
        prefs['custom_cols_newonly'] = colsnewonly
        
+        prefs['allow_custcol_from_ini'] = self.cust_columns_tab.allow_custcol_from_ini.isChecked()
+        
        prefs.save_to_db()

    def edit_shortcuts(self):
@@ -437,7 +440,7 @@ class PersonalIniTab(QWidget):
        self.ini.setText(prefs['personal.ini'])
        self.l.addWidget(self.ini)

-        self.defaults = QPushButton('View Defaults', self)
+        self.defaults = QPushButton('View Defaults (plugin-defaults.ini)', self)
        self.defaults.setToolTip("View all of the plugin's configurable settings\nand their default settings.")
        self.defaults.clicked.connect(self.show_defaults)
        self.l.addWidget(self.defaults)
@@ -456,7 +459,7 @@ class ShowDefaultsIniDialog(QDialog):
        self.resize(600, 500)
        self.l = QVBoxLayout()
        self.setLayout(self.l)
-        self.label = QLabel("Plugin Defaults (Read-Only)")
+        self.label = QLabel("Plugin Defaults (plugin-defaults.ini) (Read-Only)")
        self.label.setToolTip("These are all of the plugin's configurable options\nand their default settings.")
        self.setWindowTitle(_('Plugin Defaults'))
        self.setWindowIcon(icon)
@@ -595,6 +598,8 @@ class GenerateCoverTab(QWidget):
            horz.addWidget(dropdown)
            self.sl.addLayout(horz)
        
+        self.sl.insertStretch(-1)
+        
        self.gcnewonly = QCheckBox("Run Generate Cover Only on New Books",self)
        self.gcnewonly.setToolTip("Default is to run GC any time the calibre metadata is updated.")
        self.gcnewonly.setChecked(prefs['gcnewonly'])
@@ -605,8 +610,6 @@ class GenerateCoverTab(QWidget):
        self.allow_gc_from_ini.setChecked(prefs['allow_gc_from_ini'])
        self.l.addWidget(self.allow_gc_from_ini)
            
-        self.l.insertStretch(-1)
-        
 class CountPagesTab(QWidget):

    def __init__(self, parent_dialog, plugin_action):
@@ -628,27 +631,27 @@ class CountPagesTab(QWidget):
        self.l.addSpacing(5)
        
        # 'PageCount', 'WordCount', 'FleschReading', 'FleschGrade', 'GunningFog'
-        self.pagecount = QCheckBox('PageCount',self)
+        self.pagecount = QCheckBox('Page Count',self)
        self.pagecount.setToolTip('Which column and algorithm to use are configured in Count Pages.')
        self.pagecount.setChecked('PageCount' in prefs['countpagesstats'])
        self.l.addWidget(self.pagecount)
            
-        self.wordcount = QCheckBox('WordCount',self)
+        self.wordcount = QCheckBox('Word Count',self)
        self.wordcount.setToolTip('Which column and algorithm to use are configured in Count Words.\nWill overwrite word count from FFDL metadata if set to update the same custom column.')
        self.wordcount.setChecked('WordCount' in prefs['countpagesstats'])
        self.l.addWidget(self.wordcount)

-        self.fleschreading = QCheckBox('FleschReading',self)
+        self.fleschreading = QCheckBox('Flesch Reading Ease',self)
        self.fleschreading.setToolTip('Which column and algorithm to use are configured in Count Pages.')
        self.fleschreading.setChecked('FleschReading' in prefs['countpagesstats'])
        self.l.addWidget(self.fleschreading)
        
-        self.fleschgrade = QCheckBox('Fleschgrade',self)
+        self.fleschgrade = QCheckBox('Flesch-Kincaid Grade Level',self)
        self.fleschgrade.setToolTip('Which column and algorithm to use are configured in Count Pages.')
-        self.fleschgrade.setChecked('Fleschgrade' in prefs['countpagesstats'])
+        self.fleschgrade.setChecked('FleschGrade' in prefs['countpagesstats'])
        self.l.addWidget(self.fleschgrade)
        
-        self.gunningfog = QCheckBox('GunningFog',self)
+        self.gunningfog = QCheckBox('Gunning Fog Index',self)
        self.gunningfog.setToolTip('Which column and algorithm to use are configured in Count Pages.')
        self.gunningfog.setChecked('GunningFog' in prefs['countpagesstats'])
        self.l.addWidget(self.gunningfog)
@@ -715,6 +718,7 @@ permitted_values = {
                     'language',
                     'series',
                     'characters',
+                     'ships',
                     'status',
                     'datePublished',
                     'dateUpdated',
@@ -751,6 +755,7 @@ titleLabels = {
    'status-I':'Status:In-Progress',
    'series':'Series',
    'characters':'Characters',
+    'ships':'Relationships',
    'datePublished':'Published',
    'dateUpdated':'Updated',
    'dateCreated':'Packaged',
@@ -793,6 +798,15 @@ class CustomColumnsTab(QWidget):
        self.custcol_dropdowns = {}
        self.custcol_newonlycheck = {}

+        scrollable = QScrollArea()
+        scrollcontent = QWidget()
+        scrollable.setWidget(scrollcontent)
+        scrollable.setWidgetResizable(True)
+        self.l.addWidget(scrollable)
+
+        self.sl = QVBoxLayout()
+        scrollcontent.setLayout(self.sl)
+        
        for key, column in custom_columns.iteritems():

            if column['datatype'] in permitted_values:
@@ -823,15 +837,20 @@ class CustomColumnsTab(QWidget):
                    newonlycheck.setChecked(prefs['custom_cols_newonly'][key])
                horz.addWidget(newonlycheck)
                
-                self.l.addLayout(horz)
+                self.sl.addLayout(horz)
        
-        self.l.insertStretch(-1)
+        self.sl.insertStretch(-1)

+        self.l.addSpacing(5)
+        self.allow_custcol_from_ini = QCheckBox('Allow custom_columns_settings from personal.ini to override',self)
+        self.allow_custcol_from_ini.setToolTip("The personal.ini parameter custom_columns_settings allows you to set custom columns to site specific values that aren't common to all sites.<br \>custom_columns_settings is ignored when this is off.")
+        self.allow_custcol_from_ini.setChecked(prefs['allow_custcol_from_ini'])
+        self.l.addWidget(self.allow_custcol_from_ini)
+        
        self.l.addSpacing(5)
        label = QLabel("Special column:")
        label.setWordWrap(True)
        self.l.addWidget(label)
-        self.l.addSpacing(5)
        
        horz = QHBoxLayout()
        label = QLabel("Update/Overwrite Error Column:")
@@ -847,8 +866,7 @@ class CustomColumnsTab(QWidget):
        self.errorcol.setCurrentIndex(self.errorcol.findData(QVariant(prefs['errorcol'])))
        horz.addWidget(self.errorcol)
        self.l.addLayout(horz)
-        
-        
+
        #print("prefs['custom_cols'] %s"%prefs['custom_cols'])


@@ -7,8 +7,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2012, Jim Miller'
 __docformat__ = 'restructuredtext en'

-import time, os, copy, threading, re
-from ConfigParser import SafeConfigParser
+import time, os, copy, threading, re, platform
 from StringIO import StringIO
 from functools import partial
 from datetime import datetime
@@ -28,6 +27,7 @@ from calibre.gui2 import error_dialog, warning_dialog, question_dialog, info_dia
 from calibre.gui2.dialogs.message_box import ViewLog
 from calibre.gui2.dialogs.confirm_delete import confirm
 from calibre.utils.date import local_tz
+from calibre.library.comments import sanitize_comments_html

 # The class that all interface action plugins must inherit from
 from calibre.gui2.actions import InterfaceAction
@@ -36,7 +36,7 @@ from calibre_plugins.fanfictiondownloader_plugin.common_utils import (set_plugin
                                         create_menu_action_unique, get_library_uuid)

 from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapters, writers, exceptions
-from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.htmlcleanup import stripHTML
+from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.configurable import Configuration
 from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.epubutils import get_dcsource, get_dcsource_chaptercount, get_story_url_from_html
 from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.geturls import get_urls_from_page

@@ -143,8 +143,6 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                                                       triggered=self.add_dialog )

            self.update_action = self.create_menu_item_ex(self.menu, '&Update Existing FanFiction Book(s)', image='plusplus.png',
-                                                          unique_name='Update Existing FanFiction Book(s)',
-                                                          shortcut_name='Update Existing FanFiction Book(s)',
                                                          triggered=self.update_existing)

            if 'Reading List' in self.gui.iactions and (prefs['addtolists'] or prefs['addtoreadlists']) :
@@ -164,37 +162,34 @@ class FanFictionDownLoaderPlugin(InterfaceAction):

                if addmenutxt:
                    self.add_send_action = self.create_menu_item_ex(self.menu, addmenutxt, image='plusplus.png',
-                                                                    unique_name=addmenutxt,
-                                                                    shortcut_name=addmenutxt,
                                                                    triggered=partial(self.update_lists,add=True))

                if rmmenutxt:
                    self.add_remove_action = self.create_menu_item_ex(self.menu, rmmenutxt, image='minusminus.png',
-                                                                      unique_name=rmmenutxt,
-                                                                      shortcut_name=rmmenutxt,
                                                                      triggered=partial(self.update_lists,add=False))
                
            self.menu.addSeparator()
            self.get_list_action = self.create_menu_item_ex(self.menu, 'Get URLs from Selected Books', image='bookmarks.png',
-                                                            unique_name='Get URLs from Selected Books',
-                                                            shortcut_name='Get URLs from Selected Books',
                                                            triggered=self.get_list_urls)

            self.get_list_url_action = self.create_menu_item_ex(self.menu, 'Get Story URLs from Web Page', image='view.png',
-                                                                unique_name='Get Story URLs from Web Page',
-                                                                shortcut_name='Get Story URLs from Web Page',
                                                                triggered=self.get_urls_from_page)

-            self.menu.addSeparator()
-            self.config_action = create_menu_action_unique(self, self.menu, '&Configure Plugin', shortcut=False,
-                                                           image= 'config.png',
-                                                           unique_name='Configure FanFictionDownLoader',
-                                                           triggered=partial(do_user_config,parent=self.gui))
+            # print("platform.system():%s"%platform.system())
+            # print("platform.mac_ver()[0]:%s"%platform.mac_ver()[0])
+            if not platform.mac_ver()[0]: # Some macs crash on these menu items for unknown reasons.
+                self.menu.addSeparator()
+                self.config_action = self.create_menu_item_ex(self.menu, '&Configure Plugin',
+                                                              image= 'config.png',
+                                                              unique_name='Configure FanFictionDownLoader',
+                                                              shortcut_name='Configure FanFictionDownLoader',
+                                                              triggered=partial(do_user_config,parent=self.gui))
            
-            self.about_action = create_menu_action_unique(self, self.menu, 'About Plugin', shortcut=False,
-                                                          image= 'images/icon.png',
-                                                          unique_name='About FanFictionDownLoader',
-                                                          triggered=self.about)
+                self.about_action = self.create_menu_item_ex(self.menu, 'About Plugin',
+                                                             image= 'images/icon.png',
+                                                             unique_name='About FanFictionDownLoader', 
+                                                             shortcut_name='About FanFictionDownLoader', 
+                                                             triggered=self.about)
            
            # Before we finalize, make sure we delete any actions for menus that are no longer displayed
            for menu_id, unique_name in self.old_actions_unique_map.iteritems():
@@ -220,10 +215,12 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
    def create_menu_item_ex(self, parent_menu, menu_text, image=None, tooltip=None,
                           shortcut=None, triggered=None, is_checked=None, shortcut_name=None,
                           unique_name=None):
+        #print("create_menu_item_ex before %s"%menu_text)
        ac = create_menu_action_unique(self, parent_menu, menu_text, image, tooltip,
                                       shortcut, triggered, is_checked, shortcut_name, unique_name)
        self.actions_unique_map[ac.calibre_shortcut_unique_name] = ac.calibre_shortcut_unique_name
        self.menu_actions.append(ac)
+        #print("create_menu_item_ex after %s"%menu_text)
        return ac

    def plugin_button(self):
@@ -251,10 +248,13 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
            return
        print("get_urls_from_page URL:%s"%d.url.text())

-        ffdlconfig = SafeConfigParser()
-        ffdlconfig.readfp(StringIO(get_resources("plugin-defaults.ini")))
-        ffdlconfig.readfp(StringIO(prefs['personal.ini']))
-        url_list = get_urls_from_page("%s"%d.url.text(),ffdlconfig)
+        if 'archiveofourown.org' in url:
+            configuration = Configuration(adapters.getConfigSectionFor(url),"EPUB")
+            configuration.readfp(StringIO(get_resources("plugin-defaults.ini")))
+            configuration.readfp(StringIO(options['personal.ini']))
+        else:
+            configuration = None
+        url_list = get_urls_from_page("%s"%d.url.text(),configuration)

        if url_list:
            d = ViewLog(_("List of URLs"),"\n".join(url_list),parent=self.gui)
@@ -435,13 +435,6 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
        print("url:%s"%url)
        skip_date_update = False
        
-        ## was self.ffdlconfig, but we need to be able to change it
-        ## when doing epub update.
-        ffdlconfig = SafeConfigParser()
-        ffdlconfig.readfp(StringIO(get_resources("plugin-defaults.ini")))
-        ffdlconfig.readfp(StringIO(prefs['personal.ini']))
-        adapter = adapters.getAdapter(ffdlconfig,url,fileform)
-
        options['personal.ini'] = prefs['personal.ini']
        if prefs['includeimages']:
            # this is a cheat to make it easier for users.
@@ -451,6 +444,11 @@ keep_summary_html:true
 make_firstimage_cover:true
 ''' + options['personal.ini']

+        configuration = Configuration(adapters.getConfigSectionFor(url),fileform)
+        configuration.readfp(StringIO(get_resources("plugin-defaults.ini")))
+        configuration.readfp(StringIO(options['personal.ini']))
+        adapter = adapters.getAdapter(configuration,url)
+
        ## three tries, that's enough if both user/pass & is_adult needed,
        ## or a couple tries of one or the other
        for x in range(0,2):
@@ -472,15 +470,15 @@ make_firstimage_cover:true

        # let other exceptions percolate up.
        story = adapter.getStoryMetadataOnly()
-        writer = writers.getWriter(options['fileform'],adapter.config,adapter)
+        writer = writers.getWriter(options['fileform'],configuration,adapter)

        book['all_metadata'] = story.getAllMetadata(removeallentities=True)
        book['title'] = story.getMetadata("title", removeallentities=True)
        book['author_sort'] = book['author'] = story.getList("author", removeallentities=True)
        book['publisher'] = story.getMetadata("site")
-        book['tags'] = writer.getTags() # getTags could be moved up into adapter now.  Adapter didn't used to know the fileform
-        book['comments'] = stripHTML(story.getMetadata("description")) #, removeallentities=True) comments handles entities better.
-        book['series'] = story.getMetadata("series")
+        book['tags'] = story.getSubjectTags(removeallentities=True)
+        book['comments'] = sanitize_comments_html(story.getMetadata("description"))        
+        book['series'] = story.getMetadata("series", removeallentities=True)
        
        # adapter.opener is the element with a threadlock.  But del
        # adapter.opener doesn't work--subproc fails when it tries
@@ -777,20 +775,20 @@ make_firstimage_cover:true
 <p>Proceed with updating your library?</p>
 '''%(len(good_list),len(bad_list))

-        htmllog='<html><body><table border="1"><tr><th>Status</th><th>Title</th><th>Author</th><th>URL</th><th>Comment</th></tr>'
+        htmllog='<html><body><table border="1"><tr><th>Status</th><th>Title</th><th>Author</th><th>Comment</th><th>URL</th></tr>'
        for book in good_list:
            if 'status' in book:
                status = book['status']
            else:
                status = 'Good'
-            htmllog = htmllog + '<tr><td>' + '</td><td>'.join([status,book['title'],", ".join(book['author']),book['url'],book['comment']]) + '</td></tr>'
+            htmllog = htmllog + '<tr><td>' + '</td><td>'.join([status,book['title'],", ".join(book['author']),book['comment'],book['url']]) + '</td></tr>'
        
        for book in bad_list:
            if 'status' in book:
                status = book['status']
            else:
                status = 'Bad'
-            htmllog = htmllog + '<tr><td>' + '</td><td>'.join([status,book['title'],", ".join(book['author']),book['url'],book['comment']]) + '</td></tr>'
+            htmllog = htmllog + '<tr><td>' + '</td><td>'.join([status,book['title'],", ".join(book['author']),book['comment'],book['url']]) + '</td></tr>'
        
        htmllog = htmllog + '</table></body></html>'
        
@@ -902,18 +900,6 @@ make_firstimage_cover:true
                db.set_link_field_for_author(autid, unicode(authurls[i]),
                                             commit=False, notify=False)

-        # mi.title = oldmi.title
-        # mi.authors = oldmi.authors
-        # mi.publisher = oldmi.publisher
-        # mi.tags = oldmi.tags
-        # mi.languages = oldmi.languages
-        # mi.pubdate = oldmi.pubdate
-        # mi.timestamp = oldmi.timestamp
-        # mi.comments = oldmi.comments
-        # mi.series = oldmi.series
-                
-        # mi.set_identifiers(oldmi.get_identifiers())
-
        # implement 'newonly' flags here by setting to the current
        # value again.
        if not book['added']:
@@ -927,7 +913,7 @@ make_firstimage_cover:true
                        except AttributeError:
                            print("AttributeError? %s"%col)
                            pass
-            
+
        db.set_metadata(book_id,mi)

        # do configured column updates here.
@@ -964,6 +950,52 @@ make_firstimage_cover:true
                    val = book['all_metadata']['status'] == 'In-Progress'
                db.set_custom(book_id, val, label=label, commit=False)

+        adapter = None
+        if prefs['allow_custcol_from_ini']:
+            configuration = Configuration(adapters.getConfigSectionFor(book['url']),options['fileform'])
+            configuration.readfp(StringIO(get_resources("plugin-defaults.ini")))
+            configuration.readfp(StringIO(options['personal.ini']))
+            adapter = adapters.getAdapter(configuration,book['url'])
+
+            # meta => custcol[,a|n|r]
+            # cliches=>\#acolumn,r
+            for line in adapter.getConfig('custom_columns_settings').splitlines():
+                if "=>" in line:
+                    (meta,custcol) = map( lambda x: x.strip(), line.split("=>") )
+                    flag='r'
+                    if "," in custcol:
+                        (custcol,flag) = map( lambda x: x.strip(), custcol.split(",") )
+
+                    #print("meta:(%s) => custcol:(%s), flag(%s) "%(meta,custcol,flag))
+                    
+                    if meta not in book['all_metadata']:
+                        print("No value for %s, skipping custom column(%s) update."%(meta,custcol))
+                        continue
+                    
+                    if custcol not in custom_columns:
+                        print("No custom column(%s), skipping."%(custcol))
+                        continue
+                    else:
+                        coldef = custom_columns[custcol]
+                        label = coldef['label']
+                    
+                    if flag == 'r' or book['added']:
+                        db.set_custom(book_id, book['all_metadata'][meta], label=label, commit=False)
+
+                    if flag == 'a':
+                        try:
+                            existing=db.get_custom(book_id,label=label,index_is_id=True)
+                            if isinstance(existing,list):
+                                vallist = existing
+                            else :
+                                vallist = [existing]
+                            vallist.append(book['all_metadata'][meta])
+                        except:
+                            vallist = [book['all_metadata'][meta]]
+                            
+                        db.set_custom(book_id, ", ".join(vallist), label=label, commit=False)
+            
+                
        db.commit()

        if 'Generate Cover' in self.gui.iactions and (book['added'] or not prefs['gcnewonly']):
@@ -975,10 +1007,11 @@ make_firstimage_cover:true
            gc_plugin = self.gui.iactions['Generate Cover']
            setting_name = None
            if prefs['allow_gc_from_ini']:
-                ffdlconfig = SafeConfigParser()
-                ffdlconfig.readfp(StringIO(get_resources("plugin-defaults.ini")))
-                ffdlconfig.readfp(StringIO(prefs['personal.ini']))
-                adapter = adapters.getAdapter(ffdlconfig,book['url'],options['fileform'])
+                if not adapter: # might already have it from allow_custcol_from_ini
+                    configuration = Configuration(adapters.getConfigSectionFor(book['url']),options['fileform'])
+                    configuration.readfp(StringIO(get_resources("plugin-defaults.ini")))
+                    configuration.readfp(StringIO(options['personal.ini']))
+                    adapter = adapters.getAdapter(configuration,book['url'])

                # template => regexp to match => GC Setting to use.
                # generate_cover_settings:
@@ -987,7 +1020,7 @@ make_firstimage_cover:true
                    if "=>" in line:
                        (template,regexp,setting) = map( lambda x: x.strip(), line.split("=>") )
                        value = Template(template).safe_substitute(book['all_metadata']).encode('utf8')
-                        print("%s(%s) => %s => %s"%(template,value,regexp,setting))
+                        # print("%s(%s) => %s => %s"%(template,value,regexp,setting))
                        if re.search(regexp,value):
                            setting_name = setting
                            break
@@ -1206,11 +1239,11 @@ make_firstimage_cover:true
        return None

    def _is_good_downloader_url(self,url):
-        # this is the accepted way to 'check for existance'?  really?
+        # this is the accepted way to 'check for existance of a class variable'?  really?
        try:
            self.dummyconfig
        except AttributeError:
-            self.dummyconfig = SafeConfigParser()
+            self.dummyconfig = Configuration("test1.com","EPUB")
        # pulling up an adapter is pretty low over-head.  If
        # it fails, it's a bad url.
        try:
@@ -10,7 +10,6 @@ __docformat__ = 'restructuredtext en'

 import time, os, traceback

-from ConfigParser import SafeConfigParser
 from StringIO import StringIO

 from calibre.utils.ipc.server import Server
@@ -20,6 +19,7 @@ from calibre.utils.logging import Log
 from calibre_plugins.fanfictiondownloader_plugin.dialogs import (NotGoingToDownload,
    OVERWRITE, OVERWRITEALWAYS, UPDATE, UPDATEALWAYS, ADDNEW, SKIP, CALIBREONLY)
 from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapters, writers, exceptions
+from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.configurable import Configuration
 from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.epubutils import get_update_data

 # ------------------------------------------------------------------------------
@@ -109,16 +109,24 @@ def do_download_for_worker(book,options):
    when run as a worker job
    '''
    try:
+        # import logging
+        # logging.basicConfig(level=logging.DEBUG,format="%(levelname)s:%(filename)s(%(lineno)d):%(message)s")
+        
        book['comment'] = 'Download started...'

-        ffdlconfig = SafeConfigParser()
-        ffdlconfig.readfp(StringIO(get_resources("plugin-defaults.ini")))
-        ffdlconfig.readfp(StringIO(options['personal.ini']))
+        configuration = Configuration(adapters.getConfigSectionFor(book['url']),options['fileform'])
+        configuration.readfp(StringIO(get_resources("plugin-defaults.ini")))
+        configuration.readfp(StringIO(options['personal.ini']))
        
        if not options['updateepubcover'] and 'epub_for_update' in book and options['collision'] in (UPDATE, UPDATEALWAYS):
-            ffdlconfig.set("overrides","never_make_cover","true")
+            configuration.set("overrides","never_make_cover","true")
+
+        # images only for epub, even if the user mistakenly turned it
+        # on else where.
+        if options['fileform'] != "epub":
+            configuration.set("overrides","include_images","false")
        
-        adapter = adapters.getAdapter(ffdlconfig,book['url'],options['fileform'])
+        adapter = adapters.getAdapter(configuration,book['url'])
        adapter.is_adult = book['is_adult'] 
        adapter.username = book['username'] 
        adapter.password = book['password']
@@ -129,7 +137,7 @@ def do_download_for_worker(book,options):
            adapter.setSeries(book['calibre_series'][0],book['calibre_series'][1])
        # else:
        #     print("no calibre_series")
-        writer = writers.getWriter(options['fileform'],adapter.config,adapter)
+        writer = writers.getWriter(options['fileform'],configuration,adapter)

        outfile = book['outfile']

@@ -18,6 +18,10 @@
 ## [defaults] section applies to all formats and sites but may be
 ## overridden at several levels

+## Some sites also require the user to confirm they are adult for
+## adult content.  Uncomment by removing '#' in front of is_adult.
+#is_adult:true
+
 ## All available titlepage_entries and the label used for them:
 ## <entryname>_label:<label>
 ## Labels may be customized.
@@ -38,6 +42,7 @@ category_label:Category
 genre_label:Genre
 language_label:Language
 characters_label:Characters
+ships_label:Relationships
 series_label:Series
 ## Completed/In-Progress
 status_label:Status
@@ -62,15 +67,21 @@ authorId_label:Author ID
 ## show up in Calibre as tags.  Also carried into mobi when converted.
 extratags_label:Extra Tags
 ## The version of fanficdownloader
-##
 version_label:FFDL Version

+## Date formats used by FFDL.  Published and Update don't have time.
+## See http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+## Note that ini format requires % to be escaped as %%.
+dateCreated_format:%%Y-%%m-%%d %%H:%%M:%%S
+datePublished_format:%%Y-%%m-%%d
+dateUpdated_format:%%Y-%%m-%%d
+
 ## items to include in the title page
 ## Empty metadata entries will *not* appear, even if in the list.
 ## You can include extra text or HTML that will be included as-is in
 ## the title page. Eg: titlepage_entries: ...,<br />,summary,<br />,...
 ## All current formats already include title and author.
-titlepage_entries: series,category,genre,language,characters,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,description
+titlepage_entries: series,category,genre,language,characters,ships,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,description

 ## Try to collect series name and number of this story in series.
 ## Some sites (ab)use 'series' for reading lists and personal
@@ -121,11 +132,20 @@ allow_unsafe_filename: false

 ## entries to make epub subjects and calibre tags
 ## lastupdate creates two tags: "Last Update Year/Month: %Y/%m" and "Last Update: %Y/%m/%d"
-include_subject_tags: extratags, genre, category, characters, lastupdate, status
+include_subject_tags: extratags, genre, category, characters, ships, lastupdate, status

 ## extra tags (comma separated) to include, primarily for epub.
 extratags: FanFiction

+## extra categories, genres, characters, ships and warnings can be
+## configured.  Used primarily for sites that are dedicated to a genre
+## or 'ship and so don't included it for every story.
+#extracategories:
+#extragenres:
+#extracharacters:
+#extraships:
+#extrawarnings:
+
 ## number of seconds to sleep between calls to the story site.  May by
 ## useful if pulling large numbers of stories or if the site is slow.
 #slow_down_sleep_time:0.5
@@ -154,6 +174,45 @@ extratags: FanFiction
 ## doesn't work on some devices either.)
 #replace_hr: false

+## If set false, the summary will have all html stripped.
+## Both this and include_images must be true to get images in the
+## summary.
+keep_summary_html:true
+
+## Don't like the numbers at the start of chapter titles on some
+## sites?  You can use strip_chapter_numbers to strip them off.  Just
+## want to make them all look the same?  Strip them off, then add them
+## back on with add_chapter_numbers.  Don't like the way it strips
+## numbers or adds them back?  See chapter_title_strip_pattern and
+## chapter_title_add_pattern.
+strip_chapter_numbers:false
+add_chapter_numbers:false
+
+## (Two versions of chapter_title_strip_pattern are shown below.  You
+## should only have one uncommented.)
+## This version will remove the leading number from:
+## "1." => ""
+## "1. The Beginning" => "The Beginning"
+## "1: Start" => "Start"
+## "2, Chapter the second" => "Chapter the second"
+## etc
+chapter_title_strip_pattern:^[0-9]+[\.: -]+
+
+## This version will strip all of the above *plus* remove 'Chapter 1':
+## "Chapter 1" => ""
+## "1. Chapter 1" => ""
+## "1. Chapter 1, Bob's First Clue" => "Bob's First Clue"
+## "Chapter 2 - Pirates Place" => "Pirates Place"
+## etc
+#chapter_title_strip_pattern:^([0-9]+[\.: -]+)?(Chapter *[0-9]+[\.:, -]*)?
+
+## Uses a python template substitution.  The ${index} is the 'chapter'
+## number and ${title} is the chapter title, after applying
+## chapter_title_strip_pattern.  Those are the only variables available.
+## "The Beginning" => "1. The Beginning" 
+chapter_title_add_pattern:${index}. ${title}
+
+
 ## Each output format has a section that overrides [defaults]
 [html]

@@ -229,10 +288,12 @@ background_color: ffffff
 ## Allow customization of CSS.  Make sure to keep at least one space
 ## at the start of each line and to escape % to %%.  Also need
 ## background_color to be in the same section, if included in CSS.
+## 'adobe-text-layout: optimizeSpeed;' prevents hyphenation on newer Nooks
 output_css: 
 body { background-color: #%(background_color)s;
        text-align: justify;
-        margin: 2%%; }
+        margin: 2%%;
+	adobe-text-layout: optimizeSpeed; }
 pre { font-size: x-small; }
 sml { font-size: small; }
 h1 { text-align: center; }
@@ -259,11 +320,6 @@ output_css:
 ## stories.  Images will be converted to jpg for size if possible.
 #include_images:false

-## If not set, the summary will have all html stripped for safety.
-## Both this and include_images must be true to get images in the
-## summary.
-#keep_summary_html:false
-
 ## If set, the first image found will be made the cover image.  If
 ## keep_summary_html is true, any images in summary will be before any
 ## in chapters.
@@ -316,6 +372,11 @@ nook_img_fix:true
 ## URLs like: http://test1.com?sid=12345
 [test1.com]
 extratags: FanFiction,Testing
+extracategories:Fafner
+extragenres:Romance,Fluff
+extracharacters:Reginald Smythe-Smythe,Mokona
+extraships:Smythe-Smythe/Mokona
+extrawarnings:Extreme Bogosity

 ## If necessary, you can define [<site>:<format>] sections to
 ## customize the formats differently for the same site.  Overrides
@@ -342,6 +403,11 @@ extratags: FanFiction,Testing,HTML
 #is_adult:true

 [ashwinder.sycophanthex.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+extracharacters:Severus Snape,Hermione Granger
+extraships:Severus Snape/Hermione Granger
+
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -349,7 +415,26 @@ extratags: FanFiction,Testing,HTML
 #username:YourName
 #password:yourpassword

+[bloodties-fans.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Blood Ties
+
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
 [castlefans.org]
+## Site dedicated to these categories/characters/ships
+extracategories:Castle
+
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -373,6 +458,11 @@ extratags: FanFiction,Testing,HTML
 cover_exclusion_regexp:/images/.*?ribbon.gif

 [dark-solace.org]
+## Site dedicated to these categories/characters/ships
+extracategories:Buffy: The Vampire Slayer
+extracharacters:Buffy, Spike
+extraships:Spike/Buffy
+
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -381,6 +471,11 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 #password:yourpassword

 [dramione.org]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+extracharacters:Draco Malfoy,Hermione Granger
+extraships:Draco Malfoy/Hermione Granger
+
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -393,7 +488,43 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 ## personal.ini, not defaults.ini.
 #is_adult:true

+## Some adapters collect additional meta information beyond the
+## standard ones.  They need to be defined in extra_valid_entries to
+## tell the rest of the FFDL system about them.  They can be used in
+## include_subject_tags, titlepage_entries, extra_titlepage_entries,
+## logpage_entries, extra_logpage_entries, and include_in_* config
+## items.  You can also add additional entries here to build up
+## composite metadata entries.  dramione.org, for example, adds
+## 'cliches' and then defines as the composite of hermiones,dracos in
+## include_in_cliches.
+extra_valid_entries:themes,hermiones,dracos,timeline,cliches
+include_in_cliches:hermiones,dracos
+
+## For another example, you could, by uncommenting this line, include
+## themes in with genre metadata.  Note, however, that you couldn't
+## use cliches.  include_in_* can only include the original real
+## entries, not other include_in_* values.
+#include_in_genre:genre, themes
+
+## You can give each new valid entry a specific label for use on
+## titlepage and logpage.  If not defined, it will simply be the 
+themes_label:Themes
+hermiones_label:Hermiones
+dracos_label:Dracos
+timeline_label:Timeline
+cliches_label:Character Cliches
+
+## extra_titlepage_entries (and extra_logpage_entries) *add* to
+## titlepage_entries (and logpage_entries) so you can add site
+## specific entries to titlepage/logpage without having to copy the
+## entire titlepage_entries line.  (But if you want them higher than
+## the end, you will need to copy titlepage_entries.)
+extra_titlepage_entries: themes,hermiones,dracos,timeline,cliches
+
 [erosnsappho.sycophanthex.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
@@ -404,6 +535,9 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 cover_exclusion_regexp:/images/.*?ribbon.gif

 [fanfiction.mugglenet.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -417,6 +551,10 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 #is_adult:true

 [fanfiction.portkey.org]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+extraships:Harry Potter/Hermione Granger
+
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
@@ -439,6 +577,10 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 #password:yourpassword

 [grangerenchanted.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+extracharacters:Hermione Granger
+
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -452,26 +594,67 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 #is_adult:true

 [hlfiction.net]
+## Site dedicated to these categories/characters/ships
+extracategories:Highlander
+
+[ksarchive.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Star Trek
+extracharacters:Kirk,Spock
+extraships:Kirk/Spock

 [lumos.sycophanthex.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true

+[merlinfic.dtwins.co.uk]
+## Site dedicated to these categories/characters/ships
+extracategories:Merlin
+
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
 [national-library.net]
+## Site dedicated to these categories/characters/ships
+extracategories:West Wing

 [ncisfic.com]
+## Site dedicated to these categories/characters/ships
+extracategories:NCIS

 [nfacommunity.com]
+## Site dedicated to these categories/characters/ships
+extracategories:NCIS
+
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true

 [nha.magical-worlds.us]
+## Site dedicated to these categories/characters/ships
+extracategories:Buffy: The Vampire Slayer
+extracharacters:Willow

 [occlumency.sycophanthex.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+extracharacters:Severus Snape
+
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -480,6 +663,9 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 #password:yourpassword

 [onedirectionfanfiction.com]
+## Site dedicated to these categories/characters/ships
+extracategories:One Direction
+
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -493,18 +679,28 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 #is_adult:true

 [ponyfictionarchive.net]
+## Site dedicated to these categories/characters/ships
+extracategories:My Little Pony: Friendship is Magic
+
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true

 [pretendercentre.com]
+## Site dedicated to these categories/characters/ships
+extracategories:The Pretender
+
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true

 [samdean.archive.nu]
+## Site dedicated to these categories/characters/ships
+extracategories:Supernatural
+extracharacters:Sam,Dean
+extraships:Sam/Dean

 [sg1-heliopolis.com]
 ## Some sites do not require a login, but do require the user to
@@ -513,6 +709,8 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 #is_adult:true

 [stargate-atlantis.org]
+## Site dedicated to these categories/characters/ships
+extracategories:Stargate: Atlantis

 [svufiction.com]
 ## Some sites require login (or login for some rated stories) The
@@ -528,6 +726,26 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 #is_adult:true

 [thehexfiles.net]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+extracharacters:Draco Malfoy,Harry Potter
+extraships:Harry Potter/Draco Malfoy
+
+[thehookupzone.net]
+## Site dedicated to these categories/characters/ships
+extracategories:Criminal Minds
+
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true

 [themasque.net]
 ## Some sites require login (or login for some rated stories) The
@@ -543,6 +761,9 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 #is_adult:true

 [thequidditchpitch.org]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -556,14 +777,40 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 #is_adult:true

 [www.adastrafanfic.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Star Trek
+
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true

+[www.dracoandginny.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+extracharacters:Draco Malfoy,Ginny Weasley
+extraships:Draco Malfoy/Ginny Weasley
+
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
 [www.thealphagate.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Stargate: SG-1

 [www.checkmated.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -578,12 +825,20 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 #is_adult:true

 [www.dokuga.com]
+## Site dedicated to these categories/characters/ships
+extracategories:InuYasha
+extracharacters:Sesshoumaru,Kagome
+extraships:Sesshoumaru/Kagome

 [www.fanfiction.net]
 ## fanfiction.net's 'cover' images are really just tiny thumbnails.
-## Comment this out or change it to false to use them anyway.
+## Change this to false to use them anyway.
 never_make_cover: true

+## Extra metadata that this adapter knows about.  See [dramione.org]
+## for examples of how to use them.
+extra_valid_entries:reviews,favs,follows
+
 [www.fanfiktion.de]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -595,6 +850,9 @@ never_make_cover: true
 [www.ficbook.net]

 [www.fictionalley.org]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
@@ -607,6 +865,10 @@ output_filename: ${title}-${siteabbrev}_${authorId}_${storyId}${formatext}
 ## Clear FanFiction from defaults, fictionpress.com is original fiction.
 extratags:

+## Extra metadata that this adapter knows about.  See [dramione.org]
+## for examples of how to use them.
+extra_valid_entries:reviews,favs,follows
+
 [www.ficwad.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -616,6 +878,9 @@ extratags:
 #password:yourpassword

 [www.fimfiction.net]
+## Site dedicated to these categories/characters/ships
+extracategories:My Little Pony: Friendship is Magic
+
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
@@ -627,20 +892,33 @@ extratags:
 #fail_on_password: false

 [www.harrypotterfanfiction.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true

 [www.hpfandom.net]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true

 [www.hpfanficarchive.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter

 [www.ik-eternal.net]
+## Site dedicated to these categories/characters/ships
+extracategories:InuYasha
+extracharacters:InuYasha,Kagome
+extraships:InuYasha/Kagome
+
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -654,6 +932,9 @@ extratags:
 #is_adult:true

 [www.libraryofmoria.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Lord of the Rings
+
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
@@ -662,6 +943,9 @@ extratags:
 [www.mediaminer.org]

 [www.midnightwhispers.ca]
+## Site dedicated to these categories/characters/ships
+extracategories:Queer as Folk
+
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
@@ -672,8 +956,14 @@ extratags:
 cover_exclusion_regexp:/stories/999/images/.*?_trophy.png

 [www.ncisfiction.com]
+## Site dedicated to these categories/characters/ships
+extracategories:NCIS

 [www.phoenixsong.net]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+extraships:Harry Potter/Ginny Weasley
+
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -682,16 +972,43 @@ cover_exclusion_regexp:/stories/999/images/.*?_trophy.png
 #password:yourpassword

 [www.potionsandsnitches.net]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter

 [www.prisonbreakfic.net]
+## Site dedicated to these categories/characters/ships
+extracategories:Prison Break
+
+[www.scarvesandcoffee.net]
+## Site dedicated to these categories/characters/ships
+extracategories:Glee
+extracharacters:Kurt Hummel,Blaine Anderson
+
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+[www.sinful-desire.org]
+## Site dedicated to these categories/characters/ships
+extracategories:Supernatural
+
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true

 [www.siye.co.uk]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+extracharacters:Harry Potter,Ginny Weasley
+extraships:Harry Potter/Ginny Weasley

 [www.squidge.org/peja]
 # www.squidge.org/peja calls it Fandom <shrug>
 category_label:Fandom
 # Remove numWords -- www.squidge.org/peja word counts are inaccurate
-titlepage_entries: series,category,genre,language,characters,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,site,description
+titlepage_entries: series,category,genre,language,characters,ships,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,site,description

 [www.squidge.org/peja:txt]
 ## Add URLs since there aren't links.
@@ -699,6 +1016,16 @@ titlepage_entries: series,category,genre,language,characters,status,datePublishe
 titlepage_entries: series,category,genre,language,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,site,storyUrl, authorUrl, description

 [www.storiesofarda.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Lord of the Rings
+
+[www.thepetulantpoetess.com]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword

 [www.thewriterscoffeeshop.com]
 ## Some sites require login (or login for some rated stories) The
@@ -730,8 +1057,13 @@ collect_series: false
 #password:yourpassword

 [www.twilightarchives.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Twilight

 [www.twilighted.net]
+## Site dedicated to these categories/characters/ships
+extracategories:Twilight
+
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -743,6 +1075,9 @@ collect_series: false
 collect_series: false

 [www.twiwrite.net]
+## Site dedicated to these categories/characters/ships
+extracategories:Twilight
+
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -754,6 +1089,11 @@ collect_series: false
 collect_series: false

 [www.walkingtheplank.org]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+extracharacters:Severus Snape,Harry Potter
+extraships:Severus Snape/Harry Potter
+
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
@@ -774,7 +1114,15 @@ collect_series: false
 ## personal.ini, not defaults.ini.
 #is_adult:true

+[www.wolverineandrogue.com]
+## Site dedicated to these categories/characters/ships
+extracategories:X-Men Movie
+extracharacters:Wolverine,Rogue
+
 [www.wraithbait.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Stargate: Atlantis
+
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
@@ -27,6 +27,7 @@ import ConfigParser
 from subprocess import call

 from fanficdownloader import adapters,writers,exceptions
+from fanficdownloader.configurable import Configuration
 from fanficdownloader.epubutils import get_dcsource_chaptercount, get_update_data
 from fanficdownloader.geturls import get_urls_from_page

@@ -90,7 +91,15 @@ def main():
   if options.update and options.format != 'epub':
       parser.error("-u/--update-epub only works with epub")

-   config = ConfigParser.SafeConfigParser()
+   ## Attempt to update an existing epub.
+   if options.update:
+       (url,chaptercount) = get_dcsource_chaptercount(args[0])
+       print "Updating %s, URL: %s" % (args[0],url)
+       output_filename = args[0]
+   else:
+       url = args[0]
+           
+   configuration = Configuration(adapters.getConfigSectionFor(url),options.format)

   conflist = []
   homepath = join(expanduser("~"),".fanficdownloader")
@@ -109,41 +118,43 @@ def main():
       conflist.extend(options.configfile)
      
   logging.debug('reading %s config file(s), if present'%conflist)
-   config.read(conflist)
+   configuration.read(conflist)
+
+   print("has include_in_tags?%s"%configuration.hasConfig("include_in_tags"))

   try:
-       config.add_section("overrides")
+       configuration.add_section("overrides")
   except ConfigParser.DuplicateSectionError:
       pass

   if options.force:
-       config.set("overrides","always_overwrite","true")
+       configuration.set("overrides","always_overwrite","true")

+   if options.update:
+       configuration.set("overrides","output_filename",args[0])
+       
   if options.update and not options.updatecover:
-       config.set("overrides","never_make_cover","true")
+       configuration.set("overrides","never_make_cover","true")
+
+   # images only for epub, even if the user mistakenly turned it
+   # on else where.
+   if options.format != "epub":
+       configuration.set("overrides","include_images","false")
       
   if options.options:
       for opt in options.options:
           (var,val) = opt.split('=')
-           config.set("overrides",var,val)
+           configuration.set("overrides",var,val)

   if options.list:
-       retlist = get_urls_from_page(args[0], config)
+       retlist = get_urls_from_page(args[0], configuration)
       print "\n".join(retlist)
               
       return

   try:
-       ## Attempt to update an existing epub.
-       if options.update:
-           (url,chaptercount) = get_dcsource_chaptercount(args[0])
-           print "Updating %s, URL: %s" % (args[0],url)
-           output_filename = args[0]
-           config.set("overrides","output_filename",args[0])
-       else:
-           url = args[0]

-       adapter = adapters.getAdapter(config,url,options.format)
+       adapter = adapters.getAdapter(configuration,url)

       ## Check for include_images and absence of PIL, give warning.
       if adapter.getConfig('include_images'):
@@ -201,7 +212,7 @@ def main():
                    adapter.calibrebookmark,
                    adapter.logfile) = get_update_data(args[0])

-                   writeStory(config,adapter,"epub")
+                   writeStory(configuration,adapter,"epub")
                   
       else:
           # regular download
@@ -210,7 +221,7 @@ def main():
           
           adapter.setChaptersRange(options.begin,options.end)
           
-           output_filename=writeStory(config,adapter,options.format,options.metaonly)
+           output_filename=writeStory(configuration,adapter,options.format,options.metaonly)
       
       if not options.metaonly and adapter.getConfig("post_process_cmd"):
           metadata = adapter.story.metadata
@@ -1,40 +1,103 @@
 ## This is an example of what your personal configuration might look
-## like.
+## like.  Uncomment options by removing the '#' in front of them.

 [defaults]
 ## Some sites also require the user to confirm they are adult for
-## adult content.  In commandline version, this should go in your
-## personal.ini, not defaults.ini.
+## adult content.  Uncomment by removing '#' in front of is_adult.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
 #is_adult:true

+## Don't like the numbers at the start of chapter titles on some
+## sites?  You can use strip_chapter_numbers to strip them off.  Just
+## want to make them all look the same?  Strip them off, then add them
+## back on with add_chapter_numbers.  Don't like the way it strips
+## numbers or adds them back?  See chapter_title_strip_pattern and
+## chapter_title_add_pattern.
+#strip_chapter_numbers:true
+#add_chapter_numbers:true
+
+[epub]
+## include images from img tags in the body and summary of stories.
+## Images will be converted to jpg for size if possible.  Images work
+## in epub format only.  To get mobi or other format with images,
+## download as epub and use Calibre to convert.
+#include_images:true
+
+## If not set, the summary will have all html stripped for safety.
+## Both this and include_images must be true to get images in the
+## summary.
+#keep_summary_html:true
+
+## If set, the first image found will be made the cover image.  If
+## keep_summary_html is true, any images in summary will be before any
+## in chapters.
+#make_firstimage_cover:true
+
+## Resize images down to width, height, preserving aspect ratio.
+## Nook size, with margin.
+#image_max_size: 580, 725
+
+## Change image to grayscale, if graphics library allows, to save
+## space.
+#grayscale_images: false
+
+
 ## Most common, I expect will be using this to save username/passwords
-## for different sites.
+## for different sites.  Here are a few examples.  See defaults.ini
+## for the full list.
+
 [www.twilighted.net]
 #username:YourPenname
 #password:YourPassword
+## default is false
+#collect_series: true

 [www.ficwad.com]
 #username:YourUsername
 #password:YourPassword

+[www.twiwrite.net]
+#username:YourName
+#password:yourpassword
+## default is false
+#collect_series: true
+
 [www.adastrafanfic.com]
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.
 #is_adult:true

-## The [defaults] section here will override the system [defaults],
-## but not format, site for site:format sections.
-[defaults]
-## Directories only useful in commandline or zip files.
-#output_filename: books/${title}-${siteabbrev}_${storyId}${formatext}
-#output_filename: books/${site}/${authorId}/${title}-${storyId}${formatext}
+[www.thewriterscoffeeshop.com]
+#username:YourName
+#password:yourpassword
+#is_adult:true
+## default is false
+#collect_series: true
+
+[www.fictionalley.org]
+#is_adult:true
+
+[www.harrypotterfanfiction.com]
+#is_adult:true
+
+[www.fimfiction.net]
+#is_adult:true
+#fail_on_password: false
+
+[www.tthfanfic.org]
+#is_adult:true
+## tth is a little unusual--it doesn't require user/pass, but the site
+## keeps track of which chapters you've read and won't send another
+## update until it thinks you're up to date.  This way, on download,
+## it thinks you're up to date.
+#username:YourName
+#password:yourpassword

-## For example, zip_output here will turn on zip for html and txt, but
-## not epub because the system [epub] section explicitly says
-## zip_output: false (epubs *are* specially formated zip files.)
-#zip_output: true
-#zip_filename: ${title}-${siteabbrev}_${storyId}${formatext}.zip

 ## This section will override anything in the system defaults or other
 ## sections here.
 [overrides]
+## default varies by site.  Set true here to force all sites to
+## collect series.
+#collect_series: true
@@ -92,6 +92,14 @@ import adapter_wizardtalesnet
 import adapter_nhamagicalworldsus
 import adapter_hlfictionnet
 import adapter_grangerenchantedcom
+import adapter_dracoandginnycom
+import adapter_scarvesandcoffeenet
+import adapter_thepetulantpoetesscom
+import adapter_wolverineandroguecom
+import adapter_sinfuldesireorg
+import adapter_merlinficdtwinscouk
+import adapter_thehookupzonenet
+import adapter_bloodtiesfancom

 ## This bit of complexity allows adapters to be added by just adding
 ## importing.  It eliminates the long if/else clauses we used to need
@@ -109,10 +117,10 @@ for x in imports():
    if "fanficdownloader.adapters.adapter_" in x:
        #print x
        __class_list.append(sys.modules[x].getClass())
-            
-def getAdapter(config,url,fileform=None):
+
+def getDomainURL(url):
    ## fix up leading protocol.
-    fixedurl = re.sub(r"(?i)^[htp]+[:/]+","http://",url.strip())
+    fixedurl = re.sub(r"(?i)^[htps]+[:/]+","http://",url.strip())
    if not fixedurl.startswith("http"):
        fixedurl = "http://%s"%url
    ## remove any trailing '#' locations.
@@ -127,20 +135,17 @@ def getAdapter(config,url,fileform=None):
    if( domain != parsedUrl.netloc ):
        fixedurl = fixedurl.replace(parsedUrl.netloc,domain)

+    return (domain,fixedurl)
+    
+        
+def getAdapter(config,url):
+
    logging.debug("trying url:"+url)
-    cls = getClassFor(domain)
-    if not cls and domain.startswith("www."):
-        domain = domain.replace("www.","")
-        logging.debug("trying site:without www: "+domain)
-        cls = getClassFor(domain)
-        fixedurl = fixedurl.replace("http://www.","http://")
-    if not cls:
-        logging.debug("trying site:www."+domain)
-        cls = getClassFor("www."+domain)
-        fixedurl = fixedurl.replace("http://","http://www.")
+    (domain,fixedurl) = getDomainURL(url)
+    cls = getClassFromList(domain)
+    logging.debug("fixedurl:"+fixedurl)
    if cls:
        adapter = cls(config,fixedurl) # raises InvalidStoryURL
-        adapter.setSectionOrder(adapter.getConfigSection(),fileform)
        return adapter
    # No adapter found.
    raise exceptions.UnknownSite( url, [cls.getSiteDomain() for cls in __class_list] )
@@ -148,7 +153,28 @@ def getAdapter(config,url,fileform=None):
 def getConfigSections():
    return [cls.getConfigSection() for cls in __class_list]

+def getConfigSectionFor(url):
+    (domain,fixedurl) = getDomainURL(url)
+    cls = getClassFromList(domain)
+    if cls:
+        return cls.getConfigSection()
+    
+    # No adapter found.
+    raise exceptions.UnknownSite( url, [cls.getSiteDomain() for cls in __class_list] )
+
 def getClassFor(domain):
+    cls = getClassFromList(domain)
+    if not cls and domain.startswith("www."):
+        domain = domain.replace("www.","")
+        logging.debug("trying site:without www: "+domain)
+        cls = getClassFromList(domain)
+        fixedurl = fixedurl.replace("http://www.","http://")
+    if not cls:
+        logging.debug("trying site:www."+domain)
+        cls = getClassFromList("www."+domain)
+        fixedurl = fixedurl.replace("http://","http://www.")
+    
+def getClassFromList(domain):
    for cls in __class_list:
        if cls.matchesSite(domain):
            return cls
@@ -37,7 +37,6 @@ class AdAstraFanficComSiteAdapter(BaseSiteAdapter):
                               # Most sites that claim to be
                               # iso-8859-1 (and some that claim to be
                               # utf8) are really windows-1252.
-        self.story.addToList("category","Star Trek")
        self.is_adult=False
        
        # get storyId from url--url validation guarantees query is only sid=1234
@@ -83,7 +83,7 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):

    def getSiteURLPattern(self):
        # http://archiveofourown.org/collections/Smallville_Slash_Archive/works/159770
-        return re.escape("http://")+"(www.)?"+re.escape(self.getSiteDomain())+r"(/collections/[^/]+)?/works/(?P<id>\d+)(/chapters/\d+)?/?$"
+        return re.escape("http://")+"(www.)?"+re.escape(self.getSiteDomain())+r"(/collections/[^/]+)?/works/(?P<id>\d+)"
        
    ## Login
    def needToLoginCheck(self, data):
@@ -191,7 +191,7 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):

        a = metasoup.find('blockquote',{'class':'userstuff'})
        if a != None:
-            self.setDescription(url,a.text)
+            self.setDescription(url,a)
            #self.story.setMetadata('description',a.text)
 		
        a = metasoup.find('dd',{'class':"rating tags"})
@@ -231,9 +231,9 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
                self.story.addToList('characters',char.string)
        a = metasoup.find('dd',{'class':"relationship tags"})
        if a != None:
-            chars = a.findAll('a',{'class':"tag"})
-            for char in chars:
-                self.story.addToList('characters',char.string)
+            ships = a.findAll('a',{'class':"tag"})
+            for ship in ships:
+                self.story.addToList('ships',ship.string)
 		

        stats = metasoup.find('dl',{'class':'stats'})
@@ -56,11 +56,6 @@ class AshwinderSycophantHexComAdapter(BaseSiteAdapter):
        # Each adapter needs to have a unique site abbreviation.
        self.story.setMetadata('siteabbrev','asph')

-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Harry Potter")
-
        # The date format will vary from site to site.
        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
        self.dateformat = "%m/%d/%Y"
@@ -0,0 +1,333 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+# By virtue of being recent and requiring both is_adult and user/pass,
+# adapter_fanficcastletvnet.py is the best choice for learning to
+# write adapters--especially for sites that use the eFiction system.
+# Most sites that have ".../viewstory.php?sid=123" in the story URL
+# are eFiction.
+
+# For non-eFiction sites, it can be considerably more complex, but
+# this is still a good starting point.
+
+# In general an 'adapter' needs to do these five things:
+
+# - 'Register' correctly with the downloader
+# - Site Login (if needed)
+# - 'Are you adult?' check (if needed--some do one, some the other, some both)
+# - Grab the chapter list
+# - Grab the story meta-data (some (non-eFiction) adapters have to get it from the author page)
+# - Grab the chapter texts
+
+# Search for XXX comments--that's where things are most likely to need changing.
+
+# This function is called by the downloader in all adapter_*.py files
+# in this dir to register the adapter class.  So it needs to be
+# updated to reflect the class below it.  That, plus getSiteDomain()
+# take care of 'Registering'.
+def getClass():
+    return BloodTiesFansComAdapter # XXX
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class BloodTiesFansComAdapter(BaseSiteAdapter): # XXX
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
+        self._setURL('http://' + self.getSiteDomain() + '/fiction/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','btf') # XXX
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%b %d, %Y" # XXX
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'bloodties-fans.com' # XXX
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/fiction/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/fiction/viewstory.php?sid=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Registered Users Only' in data \
+                or 'There is no such account on our website' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['cookiecheck'] = '1'
+        params['submit'] = 'Submit'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/fiction/user.php?action=login'
+        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "Member Account" not in d : #Member Account
+            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+
+            # Furthermore, there's a couple sites now with more than
+            # one warning level for different ratings.  And they're
+            # fussy about it.  midnightwhispers has three: 4, 2 & 1.
+            # we'll try 1 first.
+            addurl = "&ageconsent=ok&warning=4" # XXX
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+            
+        # The actual text that is used to announce you need to be an
+        # adult varies from site to site.  Again, print data before
+        # the title search to troubleshoot.
+            
+        # Since the warning text can change by warning level, let's
+        # look for the warning pass url.  nfacommunity uses
+        # &amp;warning= -- actually, so do other sites.  Must be an
+        # eFiction book.
+            
+        # viewstory.php?sid=561&amp;warning=4
+        # viewstory.php?sid=561&amp;warning=1
+        # viewstory.php?sid=561&amp;warning=2
+        #print data
+        #m = re.search(r"'viewstory.php\?sid=1882(&amp;warning=4)'",data)
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&amp;ageconsent=ok)?&amp;warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid &amp; error in url.
+                addurl = addurl.replace("&amp;","&")
+                url = self.url+'&index=1'+addurl
+                logging.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/fiction/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                catstext = [cat.string for cat in cats]
+                for cat in catstext:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                charstext = [char.string for char in chars]
+                for char in charstext:
+                    self.story.addToList('characters',char.string)
+
+            ## Not all sites use Genre, but there's no harm to
+            ## leaving it in.  Check to make sure the type_id number
+            ## is correct, though--it's site specific.
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
+                genrestext = [genre.string for genre in genres]
+                self.genre = ', '.join(genrestext)
+                for genre in genrestext:
+                    self.story.addToList('genre',genre.string)
+
+            ## Not all sites use Warnings, but there's no harm to
+            ## leaving it in.  Check to make sure the type_id number
+            ## is correct, though--it's site specific.
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
+                warningstext = [warning.string for warning in warnings]
+                self.warning = ', '.join(warningstext)
+                for warning in warningstext:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/fiction/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -80,11 +80,6 @@ class CastleFansOrgAdapter(BaseSiteAdapter): # XXX
        # Each adapter needs to have a unique site abbreviation.
        self.story.setMetadata('siteabbrev','cslf') # XXX

-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Castle") # XXX
-
        # The date format will vary from site to site.
        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
        self.dateformat = "%b %d, %Y" # XXX
@@ -179,13 +174,14 @@ class CastleFansOrgAdapter(BaseSiteAdapter): # XXX
        # print data

        # Now go hunting for all the meta data and the chapter list.
-        
+
+        pagetitle = soup.find('div',{'id':'pagetitle'})
        ## Title
-        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        a = pagetitle.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
        self.story.setMetadata('title',a.string)
        
        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        a = pagetitle.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
        self.story.setMetadata('authorId',a['href'].split('=')[1])
        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
        self.story.setMetadata('author',a.string)
@@ -56,11 +56,6 @@ class ChaosSycophantHexComAdapter(BaseSiteAdapter):
        # Each adapter needs to have a unique site abbreviation.
        self.story.setMetadata('siteabbrev','csph')

-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        #self.story.addToList("category","Harry Potter")
-
        # The date format will vary from site to site.
        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
        self.dateformat = "%m/%d/%Y"
@@ -44,25 +44,20 @@ class CheckmatedComAdapter(BaseSiteAdapter):
        self.username = "" # if left empty, site doesn't return any message at all.
        self.password = ""
        self.is_adult=False
-        
+
        # get storyId from url--url validation guarantees query is only sid=1234
        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-        
+
        self._setURL('http://' + self.getSiteDomain() + '/story.php?story='+self.story.getMetadata('storyId'))
-        
+
        # Each adapter needs to have a unique site abbreviation.
        self.story.setMetadata('siteabbrev','chm')

-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Harry Potter")
-
        # The date format will vary from site to site.
        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
        self.dateformat = "%b %d, %Y"
-            
+
    @staticmethod # must be @staticmethod, don't remove it.
    def getSiteDomain():
        # The site domain.  Does have www here, if it uses it.
@@ -82,7 +77,7 @@ class CheckmatedComAdapter(BaseSiteAdapter):
            return True
        else:
            return False
-        
+
    def performLogin(self, url):
        params = {}

@@ -93,8 +88,8 @@ class CheckmatedComAdapter(BaseSiteAdapter):
            params['name'] = self.getConfig("username")
            params['pass'] = self.getConfig("password")
        params['login'] = 'yes'
-        params['submit'] = 'login'	
-	
+        params['submit'] = 'login'
+
        loginUrl = 'http://' + self.getSiteDomain()+'/login.php'
        d = self._fetchUrl(loginUrl,params)
        e = self._fetchUrl(url)
@@ -135,33 +130,33 @@ class CheckmatedComAdapter(BaseSiteAdapter):
        # The actual text that is used to announce you need to be an
        # adult varies from site to site.  Again, print data before
        # the title search to troubleshoot.
-            
+
        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
+
        # use BeautifulSoup HTML parser to make everything easier to find.
        soup = bs.BeautifulSoup(data)
        # print data

        # Now go hunting for all the meta data and the chapter list.
-        
+
        ## Title
        a = soup.findAll('span', {'class' : 'storytitle'})
        self.story.setMetadata('title',a[0].string)
-        
+
        # Find authorid and URL from... author url.
        a = a[1].find('a', href=re.compile(r"authors.php\?name\=\w+"))
        self.story.setMetadata('authorId',a['href'].split('=')[1])
        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
        self.story.setMetadata('author',a.string)
-			
+
        a = soup.find('select', {'name' : 'chapter'})
        if a == None:
            self.chapterUrls.append((self.story.getMetadata('title'),url))
        else:
            for chapter in a.findAll('option'):
                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/story.php?story='+self.story.getMetadata('storyId')+'&chapter='+chapter['value']))
-			
+
        self.story.setMetadata('numChapters',len(self.chapterUrls))

        # eFiction sites don't help us out a lot with their meta data
@@ -174,7 +169,7 @@ class CheckmatedComAdapter(BaseSiteAdapter):
            except:
                return ""

-				
+
        # website does not keep track of word count, and there is no convenient way to calculate it

        summary = soup.find('fieldset')
@@ -182,21 +177,23 @@ class CheckmatedComAdapter(BaseSiteAdapter):
        summary.name='div'
        self.setDescription(url,summary)

-		
+
        # <span class="label">Rated:</span> NC-17<br /> etc
        table = soup.findAll('div', {'class' : 'text'})[1]
        for labels in table.findAll('tr'):
            value = labels.findAll('td')[1]
            label = labels.findAll('td')[0]

-			
+
            if 'Rating' in stripHTML(label):
                self.story.setMetadata('rating', stripHTML(value))

            if 'Ship' in stripHTML(label):
+                if value.string != "none/none":
+                    self.story.addToList('ships',value.string)
                for char in value.string.split('/'):
                    if char != 'none':
-                        self.story.addToList('characters',char)		
+                        self.story.addToList('characters',char)

            if 'Status' in stripHTML(label):
                if value.find('img', {'src' : 'img/incomplete.gif'}) == None:
@@ -206,11 +203,11 @@ class CheckmatedComAdapter(BaseSiteAdapter):

            if 'Published' in stripHTML(label):
                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-            
+
            if 'Updated' in stripHTML(label):
                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-			
-        a = self._fetchUrl(self.story.getMetadata('authorUrl')+'&cat=stories')	
+
+        a = self._fetchUrl(self.story.getMetadata('authorUrl')+'&cat=stories')
        for story in bs.BeautifulSoup(a).findAll('table', {'class' : 'storyinfo'}):
            a = story.find('a', href=re.compile(r"review.php\?s\="+self.story.getMetadata('storyId')+'&act=view'))
            if a != None:
@@ -220,8 +217,8 @@ class CheckmatedComAdapter(BaseSiteAdapter):
                    if 'genre' in stripHTML(label):
                        for genre in value.findAll('img'):
                            self.story.addToList('genre',genre['title'])
-        
-            
+
+
    # grab the text for an individual chapter.
    def getChapterText(self, url):

@@ -229,11 +226,11 @@ class CheckmatedComAdapter(BaseSiteAdapter):

        soup = bs.BeautifulSoup(self._fetchUrl(url),
                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
+
        div = soup.find('div', {'id' : 'resizeableText'})
        div.find('div', {'class' : 'storyTools'}).extract()

        if None == div:
            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
+
        return self.utf8FromSoup(url,div)
@@ -56,11 +56,6 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
        # Each adapter needs to have a unique site abbreviation.
        self.story.setMetadata('siteabbrev','dksl')

-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Buffy the Vampire Slayer")
-
        # The date format will vary from site to site.
        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
        self.dateformat = "%d %B %Y"
@@ -56,11 +56,6 @@ class DokugaComAdapter(BaseSiteAdapter):
        # Each adapter needs to have a unique site abbreviation.
        self.story.setMetadata('siteabbrev','dkg')

-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","InuYasha")
-
        # The date format will vary from site to site.
        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
        if 'fanfiction' in self.story.getMetadata('section'):
@@ -0,0 +1,298 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return DracoAndGinnyComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class DracoAndGinnyComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','dcagn')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%b %d, %Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.dracoandginny.com'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Registered Users Only' in data \
+                or 'There is no such account on our website' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['cookiecheck'] = '1'
+        params['submit'] = 'Submit'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
+        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "Member Account" not in d : #Member Account
+            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=2"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&amp;ageconsent=ok)?&amp;warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid &amp; error in url.
+                addurl = addurl.replace("&amp;","&")
+                url = self.url+'&index=1'+addurl
+                logging.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        content=soup.find('div',{'class' : 'listbox'})
+        
+        self.setDescription(url,content.find('blockquote'))
+        
+        for genre in content.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')):
+            self.story.addToList('genre',genre.string)
+
+        for warning in content.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')):
+            self.story.addToList('warnings',warning.string)
+        
+        labels = content.findAll('b')
+        
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value.split(' |')[0])
+                
+            if 'Rating' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value.nextSibling).split(' |')[0], self.dateformat))
+            
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            i=1
+            for a in storyas:
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        break
+                    i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'class' : 'listbox'})
+        
+        if None == div:
+            div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -55,11 +55,6 @@ class DramioneOrgAdapter(BaseSiteAdapter):
        # Each adapter needs to have a unique site abbreviation.
        self.story.setMetadata('siteabbrev','drmn')

-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Harry Potter")
-
        # The date format will vary from site to site.
        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
        self.dateformat = "%d %B %Y"
@@ -165,6 +160,20 @@ class DramioneOrgAdapter(BaseSiteAdapter):
        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
        self.story.setMetadata('author',a.string)

+        # Use banner as cover if found
+        coverurl = ''
+        img = soup.find('img',{'class':'banner'})
+        if img:
+            coverurl = img['src']
+            #print "Cover: "+coverurl
+        a = soup.find(text="This story has a banner; click to view.")
+        if a:
+            #print "A: "+ ', '.join("(%s, %s)" %tup for tup in a.parent.attrs)
+            coverurl = a.parent['href']
+            #print "Cover: "+coverurl
+        if coverurl:
+            self.setCoverImage(url,coverurl)
+
        # Find the chapters:
        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
            # just in case there's tags, like <i> in chapter titles.
@@ -183,6 +192,22 @@ class DramioneOrgAdapter(BaseSiteAdapter):
        for warning in warnings:
            self.story.addToList('warnings',warning.string)

+        themes=soup.findAll('a', {'class' : "tag-3"})
+        for theme in themes:
+            self.story.addToList('themes',theme.string)
+
+        hermiones=soup.findAll('a', {'class' : "tag-4"})
+        for hermione in hermiones:
+            self.story.addToList('hermiones',hermione.string)
+
+        dracos=soup.findAll('a', {'class' : "tag-5"})
+        for draco in dracos:
+            self.story.addToList('dracos',draco.string)
+
+        timelines=soup.findAll('a', {'class' : "tag-6"})
+        for timeline in timelines:
+            self.story.addToList('timeline',timeline.string)
+
        # utility method
        def defaultGetattr(d,k):
            try:
@@ -56,11 +56,6 @@ class ErosnSapphoSycophantHexComAdapter(BaseSiteAdapter):
        # Each adapter needs to have a unique site abbreviation.
        self.story.setMetadata('siteabbrev','essph')

-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Harry Potter")
-
        # The date format will vary from site to site.
        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
        self.dateformat = "%d/%m/%y"
@@ -186,8 +186,14 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):

        donechars = False
        while len(metalist) > 0:
-            if  metalist[0].startswith('Reviews') or metalist[0].startswith('Chapters') or metalist[0].startswith('Status') or metalist[0].startswith('id:') or metalist[0].startswith('Favs:') or metalist[0].startswith('Follows:'):
+            if  metalist[0].startswith('Chapters') or metalist[0].startswith('Status') or metalist[0].startswith('id:'):
                pass
+            elif  metalist[0].startswith('Reviews'):
+                self.story.setMetadata('reviews',metalist[0].split(':')[1].strip())
+            elif  metalist[0].startswith('Favs:'):
+                self.story.setMetadata('favs',metalist[0].split(':')[1].strip())
+            elif  metalist[0].startswith('Follows:'):
+                self.story.setMetadata('follows',metalist[0].split(':')[1].strip())
            elif  metalist[0].startswith('Updated'):
                self.story.setMetadata('dateUpdated',makeDate(metalist[0].split(':')[1].strip(), '%m-%d-%y'))
            elif  metalist[0].startswith('Published'):
@@ -208,10 +214,9 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
        else:
            self.story.setMetadata('status', 'In-Progress')

-        if self.getConfig('include_images'):
-            img = soup.find('img',{'class':'cimage'})
-            if img:
-                self.story.addImgUrl(self,url,img['src'],self._fetchUrlRaw,cover=True)
+        img = soup.find('img',{'class':'cimage'})
+        if img:
+            self.setCoverImage(url,img['src'])
            
        # Find the chapter selector 
        select = soup.find('select', { 'name' : 'chapter' } )
@@ -201,7 +201,7 @@ class FicBookNetAdapter(BaseSiteAdapter):
                break
 				
        summary=soup.find('span', {'class' : 'urlize'})
-        self.setDescription(url,summary.text)
+        self.setDescription(url,summary)
        #self.story.setMetadata('description', summary.text)
            
    # grab the text for an individual chapter.
@@ -37,7 +37,6 @@ class FictionAlleyOrgSiteAdapter(BaseSiteAdapter):
                               # Most sites that claim to be
                               # iso-8859-1 (and some that claim to be
                               # utf8) are really windows-1252.
-        self.story.addToList("category","Harry Potter")
        self.is_adult=False
        
        # get storyId from url--url validation guarantees query correct
@@ -124,7 +124,7 @@ class FicwadComSiteAdapter(BaseSiteAdapter):

        # description
        storydiv = soup.find("div",{"id":"story"})
-        self.setDescription(url,storydiv.find("blockquote",{'class':'summary'}).p.string)
+        self.setDescription(url,storydiv.find("blockquote",{'class':'summary'}).p)
        #self.story.setMetadata('description', storydiv.find("blockquote",{'class':'summary'}).p.string)

        # most of the meta data is here:
@@ -27,6 +27,8 @@ from .. import BeautifulSoup as bs
 from ..htmlcleanup import stripHTML
 from .. import exceptions as exceptions

+from ..bbcodeutils.bbcodeparser import bbcodeparser
+
 from base_adapter import BaseSiteAdapter,  makeDate

 def getClass():
@@ -133,19 +135,23 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
            if storyMetadata["categories"][category]:
                self.story.addToList("genre", category) 

-        self.story.addToList("category", "My Little Pony")
-        
        self.story.setMetadata("numWords", str(storyMetadata["words"]))
        
-        
        # fimfic is the first site with an explicit cover image.
-        if self.getConfig('include_images') and "image" in storyMetadata.keys():
-            coverurl = storyMetadata["image"]
+        if "image" in storyMetadata.keys():
+            if "full_image" in storyMetadata:
+                coverurl = storyMetadata["full_image"]
+            else:
+                coverurl = storyMetadata["image"]
            if coverurl.startswith('//static.fimfiction.net'): # fix for img urls missing 'http:'
                coverurl = "http:"+coverurl
-            self.story.addImgUrl(self,self.url,coverurl,self._fetchUrlRaw,cover=True)
-            
-        self.setDescription(self.url, storyMetadata["description"])
+            self.setCoverImage(self.url,coverurl)
+
+
+        # the fimfic API gives bbcode for desc, not html.
+        # btw, bbcode honors newlines, html doesn't.  change newlines to br tags.
+        self.setDescription(self.url,
+                            bbcodeparser().parse(storyMetadata["description"]).html(doDeepCopy=False).replace('\r','').replace('\n','<br />'))
        
        # Dates are in Unix time
        # Take the publish date from the first chapter posted
@@ -155,8 +161,18 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
        self.story.setMetadata("dateUpdated", datetime.fromtimestamp(rawDateUpdated))
        
        soup = bs.BeautifulSoup(data).find("div", {"class":"story"})
-        for character in [character_icon["title"] for character_icon in soup.findAll("a", {"class":"character_icon"})]:
-            self.story.addToList("characters", character)
+        # fimfic stopped putting the char name on or around the char
+        # icon now for some reason.  Pull it from the image name with
+        # some heuristics.
+        for character in [character_icon["src"] for character_icon in soup.findAll("img", {"class":"character_icon"})]:
+            # //static.fimfiction.net/images/characters/twilight_sparkle.png
+            # 5th split /, remove last four, replace _, capitolize every word(title())
+            char = character.split('/')[5][:-4].replace('_',' ').title()
+            if char == 'Oc':
+                char = "OC"
+            if char == 'Cmc':
+                char = "Cutie Mark Crusaders"
+            self.story.addToList("characters", char)
            
            
    def getChapterText(self, url):
@@ -60,11 +60,6 @@ class GrangerEnchantedCom(BaseSiteAdapter):
        # Each adapter needs to have a unique site abbreviation.
        self.story.setMetadata('siteabbrev','gech')

-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Harry Potter")
-
        # The date format will vary from site to site.
        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
        self.dateformat = "%d/%b/%Y"
@@ -37,7 +37,6 @@ class HarryPotterFanFictionComSiteAdapter(BaseSiteAdapter):
                               # Most sites that claim to be
                               # iso-8859-1 (and some that claim to be
                               # utf8) are really windows-1252.
-        self.story.addToList("category","Harry Potter")
        self.is_adult=False
        
        # get storyId from url--url validation guarantees query is only psid=1234
@@ -55,11 +55,6 @@ class HLFictionNetAdapter(BaseSiteAdapter):
        # Each adapter needs to have a unique site abbreviation.
        self.story.setMetadata('siteabbrev','hlf')

-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Highlander")
-
        # The date format will vary from site to site.
        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
        self.dateformat = "%m/%d/%y"
@@ -60,11 +60,6 @@ class HPFandomNetAdapterAdapter(BaseSiteAdapter): # XXX
        # Each adapter needs to have a unique site abbreviation.
        self.story.setMetadata('siteabbrev','hpfdm') # XXX

-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Harry Potter") # XXX
-
        # The date format will vary from site to site.
        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
        self.dateformat = "%Y.%m.%d" # XXX
@@ -55,11 +55,6 @@ class HPFanficArchiveComAdapter(BaseSiteAdapter):
        # Each adapter needs to have a unique site abbreviation.
        self.story.setMetadata('siteabbrev','hpffa')

-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Harry Potter")
-
        # The date format will vary from site to site.
        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
        self.dateformat = "%B %d, %Y"
@@ -55,11 +55,6 @@ class IkEternalNetAdapter(BaseSiteAdapter):
        # Each adapter needs to have a unique site abbreviation.
        self.story.setMetadata('siteabbrev','ike')

-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","InuYasha")
-
        # The date format will vary from site to site.
        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
        self.dateformat = "%B %d, %Y"
@@ -60,11 +60,6 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
        # Each adapter needs to have a unique site abbreviation.
        self.story.setMetadata('siteabbrev','ksa') # XXX

-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Star Trek") # XXX
-
        # The date format will vary from site to site.
        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
        self.dateformat = "%b/%d/%Y" # XXX
@@ -167,7 +162,7 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
        self.story.setMetadata('authorId',a['href'].split('=')[1])
        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
+        self.story.setMetadata('author',stripHTML(a))

        # Find the chapters:
        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
@@ -190,7 +185,7 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
        labels = soup.findAll('span',{'class':'label'})
        for labelspan in labels:
            value = labelspan.nextSibling
-            label = labelspan.string
+            label = stripHTML(labelspan)

            if 'Summary' in label:
                ## Everything until the next span class='label'
@@ -215,50 +210,50 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX

            if 'Categories' in label:
                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-                catstext = [cat.string for cat in cats]
+                catstext = [stripHTML(cat) for cat in cats]
                for cat in catstext:
                    # ran across one story with an empty <a href="browse.php?type=categories&amp;catid=1"></a>
                    # tag in the desc once.
                    if cat and cat.strip() in ('Poetry','Essays'):
-                        self.story.addToList('category',cat.string)
+                        self.story.addToList('category',stripHTML(cat))

            if 'Characters' in label:
                self.story.addToList('characters','Kirk')
                self.story.addToList('characters','Spock')
                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-                charstext = [char.string for char in chars]
+                charstext = [stripHTML(char) for char in chars]
                for char in charstext:
-                    self.story.addToList('characters',char.string)
+                    self.story.addToList('characters',stripHTML(char))

            ## Not all sites use Genre, but there's no harm to
            ## leaving it in.  Check to make sure the type_id number
            ## is correct, though--it's site specific.
            if 'Genre' in label:
                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) # XXX
-                genrestext = [genre.string for genre in genres]
+                genrestext = [stripHTML(genre) for genre in genres]
                self.genre = ', '.join(genrestext)
                for genre in genrestext:
-                    self.story.addToList('genre',genre.string)
+                    self.story.addToList('genre',stripHTML(genre))

            ## In addition to Genre (which is very site specific) KSA
            ## has 'Story Type', which is much more what most sites
            ## call genre.
            if 'Story Type' in label:
                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=5')) # XXX
-                genrestext = [genre.string for genre in genres]
+                genrestext = [stripHTML(genre) for genre in genres]
                self.genre = ', '.join(genrestext)
                for genre in genrestext:
-                    self.story.addToList('genre',genre.string)
+                    self.story.addToList('genre',stripHTML(genre))

            ## Not all sites use Warnings, but there's no harm to
            ## leaving it in.  Check to make sure the type_id number
            ## is correct, though--it's site specific.
            if 'Warnings' in label:
                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
-                warningstext = [warning.string for warning in warnings]
+                warningstext = [stripHTML(warning) for warning in warnings]
                self.warning = ', '.join(warningstext)
                for warning in warningstext:
-                    self.story.addToList('warnings',warning.string)
+                    self.story.addToList('warnings',stripHTML(warning))

            if 'Completed' in label:
                if 'Yes' in value:
@@ -277,7 +272,7 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
        try:
            # Find Series name from series URL.
            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
-            series_name = a.string
+            series_name = stripHTML(a)
            series_url = 'http://'+self.host+'/'+a['href']

            # use BeautifulSoup HTML parser to make everything easier to find.
@@ -56,11 +56,6 @@ class LibraryOfMoriaComAdapter(BaseSiteAdapter):
        # Each adapter needs to have a unique site abbreviation.
        self.story.setMetadata('siteabbrev','lom')

-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Lord of the Rings")
-
        # The date format will vary from site to site.
        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
        self.dateformat = "%B %d, %Y"
@@ -56,11 +56,6 @@ class LumosSycophantHexComAdapter(BaseSiteAdapter):
        # Each adapter needs to have a unique site abbreviation.
        self.story.setMetadata('siteabbrev','lsph')

-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Harry Potter")
-
        # The date format will vary from site to site.
        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
        self.dateformat = "%m/%d/%Y"
@@ -0,0 +1,290 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return MerlinFicDtwinsCoUk
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class MerlinFicDtwinsCoUk(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','mrfd')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%b %d, %Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'merlinfic.dtwins.co.uk'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Registered Users Only' in data \
+                or 'There is no such account on our website' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['cookiecheck'] = '1'
+        params['submit'] = 'Submit'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
+        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "Member Account" not in d : #Member Account
+            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=4"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&amp;ageconsent=ok)?&amp;warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid &amp; error in url.
+                addurl = addurl.replace("&amp;","&")
+                url = self.url+'&index=1'+addurl
+                logging.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+                    
+            if 'Pairing' in label:
+                ships = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+                for ship in ships:
+                    self.story.addToList('ships',ship.string)
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=3'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            i=1
+            for a in storyas:
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        break
+                    i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -60,11 +60,6 @@ class MidnightwhispersCaAdapter(BaseSiteAdapter): # XXX
        # Each adapter needs to have a unique site abbreviation.
        self.story.setMetadata('siteabbrev','mw') # XXX

-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Queer as Folk") # XXX
-
        # The date format will vary from site to site.
        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
        self.dateformat = "%B %d, %Y" # XXX
@@ -59,11 +59,6 @@ class MuggleNetComAdapter(BaseSiteAdapter): # XXX
        # Each adapter needs to have a unique site abbreviation.
        self.story.setMetadata('siteabbrev','mgln') # XXX

-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Harry Potter") # XXX
-
        # The date format will vary from site to site.
        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
        self.dateformat = "%m/%d/%y" # XXX
@@ -55,11 +55,6 @@ class NationalLibraryNetAdapter(BaseSiteAdapter):
        # Each adapter needs to have a unique site abbreviation.
        self.story.setMetadata('siteabbrev','ntlb')

-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","West Wing")
-
        # The date format will vary from site to site.
        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
        self.dateformat = "%m-%d-%y"
@@ -154,6 +149,10 @@ class NationalLibraryNetAdapter(BaseSiteAdapter):
                for char in value.string.split(', '):
                    self.story.addToList('characters',char)
                    
+            if 'Pairing' in label:
+                for char in value.string.split(', '):
+                    self.story.addToList('ships',char)
+                    
            if 'Warnings' in label:
                for warning in value.string.split(', '):
                    self.story.addToList('warnings',warning)
@@ -55,11 +55,6 @@ class NCISFicComAdapter(BaseSiteAdapter):
        # Each adapter needs to have a unique site abbreviation.
        self.story.setMetadata('siteabbrev','ncisf')

-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","NCIS")
-
        # The date format will vary from site to site.
        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
        self.dateformat = "%m-%d-%y"
@@ -154,6 +149,10 @@ class NCISFicComAdapter(BaseSiteAdapter):
                for char in value.string.split(', '):
                    self.story.addToList('characters',char)
                    
+            if 'Pairing' in label:
+                for char in value.string.split(', '):
+                    self.story.addToList('ships',char)
+                    
            if 'Warnings' in label:
                for warning in value.string.split(', '):
                    self.story.addToList('warnings',warning)
@@ -56,11 +56,6 @@ class NCISFictionComAdapter(BaseSiteAdapter):
        # Each adapter needs to have a unique site abbreviation.
        self.story.setMetadata('siteabbrev','ncisfn')

-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","NCIS")
-
        # The date format will vary from site to site.
        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
        self.dateformat = "%d/%m/%Y"
@@ -157,6 +152,11 @@ class NCISFictionComAdapter(BaseSiteAdapter):
                for char in chars:
                    self.story.addToList('characters',char.string)

+            if 'Pairing' in label:
+                ships = value.findAll('a')
+                for ship in ships:
+                    self.story.addToList('ships',ship.string)
+
            if 'Genre' in label:
                genres = value.findAll('a')
                for genre in genres:
@@ -60,11 +60,6 @@ class NfaCommunityComAdapter(BaseSiteAdapter): # XXX
        # Each adapter needs to have a unique site abbreviation.
        self.story.setMetadata('siteabbrev','nfa') # XXX

-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","NCIS") # XXX
-
        # The date format will vary from site to site.
        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
        self.dateformat = "%m/%d/%Y" # XXX
@@ -56,11 +56,6 @@ class NHAMagicalWorldsUsAdapter(BaseSiteAdapter):
        # Each adapter needs to have a unique site abbreviation.
        self.story.setMetadata('siteabbrev','nha')

-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Buffy the Vampire Slayer")
-
        # The date format will vary from site to site.
        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
        self.dateformat = " %m/%d/%y"
@@ -56,11 +56,6 @@ class OcclumencySycophantHexComAdapter(BaseSiteAdapter):
        # Each adapter needs to have a unique site abbreviation.
        self.story.setMetadata('siteabbrev','osph')

-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Harry Potter")
-
        # The date format will vary from site to site.
        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
        self.dateformat = "%m/%d/%Y"
@@ -55,11 +55,6 @@ class OneDirectionFanfictionComAdapter(BaseSiteAdapter):
        # Each adapter needs to have a unique site abbreviation.
        self.story.setMetadata('siteabbrev','odf')

-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","One Direction")
-
        # The date format will vary from site to site.
        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
        self.dateformat = "%m/%d/%y"
@@ -55,11 +55,6 @@ class PhoenixSongNetAdapter(BaseSiteAdapter):
        # Each adapter needs to have a unique site abbreviation.
        self.story.setMetadata('siteabbrev','phs')

-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Harry Potter")
-
        # The date format will vary from site to site.
        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
        self.dateformat = "%B %d %Y"
@@ -60,11 +60,6 @@ class PonyFictionArchiveNetAdapter(BaseSiteAdapter):
        # Each adapter needs to have a unique site abbreviation.
        self.story.setMetadata('siteabbrev','pffa')

-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","My Little Pony")
-            
    @staticmethod # must be @staticmethod, don't remove it.
    def getSiteDomain():
        # The site domain.  Does have www here, if it uses it.
@@ -62,11 +62,6 @@ class PortkeyOrgAdapter(BaseSiteAdapter): # XXX
        # Each adapter needs to have a unique site abbreviation.
        self.story.setMetadata('siteabbrev','prtky') # XXX

-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Harry Potter") # XXX
-
        # The date format will vary from site to site.
        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
        self.dateformat = "%d/%m/%y" # XXX
@@ -37,7 +37,6 @@ class PotionsAndSnitchesNetSiteAdapter(BaseSiteAdapter):
                               # Most sites that claim to be
                               # iso-8859-1 (and some that claim to be
                               # utf8) are really windows-1252.
-        self.story.addToList("category","Harry Potter")
        
        # get storyId from url--url validation guarantees query is only sid=1234
        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
@@ -142,7 +141,7 @@ class PotionsAndSnitchesNetSiteAdapter(BaseSiteAdapter):
                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
                charstext = [char.string for char in chars]
                for char in charstext:
-                    if char == "!Snape and Harry (required)":
+                    if "Snape and Harry (required)" in char:
                        self.story.addToList('characters',"Snape")
                        self.story.addToList('characters',"Harry")
                    else:
@@ -55,11 +55,6 @@ class PretenderCenterComAdapter(BaseSiteAdapter):
        # Each adapter needs to have a unique site abbreviation.
        self.story.setMetadata('siteabbrev','ptdc')

-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","The Pretender")
-
        # The date format will vary from site to site.
        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
        self.dateformat = "%d/%m/%y"
@@ -55,11 +55,6 @@ class PrisonBreakFicNetAdapter(BaseSiteAdapter):
        # Each adapter needs to have a unique site abbreviation.
        self.story.setMetadata('siteabbrev','pbf')

-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Prison Break")
-
        # The date format will vary from site to site.
        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
        self.dateformat = "%B %d, %Y"
@@ -157,6 +152,11 @@ class PrisonBreakFicNetAdapter(BaseSiteAdapter):
                for char in chars:
                    self.story.addToList('characters',char.string)

+            if 'Pairing' in label:
+                ships = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=3')) # XXX
+                for ship in ships:
+                    self.story.addToList('ships',ship.string.replace(" and ","/"))
+
            if 'Genre' in label:
                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) # XXX
                for genre in genres:
@@ -55,11 +55,6 @@ class SamDeanArchiveNuAdapter(BaseSiteAdapter):
        # Each adapter needs to have a unique site abbreviation.
        self.story.setMetadata('siteabbrev','sda')

-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Supernatural")
-
        # The date format will vary from site to site.
        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
        self.dateformat = "%m/%d/%y"
@@ -0,0 +1,244 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return ScarvesAndCoffeeNetAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class ScarvesAndCoffeeNetAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','scacf')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%m/%d/%Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.scarvesandcoffee.net'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Registered Users Only' in data \
+                or 'There is no such account on our website' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=20"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&amp;ageconsent=ok)?&amp;warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid &amp; error in url.
+                addurl = addurl.replace("&amp;","&")
+                url = self.url+'&index=1'+addurl
+                logging.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('genre',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            i=1
+            for a in storyas:
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        break
+                    i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -58,7 +58,7 @@ class SG1HeliopolisComAdapter(BaseSiteAdapter):

        # If all stories from the site fall into the same category,
        # the site itself isn't likely to label them as such, so we
-        # do.
+        # do.  Can't use extracategories, could be Atlantis or SG-1
        if 'atlantis' in self.story.getMetadata('section'):
            self.story.addToList("category","Stargate: Atlantis")
        else:
@@ -0,0 +1,248 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return SinfulDesireOrgAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class SinfulDesireOrgAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/archive/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','snds')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%m/%d/%Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.sinful-desire.org'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/archive/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/archive/viewstory.php?sid=")+r"\d+$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&warning=5"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&amp;ageconsent=ok)?&amp;warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid &amp; error in url.
+                addurl = addurl.replace("&amp;","&")
+                url = self.url+'&index=1'+addurl
+                logging.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/archive/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/archive/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=3'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/archive/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            i=1
+            for a in storyas:
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        break
+                    i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -56,11 +56,6 @@ class SiyeCoUkAdapter(BaseSiteAdapter): # XXX
        # Each adapter needs to have a unique site abbreviation.
        self.story.setMetadata('siteabbrev','siye') # XXX

-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Harry Potter") # XXX
-
        # The date format will vary from site to site.
        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
        self.dateformat = "%Y.%m.%d" # XXX
@@ -156,6 +156,8 @@ class SquidgeOrgPejaAdapter(BaseSiteAdapter):
                #self.story.setMetadata('description',stripHTML(svalue))

            if 'Rated' in label:
+                if value.endswith("["):
+                    value = value[:-1]
                self.story.setMetadata('rating', value)

            if 'Characters' in label:
@@ -55,11 +55,6 @@ class StargateAtlantisOrgAdapter(BaseSiteAdapter):
        # Each adapter needs to have a unique site abbreviation.
        self.story.setMetadata('siteabbrev','stat')

-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Stargate: Atlantis")
-
        # The date format will vary from site to site.
        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
        self.dateformat = "%B %d %Y"
@@ -55,11 +55,6 @@ class StoriesOfArdaComAdapter(BaseSiteAdapter):
        # Each adapter needs to have a unique site abbreviation.
        self.story.setMetadata('siteabbrev','soa')

-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Lord of the Rings")
-
        # The date format will vary from site to site.
        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
        self.dateformat = "%m/%d/%Y"
@@ -73,8 +73,8 @@ class TestSiteAdapter(BaseSiteAdapter):
            self.story.setMetadata(u'title',"Test Story Title "+self.story.getMetadata('storyId'))
            self.story.setMetadata('author','Test Author aa')
        self.story.setMetadata('storyUrl',self.url)
-        self.story.setMetadata('description',u'Description '+self.crazystring+u''' Done
-
+        self.setDescription(self.url,u'Description '+self.crazystring+u''' Done
+<p>
 Some more longer description.  "I suck at summaries!"  "Better than it sounds!"  "My first fic"
 ''')
        self.story.setMetadata('datePublished',makeDate("1975-03-15","%Y-%m-%d"))
@@ -55,11 +55,6 @@ class TheAlphaGateComAdapter(BaseSiteAdapter):
        # Each adapter needs to have a unique site abbreviation.
        self.story.setMetadata('siteabbrev','tag')

-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Stargate: SG-1")
-
        # The date format will vary from site to site.
        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
        self.dateformat = "%d %b %Y"
@@ -55,11 +55,6 @@ class TheHexFilesNetAdapter(BaseSiteAdapter):
        # Each adapter needs to have a unique site abbreviation.
        self.story.setMetadata('siteabbrev','thf')

-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Harry Potter")
-
        # The date format will vary from site to site.
        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
        self.dateformat = "%Y.%m.%d"
@@ -0,0 +1,305 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+# By virtue of being recent and requiring both is_adult and user/pass,
+# adapter_fanficcastletvnet.py is the best choice for learning to
+# write adapters--especially for sites that use the eFiction system.
+# Most sites that have ".../viewstory.php?sid=123" in the story URL
+# are eFiction.
+
+# For non-eFiction sites, it can be considerably more complex, but
+# this is still a good starting point.
+
+# In general an 'adapter' needs to do these five things:
+
+# - 'Register' correctly with the downloader
+# - Site Login (if needed)
+# - 'Are you adult?' check (if needed--some do one, some the other, some both)
+# - Grab the chapter list
+# - Grab the story meta-data (some (non-eFiction) adapters have to get it from the author page)
+# - Grab the chapter texts
+
+# Search for XXX comments--that's where things are most likely to need changing.
+
+# This function is called by the downloader in all adapter_*.py files
+# in this dir to register the adapter class.  So it needs to be
+# updated to reflect the class below it.  That, plus getSiteDomain()
+# take care of 'Registering'.
+def getClass():
+    return TheHookupZoneNetAdapter # XXX
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class TheHookupZoneNetAdapter(BaseSiteAdapter): # XXX
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
+        self._setURL('http://' + self.getSiteDomain() + '/CriminalMinds/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','thupz') # XXX
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%b %d, %Y" # XXX
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'thehookupzone.net' # XXX
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/CriminalMinds/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/CriminalMinds/viewstory.php?sid=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Registered Users Only' in data \
+                or 'There is no such account on our website' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['cookiecheck'] = '1'
+        params['submit'] = 'Submit'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/CriminalMinds/user.php?action=login'
+        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "Member Account" not in d : #Member Account
+            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=4" # XXX
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+
+        # The actual text that is used to announce you need to be an
+        # adult varies from site to site.  Again, print data before
+        # the title search to troubleshoot.
+        if "Age Consent Required" in data: # XXX 
+            raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/CriminalMinds/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                catstext = [cat.string for cat in cats]
+                for cat in catstext:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                charstext = [char.string for char in chars]
+                for char in charstext:
+                    self.story.addToList('characters',char.string)
+
+            ## Not all sites use Genre, but there's no harm to
+            ## leaving it in.  Check to make sure the type_id number
+            ## is correct, though--it's site specific.
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
+                genrestext = [genre.string for genre in genres]
+                self.genre = ', '.join(genrestext)
+                for genre in genrestext:
+                    self.story.addToList('genre',genre.string)
+
+            ## Not all sites use Warnings, but there's no harm to
+            ## leaving it in.  Check to make sure the type_id number
+            ## is correct, though--it's site specific.
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
+                warningstext = [warning.string for warning in warnings]
+                self.warning = ', '.join(warningstext)
+                for warning in warningstext:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/CriminalMinds/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -0,0 +1,239 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return ThePetulantPoetessComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class ThePetulantPoetessComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId') +'&i=1')
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','tpp')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%Y/%m/%d"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.thepetulantpoetess.com'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'You must be a member to read this story.' in data \
+                or "The Ministry of Magic does not have a record of that password. " in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['cookiecheck'] = '1'
+        params['submit'] = 'Submit'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
+        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "My Account Page" not in d : #Member Account
+            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        chapters=soup.find('select', {'name' : 'sid'})
+        if chapters == None:
+            self.chapterUrls.append((self.story.getMetadata('title'),url))
+        else:
+            for chapter in chapters.findAll('option', value=re.compile(r"viewstory.php\?sid=\d+&i=1")):
+                # just in case there's tags, like <i> in chapter titles.
+                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['value']))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+        
+        # make sure that the story id is from the first chapter
+        self.story.setMetadata('storyId',self.chapterUrls[0][1].split('=')[1].split('&')[0])
+        
+        #locate the story on author's page
+        index = 1
+        found = 0
+        while found == 0:
+            asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')+"&page="+str(index)))
+        
+            for info in asoup.findAll('td', {'class' : 'highlightcolor1'}):
+                a = info.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+                if a != None:
+                    self.story.setMetadata('title',a.text)                
+                    found = 1
+                    break
+            index=index+1
+            
+        # extract metadata
+        b=info.find('b')
+        b.find('a').extract()
+        self.story.setMetadata('rating', b.text.split('[')[1].split(']')[0])
+        
+        info = info.findNext('td', {'colspan' : '2'}) 
+        for label in info.findAll('b'):
+            value = label.nextSibling
+            
+            if "Category" in label.text:
+                for cat in info.findAll('a'):
+                    self.story.addToList('category',cat.string)
+            
+            if "Characters" in label.text:
+                for char in value.split(', '):
+                    self.story.addToList('characters',char)
+                    
+            if "Genres" in label.text:
+                for genre in value.split(', '):
+                    if "General" not in genre:
+                        self.story.addToList('genre',genre)
+                    
+            if "Warnings" in label.text:
+                for warning in value.split(', '):
+                    if "none" not in warning.lower():
+                        self.story.addToList('warnings',warning)
+                        
+                        
+        info = info.findNext('td', {'class' : 'tblborder'}) 
+        info.find('b').extract()
+        self.setDescription(url,info)
+        
+        info = info.findNext('td', {'class' : 'highlightcolor2'}) 
+        for label in info.findAll('b'):
+            value = label.nextSibling
+            
+            if "Completed" in label.text:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+                    
+            if 'Published' in label.text:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label.text:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+                
+            if 'Word Count' in label.text:
+                self.story.setMetadata('numWords', value)
+            
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.findAll('table')[2].findAll('td')[1]
+        for a in div.findAll('div'):
+            a.extract()
+        div.name='div'
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -60,11 +60,6 @@ class TheQuidditchPitchOrgAdapter(BaseSiteAdapter): # XXX
        # Each adapter needs to have a unique site abbreviation.
        self.story.setMetadata('siteabbrev','tqdpch') # XXX

-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Harry Potter") # XXX
-
        # The date format will vary from site to site.
        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
        self.dateformat = "%m/%d/%Y" # XXX
@@ -55,11 +55,6 @@ class TwilightArchivesComAdapter(BaseSiteAdapter):
        # Each adapter needs to have a unique site abbreviation.
        self.story.setMetadata('siteabbrev','twa')

-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Twilight")
-
        # The date format will vary from site to site.
        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
        self.dateformat = "%d %b %y"
@@ -37,7 +37,6 @@ class TwilightedNetSiteAdapter(BaseSiteAdapter):
                               # Most sites that claim to be
                               # iso-8859-1 (and some that claim to be
                               # utf8) are really windows-1252.
-        self.story.addToList("category","Twilight")
        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
        self.password = ""
        
@@ -37,7 +37,6 @@ class TwiwriteNetSiteAdapter(BaseSiteAdapter):
                               # Most sites that claim to be
                               # iso-8859-1 (and some that claim to be
                               # utf8) are really windows-1252.
-        self.story.addToList("category","Twilight")
        self.is_adult = False
        self.username = "NoneGiven" # if left empty, twiwrite.net doesn't return any message at all.
        self.password = ""
@@ -53,11 +53,6 @@ class WalkingThePlankOrgAdapter(BaseSiteAdapter):
        # Each adapter needs to have a unique site abbreviation.
        self.story.setMetadata('siteabbrev','wtp')

-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Harry Potter")
-
        # The date format will vary from site to site.
        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
        self.dateformat = "%b %d, %Y"
@@ -0,0 +1,216 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return WolverineAndRogueComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class WolverineAndRogueComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/wrfa/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','wrfa')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%m/%d/%Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.wolverineandrogue.com'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/wrfa/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/wrfa/viewstory.php?sid=")+r"\d+$"
+
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        pt = soup.find('div', {'id' : 'pagetitle'})
+        a = pt.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.text)
+        
+        # Find authorid and URL from... author url.
+        a = pt.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/wrfa/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        rating=pt.text.split('(')[1].split(')')[0]
+        self.story.setMetadata('rating', rating)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/wrfa/'+chapter['href']))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        content=soup.find('div',{'class' : 'content'})
+        labels = soup.findAll('span',{'class':'label'})
+        
+        value = labels[0].previousSibling
+        svalue = ""
+        while value != None:
+            val = value
+            value = value.previousSibling
+        while "Categories" not in str(val):
+            svalue += str(val)
+            val = val.nextSibling
+        self.setDescription(url,svalue)
+        
+        
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value.split(' -')[0])
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Complete' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(value.split(' -')[0], self.dateformat))
+            
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/wrfa/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            i=1
+            for a in storyas:
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        break
+                    i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -56,11 +56,6 @@ class WraithBaitComAdapter(BaseSiteAdapter):
        # Each adapter needs to have a unique site abbreviation.
        self.story.setMetadata('siteabbrev','wb')

-        # If all stories from the site fall into the same category,
-        # the site itself isn't likely to label them as such, so we
-        # do.
-        self.story.addToList("category","Stargate")
-
        # The date format will vary from site to site.
        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
        self.dateformat = "%d  %b %Y"
@@ -141,7 +141,6 @@ class YourFanfictionComAdapter(BaseSiteAdapter):
        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
            
-        # use BeautifulSoup HTML parser to make everything easier to find.
        # because for some reason, this works while simple 'print data' errors on ascii conversion.
        # loopdata = data
        # chklen=5000
@@ -151,6 +150,7 @@ class YourFanfictionComAdapter(BaseSiteAdapter):
        #     logging.info("loopdata: %s" % loopdata[:chklen])
        #     loopdata = loopdata[chklen:]
            
+        # use BeautifulSoup HTML parser to make everything easier to find.
        soup = bs.BeautifulSoup(data)

        # Now go hunting for all the meta data and the chapter list.
@@ -191,9 +191,11 @@ class YourFanfictionComAdapter(BaseSiteAdapter):
            if 'Summary' in label:
                ## Everything until the next span class='label'
                svalue = ""
-                while not defaultGetattr(value,'class') == 'label':
+                while value and not defaultGetattr(value,'class') == 'label':
                    svalue += str(value)
                    value = value.nextSibling
+                # sometimes poorly formated desc (<p> w/o </p>) leads
+                # to all labels being included.
                self.setDescription(url,svalue)
                #self.story.setMetadata('description',stripHTML(svalue))

@@ -67,10 +67,8 @@ class BaseSiteAdapter(Configurable):
    def validateURL(self):
        return re.match(self.getSiteURLPattern(), self.url)

-    def __init__(self, config, url):
-        self.config = config
-        Configurable.__init__(self, config)
-        self.setSectionOrder(self.getConfigSection())
+    def __init__(self, configuration, url):
+        Configurable.__init__(self, configuration)
        
        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
        self.password = ""
@@ -79,7 +77,7 @@ class BaseSiteAdapter(Configurable):
        self.opener = u2.build_opener(u2.HTTPCookieProcessor(),GZipProcessor())
        self.storyDone = False
        self.metadataDone = False
-        self.story = Story()
+        self.story = Story(configuration)
        self.story.setMetadata('site',self.getConfigSection())
        self.story.setMetadata('dateCreated',datetime.datetime.now())
        self.chapterUrls = [] # tuples of (chapter title,chapter url)
@@ -138,7 +136,7 @@ class BaseSiteAdapter(Configurable):
                logging.debug("try code:"+code)
                return data.decode(code)
            except:
-                logging.info("code failed:"+code)
+                logging.debug("code failed:"+code)
                pass
        logging.info("Could not decode story, tried:%s Stripping non-ASCII."%decode)
        return "".join([x for x in data if ord(x) < 128])
@@ -215,8 +213,7 @@ class BaseSiteAdapter(Configurable):
            if self.getConfig('include_images') and \
                    not self.story.cover and \
                    self.getConfig('default_cover_image'):
-                self.story.addImgUrl(self,
-                                     None,
+                self.story.addImgUrl(None,
                                     #self.getConfig('default_cover_image'),
                                     self.story.formatFileName(self.getConfig('default_cover_image'),
                                                               self.getConfig('allow_unsafe_filename')),
@@ -289,13 +286,17 @@ class BaseSiteAdapter(Configurable):
    def setDescription(self,url,svalue):
        #print("\n\nsvalue:\n%s\n"%svalue)
        if self.getConfig('keep_summary_html'):
-            if isinstance(svalue,str) or isinstance(svalue,unicode):
+            if isinstance(svalue,basestring):
                svalue = bs.BeautifulSoup(svalue)
            self.story.setMetadata('description',self.utf8FromSoup(url,svalue))
        else:
            self.story.setMetadata('description',stripHTML(svalue))
        #print("\n\ndescription:\n"+self.story.getMetadata('description')+"\n\n")

+    def setCoverImage(self,storyurl,imgurl):
+        if self.getConfig('include_images'):
+            self.story.addImgUrl(storyurl,imgurl,self._fetchUrlRaw,cover=True)
+
    # This gives us a unicode object, not just a string containing bytes.
    # (I gave soup a unicode string, you'd think it could give it back...)
    # Now also does a bunch of other common processing for us.
@@ -311,7 +312,7 @@ class BaseSiteAdapter(Configurable):
                # some pre-existing epubs have img tags that had src stripped off.
                if img.has_key('src'):
                    img['longdesc']=img['src']
-                    img['src']=self.story.addImgUrl(self,url,img['src'],fetch,
+                    img['src']=self.story.addImgUrl(url,img['src'],fetch,
                                                    coverexclusion=self.getConfig('cover_exclusion_regexp'))

        for attr in soup._getAttrMap().keys():
@@ -0,0 +1,325 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+#
+# Author:        Pau Sanchez (contact@pausanchez.com)
+# Version:       v1.0
+# Last Modified: 2010/09/15
+# 
+# For the latest version check out:
+#   http://www.codigomanso.com/en/projects
+# 
+# My blog:
+#   http://www.codigomanso.com/en/  - English Version
+#   http://www.codigomanso.com/es/  - Spanish Version
+#
+
+import sys
+import os
+import re
+import urllib
+
+class bbcode2html:
+  '''
+  This class gets a parsed BBCode and transforms it to valid HTML
+
+  Useful functions of this class:
+    html
+    convertToHTML
+
+  Example:
+    > parser = bbcodeparser ()
+    > parser.parse ('[b]bold[/b]')
+    > bbcode2html (parser).html()
+    <b>bold</b>
+
+    # This is faster for huge strings but changes the parser object internally
+    > bbcode2html (parser).html(doDeepCopy = False)
+    <b>bold</b>
+  '''
+  def __init__ (self, parser):
+    self._parser = parser
+    return
+
+  def html (self, allowClassAttr = False, doDeepCopy = True, parser = None):
+    '''
+    Convert current parsed code to HTML
+
+    Example:
+      code = bbcodeparser ('[b]bold[/b]')
+      code.html() -> '<b>bold</b>'
+    '''
+    if parser is None:
+      parser = self._parser
+
+    tokens = parser
+    if instanceof (parser, bbcodeparser):
+      tokens = parser.getTokens()
+
+    return bbcode2html.convertToHTML (tokens, allowClassAttr = allowClassAttr, doDeepCopy = doDeepCopy)
+
+  @staticmethod
+  def htmlString (string):
+    toReplace = {
+      u'<' : '&lt;',
+      u'>' : '&gt;',
+      u'"' : "&quot;",
+      u'&' : "&amp;"
+    }
+    for entity in toReplace:
+      string = string.replace(entity, toReplace[entity])
+    return string
+
+  @staticmethod
+  def getValidTags ():
+    simpleTags = ['b', 'u', 'i', 'sup', 'sub', 'ul', 'ol', 'li', 'table', 'tr', 'th', 'td', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6']
+    validTags = {
+      'p'         : { 'color' : 'color', 'size' : 'size', 'font' : 'font' },
+      'color'     : { 'color' : 'color' },
+      'size'      : { 'size' : 'size' },
+      'font'      : { 'font' : 'font' },
+      'img'       : { 'alt' : 'alt', 'title' : 'title', 'width' : 'width' , 'height' : 'height', 'img' : 'img'},
+      'url'       : { 'href' : 'href', 'url' : 'href', 'link'  : 'href', 'title' : 'title' },
+      's'         : { },
+      'code'      : { },
+      'quote'     : { },
+      'list'      : { 'list' : 'type' },
+      'email'     : { 'email': 'href'},
+      'google'    : { 'google':  'google'},
+      'wikipedia' : { 'wikipedia' : 'wikipedia', 'language' : 'language', 'lang' : 'lang'}
+    }
+
+    for tag in simpleTags:
+      validTags[tag] = { }
+    return validTags
+
+  @staticmethod
+  def convertToHTML (tokens, allowClassAttr = False, validTags = None, doDeepCopy = True):
+    '''
+    Convert internally parsed BBCode to XHTML
+
+    @doDeepCopy 
+      True:  it does a deep copy of tokens so this list will remain unchanged
+      False: tokens will be modified internally, but the output will be produced like 5x faster
+             it's a good idea to use False only when this is the last operation
+    '''
+    # do a deep copy
+    if doDeepCopy:
+      import copy
+      tokens = copy.deepcopy (tokens)
+
+    # filter invalid tags and attributes
+    if validTags is None:
+      validTags = bbcode2html.getValidTags()    
+
+    bbcode2html._filterInvalidTagsAndAttributes (tokens, validTags, allowClassAttr)
+    
+    # Start to convert
+    index       = 0
+    tokenLength = len (tokens)
+
+    # use a list for the output (an order of magnitude faster than using string concatenation)
+    htmlList = []
+    lastListOpener = []
+
+    while index < tokenLength:
+
+      if isinstance (tokens [index], basestring):
+        htmlList.append (bbcode2html.htmlString (tokens [index]))
+        index += 1
+        continue
+
+      token     = tokens[index] 
+      tag       = token['tag']  # opening or closing simple tag. e.g: 'b', '/b', '/u', ...
+      tagName   = (tag[1:] if tag[0] == '/' else tag)
+      tagOpener = (u'/' if tag[0] == '/' else u'')
+      tokenArgs = (token['args'] if 'args' in token else {})
+
+      # opening or closing simple tag COLOR / SIZE
+      if (tagName in ['p', 'color', 'size', 'font']): 
+        style  = ''
+        style += ((u' color: ' + tokenArgs['color'] + u';') if ('color' in tokenArgs) else '')
+        style += ((u' font-size: ' + tokenArgs['size'] + u'pt;') if ('size' in tokenArgs) else '')
+        style += ((u' font-family: ' + tokenArgs['font'] + u';') if ('font' in tokenArgs) else '')
+        style  = style.strip()
+
+        pArgs = {}
+        if style != '':
+          pArgs ['style'] = style
+
+        if 'class' in tokenArgs:
+          pArgs ['class'] = tokenArgs['class']
+
+        if ('args' not in token) and (tagName != 'p'):
+          if (tagOpener == '/'): # if closing tag, close it
+            htmlList.append (u'</span>')
+          index += 1
+          continue
+
+        if tagName != 'p':
+          tag = tagOpener + u'span'
+
+        htmlList.append (bbcode2html.xml (tag, pArgs))
+
+      # IMG tag
+      elif tag == 'img' and (index+2 < tokenLength):
+        if 'img' in tokenArgs:
+          # has the form of <width>x<height> ?
+          sizeMatch = re.match (u'^\s*(\d+)[xX](\d+)\s*$', tokenArgs['img'])
+          if sizeMatch is not None:
+            tokenArgs['width']  = sizeMatch.group(1)
+            tokenArgs['height'] = sizeMatch.group(2)
+          # then assume is the alternative text
+          else:
+            tokenArgs['alt'] = tokenArgs['img']
+          del tokenArgs['img']
+
+        # add the source of the image
+        tokenArgs ['src'] = tokens[index+1]
+
+        # [img]http://www.whatever.com/pic.jpg[/img]
+        htmlList.append (
+          bbcode2html.xml ('img', tokenArgs, soloTag=True)
+        )
+        index += 2 # skip next token and closing tag
+
+      # URL tag
+      elif tag == 'url':
+        if ('args' not in token) and (index+2 < tokenLength):
+          # [url]http://www.google.com[/url]
+          htmlList.append (bbcode2html.xml ('a', { 'href' : tokens[index+1] }))
+        else:
+          # [url=http://www.google.com]Google[/url]
+          # [url link=http://www.google.com title="This is Google"]Google[/url]
+          htmlList.append (bbcode2html.xml ('a', tokenArgs))
+  
+      # URL closing tag (sometimes needed)
+      elif (tag == '/url') or (tag == '/email'):
+        htmlList.append (u'</a>')
+
+      # Email tag
+      elif tag == 'email':
+        if ('args' not in token) and (index+2 < tokenLength):
+          # [email]asdf@asdf.com]
+          htmlList.append (bbcode2html.xml ('a', { 'href' : u'mailto:' + tokens[index+1].strip() }))
+        else:
+          # [email=asdf@asfd.com]john smith[/email]
+          if 'href' in tokenArgs:
+            tokenArgs['href'] = u'mailto:' + tokenArgs['href']
+          htmlList.append (bbcode2html.xml ('a', tokenArgs))
+  
+      elif tagName == 'list':
+        if tagOpener == '/':
+          htmlList.append (bbcode2html.xml (u'/' + lastListOpener.pop()))
+        else:
+          if ('type' not in tokenArgs):
+            htmlList.append (bbcode2html.xml (tagOpener + u'ul', tokenArgs))
+            lastListOpener.append ('ul')
+          else:
+            htmlList.append (bbcode2html.xml (tagOpener + u'ol', tokenArgs))
+            lastListOpener.append ('ol')
+
+      elif tagName == '*':
+        htmlList.append (bbcode2html.xml (tagOpener + u'li', tokenArgs))
+
+      elif (tagName == 's'):
+        tokenArgs['style'] =  'text-decoration: line-through;'
+        htmlList.append (bbcode2html.xml (tagOpener + u'span', tokenArgs))
+
+      elif (tagName == 'code'):
+        htmlList.append (bbcode2html.xml (tagOpener + u'pre', tokenArgs))
+
+      elif (tagName == 'quote'):
+        htmlList.append (bbcode2html.xml (tagOpener + u'blockquote', tokenArgs))
+
+      elif (tagName == 'google'):
+        htmlList.append (
+          bbcode2html.xml (
+            tagOpener + u'a', 
+            {'href' : 'http://www.google.com/search?q=' + urllib.quote_plus (tokens[index+1])},
+            tokens[index+1]
+          )
+        )
+        index += 2
+
+      elif (tagName == 'wikipedia'):
+        subdomain = 'www'
+        for arg  in ['lang', 'language', 'wikipedia']:
+          if arg in tokenArgs:
+            subdomain = tokenArgs[arg] 
+
+        htmlList.append (
+          bbcode2html.xml (
+            tagOpener + u'a', 
+            {'href' : 'http://' + subdomain + '.wikipedia.org/wiki/' + tokens[index+1].replace (' ', '_')},
+            tokens[index+1]
+          )
+        )
+        index += 2
+
+      elif (tagName in validTags):
+        htmlList.append (
+          bbcode2html.xml (tag, tokenArgs)
+        )
+
+      else:
+        # ignore this tag
+        pass 
+
+      index += 1
+
+    return ''.join (htmlList)
+
+  @staticmethod
+  def _filterInvalidTagsAndAttributes (tokens, validTags, allowClassAttr):
+    '''
+    Helper function to filter out invalid attributes from the tokens list
+    '''
+    # add 'class' attribute as valid (mapping 'class' itself)
+    if allowClassAttr:
+      for attr in validTags:
+        validTags[attr]['class'] = 'class'
+
+    # remove invalid attributes from tokens
+    for tindex in range(0, len(tokens)):
+      if isinstance (tokens[tindex], dict) and ('args' in tokens[tindex]) and (tokens[tindex]['tag'] in validTags):
+        validList = validTags[tokens[tindex]['tag']]
+
+        filteredArgs = {}
+        for arg in tokens[tindex]['args']:
+          if arg in validList:
+            # rename the argument
+            filteredArgs[validList[arg]] = tokens[tindex]['args'][arg]
+          else:
+            pass # do not include this arg in the filteredArgs
+
+        tokens[tindex]['args'] = filteredArgs
+
+    return
+
+  @staticmethod
+  def xml (tag, attrs = {}, text = None, soloTag = False):
+    '''
+    Helper function to produce valid XML output
+    '''
+    xml = u'<' + tag.lower()
+
+    # make sure we sort attributes alphabetically (for deterministic output)
+    # Faster but non-deterministic:
+    #  for (key, value) in attrs.iteritems():
+    #    xml += u' ' + key + u'="' + value + u'"'
+    for key in sorted (attrs.keys()):
+      xml += u' ' + key + u'="' + attrs[key] + u'"'
+      
+    # close tag
+    if text is None:
+      if soloTag:
+        xml += u' />'
+      else:
+        xml += u'>'
+    else:
+      xml += u'>' + text + u'</' + tag.lower() + '>'
+
+    return xml
+
+
+
@@ -0,0 +1,77 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+#
+# Author:        Pau Sanchez (contact@pausanchez.com)
+# Version:       v1.0
+# Last Modified: 2010/09/15
+# 
+# For the latest version check out:
+#   http://www.codigomanso.com/en/projects
+# 
+# My blog:
+#   http://www.codigomanso.com/en/  - English Version
+#   http://www.codigomanso.com/es/  - Spanish Version
+#
+
+import sys
+import os
+import re
+import hashlib
+
+class bbcodebuilder:
+  '''
+  This class helps to build BBCode programmatically.
+
+  The function names are used as the tag name, then the first parameter
+  is the string that goes inside the tags and any extra parameter is 
+  appended as a parameter to the tag
+
+  Examples:
+    > bbcode = bbcodebuilder()  # create a instance!
+
+    > print bbcode.b ('bold')
+    [b]bold[/b]
+
+    > print bbcode.color ('this goes in red', 'red')
+    [color=red]this goes in red[/color]
+
+    > print bbcode.url ('Google', 'http://www.google.com')
+    [url=http://www.google.com]Google[/url]
+    
+    > print bbcode.alist('item 1', 'item 2')
+    [list=a]
+      [*]item 1
+      [*]item 2
+    [/list]
+
+
+  This solution is based on the recipe found on:
+    http://code.activestate.com/recipes/576831-simple-bbcode-support/
+  '''
+
+  def __getattr__(self, name):
+    '''
+    This is a generic getter that returns a function which gets the first parameter
+    as the string that goes between the tags, and extra parameters as tag parameters.
+
+    The name of the attribute is used as the tag name
+    '''
+    class bbcodebuilder_helper:
+      def __init__(self, name):
+        self._name = name
+
+      def __call__(self, string, *args):
+        return u'[{0}{1}]{2}[/{0}]'.format(self._name, (u'=' + u','.join(map(str, args))) if args else u'', string)
+    
+    return bbcodebuilder_helper (name)
+
+  def list(self, *items):
+    return u'[list]' + u''.join(map(lambda item: u"\n  [*]" + item, items)) + u"\n[/list]"
+
+  def nlist(self, *items):
+    return u'[list=1]' + u''.join(map(lambda item: u"\n  [*]" + item, items)) + u"\n[/list]"
+
+  def alist(self, *items):
+    return u'[list=a]' + u''.join(map(lambda item: u"\n  [*]" + item, items)) + u"\n[/list]"
+
+
@@ -0,0 +1,251 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+#
+# Author:        Pau Sanchez (contact@pausanchez.com)
+# Version:       v1.0
+# Last Modified: 2010/09/15
+# 
+# For the latest version check out:
+#   http://www.codigomanso.com/en/projects
+# 
+# My blog:
+#   http://www.codigomanso.com/en/  - English Version
+#   http://www.codigomanso.com/es/  - Spanish Version
+#
+
+import sys
+import os
+import re
+import hashlib
+
+class bbcodeparser:
+  '''
+  This class parses BBCode into a internal structure to allow later processing and
+  conversion to HTML.
+
+  The parser tries to fix invalid code (like unclosed tags)
+
+  Useful URLs:
+    http://en.wikipedia.org/wiki/BBCode
+    http://www.bbcode.org/reference.php
+
+  Example:
+    > bbcode = bbcodeparser ()
+    > bbcode.parse ('[b]text in bold[/b]').html()
+    <b>text in bold</b>
+
+    # dump HTML
+    > bbcode.parse ('[p][color=red]text in red').html()
+    <p><span style="color:red;">text in red</span></p>
+
+    # dump fixed BBCode
+    > bbcode.parse ('[p][color=red]text in red').bbcode()
+    [p][color=red]text in red[/color][/p]
+
+    > bbcode.parse ('This [b][i]code[/b] will be fixed[/invalid]').bbcode()
+    This [b][i]code[/i][/b] will be fixed
+
+    # dump fixed bbcode
+    > str (bbcodeparse ('This [b][i]code[/b] will be fixed[/invalid]'))
+    This [b][i]code[/i][/b] will be fixed
+  '''
+  _bbcode = ''
+  _tokens = []
+
+  def __init__ (self, bbcode = '', fixInvalidCode = True):
+    ''' Initialize and parse bbcode string (if any is given)
+    '''
+    self.parse (bbcode, fixInvalidCode)
+    return
+
+  def __str__ (self):
+    return self.bbcode()
+
+  def parse (self, bbcode = None, fixInvalidCode = True):
+    '''
+    It will parse and return the token list, trying to fix tags if
+    fixInvalidCode is True
+
+    It will return the current object to allow chaining
+
+    Example:
+      code = bbcode()
+      code.parse ('<b>bold</b>', True) -> 
+      code.parse ('<b>bold<i>italics</b>', True) -> internally will add the missing '</i>'
+    '''
+    if bbcode is not None:
+      self._bbcode = bbcode
+      self._tokens = self.tokenize (bbcode)
+      if fixInvalidCode:
+        self._tokens = self.fixWrongTags (self._tokens)
+
+    return self
+
+  # return ALL tokens
+  def getTokens (self):
+    return self._tokens
+
+  def bbcode (self):
+    '''
+    Dump BBCode again. This is useful for dumping valid BBCode
+    '''
+    bbcode = []
+    for token in self._tokens:
+      if token is None:
+        continue
+
+      if isinstance (token, basestring):
+        bbcode.append (token.replace (u'[', u'\[').replace (u']', u'\]'))
+        continue
+      
+      tag       = token['tag']  # opening or closing simple tag. e.g: 'b', '/b', '/u', ...
+      tagOpener = (u'/' if tag[0] == u'/' else u'')
+
+      if (tagOpener == '/') or ('args' not in token):
+        bbcode.append (u'[' + tag + u']')
+      else:
+        # process args
+        argstr = ''
+
+        # the arg with the same name as the tag repersents the '=whatever'
+        if tag in token['args']:
+          if re.match ('\s|"', token['args'][tag]) is None:
+            argstr = u'=' + token['args'][tag]
+          else:
+            argstr = u'="' + token['args'][tag].replace (u'"', u'\"') + u'"'
+
+        for (k,v) in token['args'].iteritems():
+          if k == tag: # already processed
+            continue
+          argstr += ' ' + k + u'="' + v.replace (u'"', u'\"') + u'"'
+          
+        bbcode.append (u'[' + tag + argstr + ']')
+
+    return u''.join (bbcode)
+
+  def html (self, allowClassAttr = False, doDeepCopy = True):
+    '''
+    Convert current parsed code to HTML
+
+    @allowClassAttr
+      Is something like [b class="asdf"] allowed?
+
+    @doDeepCopy 
+      True:  it does a deep copy of tokens so this list will remain unchanged
+      False: tokens will be modified internally, but the output will be produced like 5x faster
+             it's a good idea to use False when the string parsed is huge and this is the
+             last operation on the string
+
+    Example:
+      code = bbcode ('[b]bold[/b]')
+      code.html() -> '<b>bold</b>'
+    '''
+    from bbcode2html import bbcode2html
+    return bbcode2html.convertToHTML (self._tokens, allowClassAttr = allowClassAttr, doDeepCopy = doDeepCopy)
+
+
+  @staticmethod
+  def fixWrongTags (inTokenList):
+    ''' Add missing tokens that have not been closed properly and try to fix some scenarios
+    '''
+    opened       = []
+    outTokenList = []
+    for token in inTokenList:
+      # normal string... do nothing
+      if isinstance(token, basestring):
+        outTokenList.append (token)
+      else:
+        # if starts with '/' is closing a tag
+        if token['tag'][0] == '/':
+          while (len (opened) > 0) and (opened[-1] != token['tag'][1:]):
+            outTokenList.append ({'tag' : '/' + opened[-1] })
+            del opened[-1]
+
+          if len(opened):
+            del opened[-1]
+            outTokenList.append (token)
+
+        # opening tag
+        else:
+          # if I open the same tag I opened before, close it, and open it again
+          if (len(opened) > 0) and (token['tag'] == opened[-1]):
+            outTokenList.append ({'tag' : '/' + opened[-1] })
+          else:
+            opened.append (token['tag'])
+          outTokenList.append (token)
+
+    # close all elements that have not been closed
+    while len(opened):
+      outTokenList.append ({'tag' : '/' + opened[-1] })
+      del opened[-1]
+
+    return outTokenList
+
+  @staticmethod
+  def tokenize(code):
+    '''
+    Tokenize BBCode tags and parameters
+
+    Return the token list using a internal format. See the example:
+      [
+        { 'tag' : 'p', 'args' : { 'font' : 'arial' } },
+        'This is ',
+        { 'tag' : 'url', 'args' : {'url' : 'http://www.google.com'} },
+        'a link to google',
+        { 'tag' : '/url' },
+        { 'tag' : '/p' }
+      ]
+    '''
+    re_tags    = re.compile (r'(\[[^]]+\])', re.DOTALL | re.UNICODE)
+    re_tagName = re.compile (r'\[([^]=\s]+)([^]]*)\]',  re.DOTALL | re.UNICODE)
+    #re_tagArgs = re.compile (r'\s*([^=]*)=(("([^"]+)")|([^\s]+))',  re.DOTALL | re.UNICODE)
+    re_tagArgs = re.compile (r'\s*([\w]*)=(("([^"]+)")|([^\s]+))',  re.DOTALL | re.UNICODE)
+
+    # get a unique name and replace escaped braces 
+    unique = hashlib.md5(code).hexdigest()
+    code   = code.replace ('\[', unique+'_OPEN_BRACE')
+    code   = code.replace ('\]', unique+'_CLOSE_BRACE')
+
+    splitted = re_tags.split(code)
+
+    outTokenList = []
+    for token in splitted:
+      if len(token) == 0:
+        continue
+      
+      if token[0] == '[':
+        match   = re_tagName.match (token)
+        tagName = match.group(1)
+        tagArgs = match.group(2)
+        
+        tagToken = { 'tag' : tagName.lower() }
+
+        # parse arguments (if any)
+        if len(tagArgs) > 0:
+          allArgs = re_tagArgs.findall(tagArgs)
+
+          tagArgs = {}
+          for arg in allArgs:
+            # if the argument has no name, use the tagName itself
+            argName  = (arg[0] if arg[0] != '' else tagName)
+            argValue = (arg[3] if (arg[1][0] == '"') else arg[4])
+
+            tagArgs[argName.lower()] = argValue.replace ('\"', '"')
+
+          tagToken['args'] = tagArgs
+
+        outTokenList.append (tagToken)
+
+      # append the text as it is
+      else:
+        outTokenList.append (token)
+
+    # restore escaped braces back (once code is parsed)
+    restoredTokenList = []
+    for token in outTokenList:
+      if isinstance (token, basestring):
+        token = token.replace (unique+'_OPEN_BRACE', '[').replace (unique+'_CLOSE_BRACE', ']')
+      restoredTokenList.append (token)
+
+    return restoredTokenList
+
@@ -0,0 +1,81 @@
+AUTHOR
+  Pau Sanchez
+  http://www.codigomanso.com/
+
+VERSION:
+  bbcodeutils v1.0
+  
+LICENSE
+  This code is licensed under Creative Commons Attribution 3.0
+  http://creativecommons.org/licenses/by/3.0/
+
+  You can use this python module or any part of the code you want as long as you add
+  my name as a contributor to your project.
+  
+DESCRIPTION
+  This module can be used to produce HTML from BBCode, to generate BBCode or to fix invalid BBCode.
+
+  The classes are:
+    - bbcodeparser
+    - bbcodebuilder
+    - bbcode2html
+  
+  You can use bbcodeparser to parse BBCode and to produce output in any format you want.
+
+  Open the python file to find more information and examples of use of each class. It can
+  be a good idea to check the test.py for examples
+
+  To run the unit tests:
+    > python test.py
+  
+  To run the performance test:
+    > python test.py BBCodeTests.performanceTest  
+
+
+EXAMPLES OF BBCode:
+
+  [b] -> bold
+  [u] -> underline
+  [i] -> italic
+
+  [center] -> center the text inside
+  [color=XXX] -> change color of text
+  [size=XXX] -> change size of text
+
+  Lists:
+  [ul] -> unordered list
+  [ol] -> ordered list
+  [li] -> list item
+
+  [list] -> start unordered list
+  [*]    -> list item
+  [list=1] -> start a list of numbers
+  [list=a] -> start a list of alphabetic characters
+
+  Advanced:
+  [url] -> link to url
+  [url=http://link/url/]text[/url]  
+  [url link=http://link/url/ title="This is the title"]text[/url]  
+
+  [img]http://to/image[/img]
+  [img=230x330]http://to/image[/img]
+  [img="Alt text here"]http://to/image[/img]
+  [img="Alt text here" width=320 height=240]http://to/image[/img]
+
+  [email]asdf@asdf.com[/email]
+  [email=john@asdf.com]John Smith[/email]
+
+  [google]search this[/google]
+  [wikipedia]Tom Hanks[/wikipedia]
+  [wikipedia lang=es]Tom Hanks[/wikipedia]
+
+  Tables:
+  [table]
+  [tr]
+  [th]
+  [td]
+
+  Advanced:
+  [google]
+  [wikipedia]
+
@@ -0,0 +1,420 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+#
+# Author:        Pau Sanchez (contact@pausanchez.com)
+# Version:       v1.0
+# Last Modified: 2010/09/15
+# 
+# For the latest version check out:
+#   http://www.codigomanso.com/en/projects
+# 
+# My blog:
+#   http://www.codigomanso.com/en/  - English Version
+#   http://www.codigomanso.com/es/  - Spanish Version
+#
+
+from bbcodeparser import bbcodeparser
+from bbcodebuilder import bbcodebuilder
+
+import random
+import unittest
+
+class BBCodeTests(unittest.TestCase):
+  def setUp (self):
+    self.bbcode = bbcodeparser()
+    return
+
+  def testConstructor (self):
+    self.assertEqual (bbcodeparser ('whatever').html(), 'whatever')
+    self.assertEqual (bbcodeparser ('[b]bold[/b]').html(), '<b>bold</b>')
+    self.assertEqual (str (bbcodeparser ('[b]bold[/b]')), '[b]bold[/b]')
+    return
+
+  def testBold (self):
+    self.assertEqual (self.bbcode.parse ('whatever').html(), 'whatever')
+    self.assertEqual (self.bbcode.parse ('[b]bold[/b]').html(), '<b>bold</b>')
+    self.assertEqual (self.bbcode.parse ('[B]bold[/b]').html(), '<b>bold</b>')
+    self.assertEqual (self.bbcode.parse ('this is [B]bold[/B]').html(), 'this is <b>bold</b>')
+    return
+
+  def testItalic (self):
+    self.assertEqual (self.bbcode.parse ('[i]italic[/i]').html(), '<i>italic</i>')
+    return
+
+  def testUnderline (self):
+    self.assertEqual (self.bbcode.parse ('[u]italic[/u]').html(), '<u>italic</u>')
+    return
+
+  def testURLs (self):
+    self.assertEqual (
+      self.bbcode.parse ('[url]http://www.google.com[/url]').html(),
+      '<a href="http://www.google.com">http://www.google.com</a>'
+    )
+    self.assertEqual (
+      self.bbcode.parse ('[url=http://www.google.com]Google[/url]').html(),
+      '<a href="http://www.google.com">Google</a>'
+    )
+    self.assertEqual (
+      self.bbcode.parse ('[url="http://www.google.com"]Google[/url]').html(),
+      '<a href="http://www.google.com">Google</a>'
+    )
+    self.assertEqual (
+      self.bbcode.parse ('[url="http://www.google.com" title="Search Engine"]Google[/url]').html(),
+      '<a href="http://www.google.com" title="Search Engine">Google</a>'
+    )
+    self.assertEqual (
+      self.bbcode.parse ('[url link="http://www.google.com"]Google[/url]').html(),
+      '<a href="http://www.google.com">Google</a>'
+    )
+    return
+
+  def testPTag (self):
+    self.assertEqual (
+      self.bbcode.parse ('[p color=#0000ff]blue[/p]').html(),
+      u'<p style="color: #0000ff;">blue</p>'
+    )
+    self.assertEqual (
+      self.bbcode.parse ('[p size=12]12pt font[/p]').html(),
+      u'<p style="font-size: 12pt;">12pt font</p>'
+    )
+
+    self.assertEqual (
+      self.bbcode.parse ('[p font=arial]arial[/p]').html(),
+      u'<p style="font-family: arial;">arial</p>'
+    )
+
+    self.assertEqual (
+      self.bbcode.parse ('[p font=arial color=blue size=14]blue 14pt arial').html(),
+      u'<p style="color: blue; font-size: 14pt; font-family: arial;">blue 14pt arial</p>' 
+    )
+
+    self.assertEqual (
+      self.bbcode.parse ('[p class=whatever]text[/p]').html(),
+      u'<p>text</p>' 
+    )
+    return
+
+  def testColorTag (self):
+    self.assertEqual (
+      self.bbcode.parse ('[color=#0000ff]blue[/color]').html(),
+      u'<span style="color: #0000ff;">blue</span>'
+    )
+    return
+
+  def testSizeTag (self):
+    self.assertEqual (
+      self.bbcode.parse ('[size=12]12pt font[/size]').html(),
+      u'<span style="font-size: 12pt;">12pt font</span>'
+    )
+    return
+
+  def testEmail(self):
+    self.assertEqual (
+      self.bbcode.parse ('[email]asdf@asdf.com[/email]').html(),
+      u'<a href="mailto:asdf@asdf.com">asdf@asdf.com</a>'
+    )
+
+    self.assertEqual (
+      self.bbcode.parse ('[email=john@smith.com]John Smith[/email]').html(),
+      u'<a href="mailto:john@smith.com">John Smith</a>'
+    )
+    return
+
+  def testImgTag (self):
+    self.assertEqual (
+      self.bbcode.parse ('[img]http://www.codigomanso.com/image.jpg[/img]').html(),
+      u'<img src="http://www.codigomanso.com/image.jpg" />'
+    )
+
+    self.assertEqual (
+      self.bbcode.parse ('[img="This is the ALT of the image"]http://www.codigomanso.com/image.jpg[/img]').html(),
+      u'<img alt="This is the ALT of the image" src="http://www.codigomanso.com/image.jpg" />'
+    )
+
+    self.assertEqual (
+      self.bbcode.parse ('[img=320x200]http://www.codigomanso.com/image.jpg[/img]').html(),
+      u'<img height="200" src="http://www.codigomanso.com/image.jpg" width="320" />'
+    )
+
+    self.assertEqual (
+      self.bbcode.parse ('[img=320x200 title="Image Test"]http://www.codigomanso.com/image.jpg[/img]').html(),
+      u'<img height="200" src="http://www.codigomanso.com/image.jpg" title="Image Test" width="320" />'
+    )
+
+    self.assertEqual (
+      self.bbcode.parse ('[img="whatever" width=320 height="212" title="Image Test"]http://www.codigomanso.com/image.jpg[/img]').html(),
+      u'<img alt="whatever" height="212" src="http://www.codigomanso.com/image.jpg" title="Image Test" width="320" />'
+    )
+    return
+
+  def testGoogleURL (self):
+    self.assertEqual (
+      self.bbcode.parse ('[google]asdf[/google]').html(),
+      u'<a href="http://www.google.com/search?q=asdf">asdf</a>'
+    )
+    self.assertEqual (
+      self.bbcode.parse ('[google]Tom Hanks[/google]').html(),
+      u'<a href="http://www.google.com/search?q=Tom+Hanks">Tom Hanks</a>'
+    )
+    return
+    
+  def testWikipediaURL (self):
+    self.assertEqual (
+      self.bbcode.parse ('[wikipedia]Tom Hanks[/wikipedia]').html(),
+      u'<a href="http://www.wikipedia.org/wiki/Tom_Hanks">Tom Hanks</a>'
+    )
+
+    self.assertEqual (
+      self.bbcode.parse ('[wikipedia language=en]Tom Hanks[/wikipedia]').html(),
+      u'<a href="http://en.wikipedia.org/wiki/Tom_Hanks">Tom Hanks</a>'
+    )
+    
+    self.assertEqual (
+      self.bbcode.parse ('[wikipedia lang=es]Tom Hanks[/wikipedia]').html(),
+      u'<a href="http://es.wikipedia.org/wiki/Tom_Hanks">Tom Hanks</a>'
+    )
+    
+    self.assertEqual (
+      self.bbcode.parse ('[wikipedia=es]Tom Hanks[/wikipedia]').html(),
+      u'<a href="http://es.wikipedia.org/wiki/Tom_Hanks">Tom Hanks</a>'
+    )
+    return
+    
+  def testListTags (self):
+    self.assertEqual (
+      self.bbcode.parse ('[ul][li]item 1[/li][li]item 2[/li][/ul]').html(),
+      u'<ul><li>item 1</li><li>item 2</li></ul>'
+    )
+    
+    self.assertEqual (
+      self.bbcode.parse ('[ol][li]item 1[/li][li]item 2[/li][/ol]').html(),
+      u'<ol><li>item 1</li><li>item 2</li></ol>'
+    )
+
+    self.assertEqual (
+      self.bbcode.parse ('[list][li]item 1[/li][li]item 2[/li][/list]').html(),
+      u'<ul><li>item 1</li><li>item 2</li></ul>'
+    )
+
+    self.assertEqual (
+      self.bbcode.parse ('[list][*]item 1[*]item 2[/list]').html(),
+      u'<ul><li>item 1</li><li>item 2</li></ul>'
+    )
+
+    self.assertEqual (
+      self.bbcode.parse ('[list=1][li]item 1[/li][li]item 2[/li][/list]').html(),
+      u'<ol type="1"><li>item 1</li><li>item 2</li></ol>'
+    )
+    return
+
+  def testInvalidCode (self):
+    self.assertEqual (self.bbcode.parse ('[invalid]valid text[/invalid]').html(), 'valid text')
+    self.assertEqual (
+      self.bbcode.parse ('[b]bold and [i]italics[/b]').html(),
+      '<b>bold and <i>italics</i></b>'
+    )
+    self.assertEqual (
+      self.bbcode.parse ('[/b]invalid[/b][/p]').html(),
+      'invalid'
+    )
+    self.assertEqual (
+      self.bbcode.parse ('[p][b]bold').html(),
+      '<p><b>bold</b></p>'
+    )
+    self.assertEqual (
+      self.bbcode.parse ('[p][b]a <b>').html(),
+      '<p><b>a &lt;b&gt;</b></p>'
+    )
+
+    self.assertEqual (
+      self.bbcode.parse ('[ol][li]item 1[li]item 2[/li][/ol]').html(),
+      u'<ol><li>item 1</li><li>item 2</li></ol>'
+    )
+
+    self.assertEqual (
+      self.bbcode.parse ('[b]\[b\] stands for [b]bold[/b]').html(),
+      u'<b>[b] stands for </b><b>bold</b>'
+    )
+    return
+
+  def testEscapedBrackets (self):
+    self.assertEqual (
+      self.bbcode.parse ('\[b\]not bold\[/b\]').html(),
+      u'[b]not bold[/b]'
+    )
+
+    self.assertEqual (
+      self.bbcode.parse ('[b]\[b\] stands for bold[/b]').html(),
+      u'<b>[b] stands for bold</b>'
+    )
+
+    self.assertEqual (
+      self.bbcode.parse ('\[b\][b]stands for bold[/b]').html(),
+      u'[b]<b>stands for bold</b>'
+    )
+
+    self.assertEqual (
+      self.bbcode.parse ('\[b\][b]stands for bold[/b] just like <b> in HTML').html(),
+      u'[b]<b>stands for bold</b> just like &lt;b&gt; in HTML'
+    )
+
+  def testBigExample (self):
+    inputText = """check this out
+
+    [h1 class=circle]heading[/h1]
+
+    [p size=14 color=blue font="verdana, Times New Roman"]This is [b] bold [/b] and this [i]italic[/i] and this is [color=red]red[/color] and this is [color="red"]also red[/color].
+    [/p]
+
+    fix [b][i]bold [font=verdana][size=12]and[/size][/font] italic[/b]
+    [img]http://www.codigomanso.com/b.jpg[/img]
+    [url]http://www.codigomanso.com/[/url]
+    [url=http://www.codigomanso.com/]Codigo Manso[/url]
+    [uRl link=http://www.codigomanso.com title="Codigo Manso Blog"]Codigo Manso[/url]
+
+    [ul]
+     [Li]item 1[/Li]
+     [li]item 2[/LI]
+    [/UL]
+
+    [list=1 ]
+     [*]item 1
+     [*]item 2
+    [/list]
+
+    [table class="big"]
+      [tr]
+        [th]big[/th]
+      [/tr]
+    [/table]
+    [invalid class="extra"]whatever[/invalid]"""
+
+    out = self.bbcode.parse (inputText).html(allowClassAttr = True)
+    self.assertEquals (out, '''check this out
+
+    <h1 class="circle">heading</h1>
+
+    <p style="color: blue; font-size: 14pt; font-family: verdana, Times New Roman;">This is <b> bold </b> and this <i>italic</i> and this is <span style="color: red;">red</span> and this is <span style="color: red;">also red</span>.
+    </p>
+
+    fix <b><i>bold <span style="font-family: verdana;"><span style="font-size: 12pt;">and</span></span> italic</i></b>
+    <img src="http://www.codigomanso.com/b.jpg" />
+    <a href="http://www.codigomanso.com/">http://www.codigomanso.com/</a>
+    <a href="http://www.codigomanso.com/">Codigo Manso</a>
+    <a href="http://www.codigomanso.com" title="Codigo Manso Blog">Codigo Manso</a>
+
+    <ul>
+     <li>item 1</li>
+     <li>item 2</li>
+    </ul>
+
+    <ol type="1">
+     <li>item 1
+     </li><li>item 2
+    </li></ol>
+
+    <table class="big">
+      <tr>
+        <th>big</th>
+      </tr>
+    </table>
+    whatever''')
+
+
+  def testBBCodeDumper (self):
+    self.assertEquals (
+      self.bbcode.parse ('[b]bold[/b]').bbcode(),
+      '[b]bold[/b]'
+    )
+
+    self.assertEquals (
+      self.bbcode.parse ('[color=red]text in red[/color]').bbcode(),
+      '[color=red]text in red[/color]'
+    )
+    self.assertEquals (
+      self.bbcode.parse ('[p][color=red]text in red').bbcode(),
+      '[p][color=red]text in red[/color][/p]'
+    )
+
+    self.assertEquals (
+      self.bbcode.parse ('This [b][i]code[/b] will be fixed[/invalid]').bbcode(),
+      'This [b][i]code[/i][/b] will be fixed'
+    )
+
+    self.assertEquals (
+      self.bbcode.parse ('\[[url]http://www.codigomanso.com/en[/url]\]').bbcode(),
+      "\[[url]http://www.codigomanso.com/en[/url]\]"
+    )
+
+  def performanceTest(self):
+    '''
+    This test checks the performance of parse and html operations
+
+    To run this test type:
+      > python test.py BBCodeTests.performanceTest
+    '''
+    inputText = """check this out
+
+    [h1 class=circle]heading[/h1]
+
+    [p size=14 color=blue font="verdana, Times New Roman"]This is [b] bold [/b] and this [i]italic[/i] and this is [color=red]red[/color] and this is [color="red"]also red[/color].
+    [/p]
+
+    fix [b][i]bold [font=verdana][size=12]and[/size][/font] italic[/b]
+    [img]http://www.codigomanso.com/b.jpg[/img]
+    [url]http://www.codigomanso.com/[/url]
+    [url=http://www.codigomanso.com/]Codigo Manso[/url]
+    [uRl link=http://www.codigomanso.com title="Codigo Manso Blog"]Codigo Manso[/url]
+
+    [ul]
+     [Li]item 1[/Li]
+     [li]item 2[/LI]
+    [/UL]
+
+    [list=1 ]
+     [*]item 1
+     [*]item 2
+    [/list]
+
+    [table class="big"]
+      [tr]
+        [th]big[/th]
+      [/tr]
+    [/table]
+    [invalid class="extra"]whatever[/invalid]"""
+
+    import time
+    start = time.time()
+
+    for i in range(0, 12):
+      inputText += inputText
+
+    print "len(inputText) = %.2f MB  (took %.2f seconds)" % (len(inputText)/(1024.0*1024.0), time.time() - start)
+    
+    bbcode = bbcodeparser()
+    start = time.time()
+    bbcode.parse (inputText)
+    total = (time.time() - start)
+    print "time (bbcode.parse()) = %f" % total
+    print "  >> %.2f chars/second" % (len(inputText) / total)
+
+    start = time.time()
+    bbcode.html(doDeepCopy = False)
+    total = (time.time() - start)
+    print "time (bbcode.html()) = %f" % total
+    print "  >> %.2f chars/second" % (len(inputText) / total)
+    return
+    
+  def testCodeBuilder (self):
+    bbcode = bbcodebuilder ()
+    self.assertEquals (bbcode.b ('bold'), u'[b]bold[/b]')
+    self.assertEquals (bbcode.color ('this goes in red', 'red'), u'[color=red]this goes in red[/color]')
+    self.assertEquals (bbcode.url ('Google', 'http://www.google.com'), u'[url=http://www.google.com]Google[/url]')
+    self.assertEquals (bbcode.alist('item 1', 'item 2'), u"[list=a]\n  [*]item 1\n  [*]item 2\n[/list]")
+    return
+
+if __name__ == '__main__':
+    unittest.main()
+
+
+
@@ -21,10 +21,6 @@ import ConfigParser
 # inherit from Configurable.  The config file(s) uses ini format:
 # [sections] with key:value settings.
 #
-# writer does [defaults], [www.whofic.com], [epub], [www.whofic.com:epub], [overrides]
-#
-# Until a write is created, the adapter only has [defaults], [www.whofic.com], [overrides]
-#
 # [defaults]
 # titlepage_entries: category,genre, status
 # [www.whofic.com]
@@ -36,28 +32,67 @@ import ConfigParser
 # [overrides]
 # titlepage_entries: category

+class Configuration(ConfigParser.SafeConfigParser):

-class Configurable(object):
-
-    def __init__(self, config):
-        self.config = config
-        self.sectionslist = ['defaults']
-
-    def setSectionOrder(self,site,fileform=None):
+    def __init__(self, site, fileform):
+        ConfigParser.SafeConfigParser.__init__(self)
        self.sectionslist = ['defaults']
        self.addConfigSection(site)
        if fileform:
            self.addConfigSection(fileform)
            self.addConfigSection(site+":"+fileform)
        self.addConfigSection("overrides")
-
+        
+        self.validEntries = [
+            'category',
+            'genre',
+            'language',
+            'characters',
+            'ships',
+            'series',
+            'status',
+            'datePublished',
+            'dateUpdated',
+            'dateCreated',
+            'rating',
+            'warnings',
+            'numChapters',
+            'numWords',
+            'site',
+            'storyId',
+            'authorId',
+            'extratags',
+            'title',
+            'storyUrl',
+            'description',
+            'author',
+            'authorUrl',
+            'formatname',
+            'formatext',
+            'siteabbrev',
+            'version',
+            # internal stuff.
+            'langcode',
+            'output_css',
+            'authorHTML'
+            ]
+        
    def addConfigSection(self,section):
        self.sectionslist.insert(0,section)

+    def isValidMetaEntry(self, key):
+        return key in self.getValidMetaList()
+
+    def getValidMetaList(self):
+        vl = []
+        vl.extend(self.validEntries)
+        vl.extend(self.getConfigList("extra_valid_entries"))
+        return vl
+
    def hasConfig(self, key):
        for section in self.sectionslist:
            try:
-                self.config.get(section,key)
+                self.get(section,key)
                #print("found %s in section [%s]"%(key,section))
                return True
            except:
@@ -65,11 +100,11 @@ class Configurable(object):

        return False
        
-    def getConfig(self, key):
-        val = ""
+    def getConfig(self, key, default=""):
+        val = default
        for section in self.sectionslist:
            try:
-                val = self.config.get(section,key)
+                val = self.get(section,key)
                if val and val.lower() == "false":
                    val = False
                #print "getConfig(%s)=[%s]%s" % (key,section,val)
@@ -84,5 +119,26 @@ class Configurable(object):
        vlist = self.getConfig(key).split(',')
        vlist = [ v.strip() for v in vlist ]
        #print "vlist("+key+"):"+str(vlist)
-        return vlist
+        return vlist        
+
+# extended by adapter, writer and story for ease of calling configuration.
+class Configurable(object):
+
+    def __init__(self, configuration):
+        self.configuration = configuration
+
+    def isValidMetaEntry(self, key):
+        return self.configuration.isValidMetaEntry(key)
+
+    def getValidMetaList(self):
+        return self.configuration.getValidMetaList()
+    
+    def hasConfig(self, key):
+        return self.configuration.hasConfig(key)
+        
+    def getConfig(self, key, default=""):
+        return self.configuration.getConfig(key)
+
+    def getConfigList(self, key):
+        return self.configuration.getConfigList(key)
        
@@ -18,19 +18,19 @@
 import re
 import urlparse
 import urllib2 as u2
-import ConfigParser

 from BeautifulSoup import BeautifulSoup 
 from gziphttp import GZipProcessor

 import adapters
+from configurable import Configuration

-def get_urls_from_page(url,config=None):
+def get_urls_from_page(url,configuration=None):

    normalized = set() # normalized url
    retlist = [] # orig urls.
-    if not config:
-        config = ConfigParser.SafeConfigParser()
+    if not configuration:
+        configuration = Configuration("test1.com","EPUB")

    data = None
    
@@ -39,7 +39,7 @@ def get_urls_from_page(url,config=None):
    # them, AO3 doesn't even show them if not logged in.  Only works
    # with saved user/pass--not going to prompt for list.
    if 'archiveofourown.org' in url:
-        ao3adapter = adapters.getAdapter(config,"http://www.archiveofourown.org/works/0","EPUB")
+        ao3adapter = adapters.getAdapter(configuration,"http://www.archiveofourown.org/works/0","EPUB")
        if ao3adapter.getConfig("username"):
            if ao3adapter.getConfig("is_adult"):
                addurl = "?view_adult=true"
@@ -72,7 +72,7 @@ def get_urls_from_page(url,config=None):
                    
            try:
                href = href.replace('&index=1','')
-                adapter = adapters.getAdapter(config,href,"EPUB")
+                adapter = adapters.getAdapter(configuration,href,"EPUB")
                if adapter.story.getMetadata('storyUrl') not in normalized:
                    normalized.add(adapter.story.getMetadata('storyUrl'))
                    retlist.append(href)
@@ -22,6 +22,7 @@ from math import floor

 import exceptions
 from htmlcleanup import conditionalRemoveEntities, removeAllEntities
+from configurable import Configurable

 # Create convert_image method depending on which graphics lib we can
 # load.  Preferred: calibre, PIL, none
@@ -186,9 +187,10 @@ langs = {
    "Devanagari":"hi",
    }

-class Story:
+class Story(Configurable):
    
-    def __init__(self):
+    def __init__(self, configuration):
+        Configurable.__init__(self, configuration)
        try:
            self.metadata = {'version':os.environ['CURRENT_VERSION_ID']}
        except:
@@ -197,35 +199,50 @@ class Story:
        self.chapters = [] # chapters will be tuples of (title,html)
        self.imgurls = []
        self.imgtuples = []
-        self.listables = {} # some items (extratags, category, warnings & genres) are also kept as lists.
+        
        self.cover=None # *href* of new cover image--need to create html.
        self.oldcover=None # (oldcoverhtmlhref,oldcoverhtmltype,oldcoverhtmldata,oldcoverimghref,oldcoverimgtype,oldcoverimgdata)
        self.calibrebookmark=None # cheesy way to carry calibre bookmark file forward across update.
        self.logfile=None # cheesy way to carry log file forward across update.

-    def setMetadata(self, key, value):
+        self.setReplace(self.getConfig('replace_metadata'))
+        
+    def setMetadata(self, key, value, condremoveentities=True):
        ## still keeps &lt; &lt; and &amp;
-        self.metadata[key]=conditionalRemoveEntities(value)
+        if condremoveentities:
+            self.metadata[key]=conditionalRemoveEntities(value)
+        else:
+            self.metadata[key]=value
        if key == "language":
            try:
                self.metadata['langcode'] = langs[self.metadata[key]]
            except:
                self.metadata['langcode'] = 'en'
+        if key == 'dateUpdated':
+            # Last Update tags for Bill.
+            self.addToList('lastupdate',value.strftime("Last Update Year/Month: %Y/%m"))
+            self.addToList('lastupdate',value.strftime("Last Update: %Y/%m/%d"))
+

    def getMetadataRaw(self,key):
-        if self.metadata.has_key(key):
+        if self.isValidMetaEntry(key) and self.metadata.has_key(key):
            return self.metadata[key]

    def doReplacments(self,value):
        for (p,v) in self.replacements:
-            if (isinstance(value,str) or isinstance(value,unicode)) and re.match(p,value):
+            if (isinstance(value,basestring)) and re.match(p,value):
                value = re.sub(p,v,value)                
        return value
        
-    def getMetadata(self, key, removeallentities=False, doreplacements=True):
+    def getMetadata(self, key,
+                    removeallentities=False,
+                    doreplacements=True):
        value = None
+        if not self.isValidMetaEntry(key):
+            return value
+
        if self.isList(key):
-            value = ', '.join(self.getList(key))
+            value = u', '.join(self.getList(key, removeallentities, doreplacements=True))
        elif self.metadata.has_key(key):
            value = self.metadata[key]
            if value:
@@ -233,10 +250,8 @@ class Story:
                    value = commaGroups(value)
                if key == "numChapters":
                    value = commaGroups("%d"%value)
-                if key == "dateCreated":
-                    value = value.strftime("%Y-%m-%d %H:%M:%S")
-                if key == "datePublished" or key == "dateUpdated":
-                 value = value.strftime("%Y-%m-%d")
+                if key in ("dateCreated","datePublished","dateUpdated"):
+                    value = value.strftime(self.getConfig(key+"_format","%Y-%m-%d"))

        if doreplacements:
            value=self.doReplacments(value)
@@ -244,19 +259,22 @@ class Story:
            return removeAllEntities(value)
        else:
            return value
-
-    def getAllMetadata(self, removeallentities=False, doreplacements=True):
+        
+    def getAllMetadata(self,
+                       removeallentities=False,
+                       doreplacements=True,
+                       keeplists=False):
        '''
-        All single value *and* list value metadata as strings.
+        All single value *and* list value metadata as strings (unless keeplists=True, then keep lists).
        '''
        allmetadata = {}
        
        # special handling for authors/authorUrls
        authlinkhtml="<a class='authorlink' href='%s'>%s</a>"
-        if 'author' in self.listables.keys(): # more than one author, assume multiple authorUrl too.
+        if self.isList('author'): # more than one author, assume multiple authorUrl too.
            htmllist=[]
-            for i, v in enumerate(self.listables['author']):
-                aurl = self.listables['authorUrl'][i]                    
+            for i, v in enumerate(self.getList('author')):
+                aurl = self.getList('authorUrl')[i]                    
                auth = v
                # make sure doreplacements & removeallentities are honored.
                if doreplacements:
@@ -271,11 +289,12 @@ class Story:
        else:
           self.setMetadata('authorHTML',authlinkhtml%(self.getMetadata('authorUrl', removeallentities, doreplacements),
                                                       self.getMetadata('author', removeallentities, doreplacements)))
-            
-        for k in self.metadata.keys():
-            allmetadata[k] = self.getMetadata(k, removeallentities, doreplacements)
-        for l in self.listables.keys():
-            allmetadata[l] = self.getMetadata(l, removeallentities, doreplacements)
+
+        for k in self.getValidMetaList():
+            if self.isList(k) and keeplists:
+                allmetadata[k] = self.getList(k, removeallentities, doreplacements)
+            else:
+                allmetadata[k] = self.getMetadata(k, removeallentities, doreplacements)
                
        return allmetadata

@@ -288,43 +307,84 @@ class Story:
        if value==None:
            return
        value = conditionalRemoveEntities(value)
-        if not self.isList(listname):
-            self.listables[listname]=[]
+        if not self.isList(listname) or not listname in self.metadata:
+            # Calling addToList to a non-list meta will overwrite it.
+            self.metadata[listname]=[]
        # prevent duplicates.
-        if not value in self.listables[listname]:
-            self.listables[listname].append(value)
+        if not value in self.metadata[listname]:
+            self.metadata[listname].append(value)

-    def getList(self,listname, removeallentities=False, doreplacements=True):
+    def isList(self,listname):
+        'Everything set with an include_in_* is considered a list.'
+        return self.hasConfig("include_in_"+listname) or \
+            ( self.isValidMetaEntry(listname) and self.metadata.has_key(listname) \
+                  and isinstance(self.metadata[listname],list) )
+    
+    def getList(self,listname,
+                removeallentities=False,
+                doreplacements=True,
+                doincludein=True):
        retlist = []
-        if not self.isList(listname):
-            retlist = [self.getMetadata(listname,removeallentities=removeallentities)]
+        if not self.isValidMetaEntry(listname):
+            return retlist
+        
+        # doincludein prevents recursion of include_in_'s
+        if doincludein and self.hasConfig("include_in_"+listname):
+            for k in self.getConfigList("include_in_"+listname):
+                retlist.extend(self.getList(k,removeallentities,doreplacements,doincludein=False))
        else:
-            retlist = self.listables[listname]
+        
+            if not self.isList(listname):
+                retlist = [self.getMetadata(listname,removeallentities, doreplacements)]
+            else:
+                retlist = self.getMetadataRaw(listname)

-        if doreplacements:
-            retlist = filter( lambda x : x!=None and x!='' ,
-                              map(self.doReplacments,retlist) )
-        if removeallentities:
-            retlist = filter( lambda x : x!=None and x!='' ,
-                              map(removeAllEntities,retlist) )
+            if doreplacements:
+                retlist = filter( lambda x : x!=None and x!='' ,
+                                  map(self.doReplacments,retlist) )
+            if removeallentities:
+                retlist = filter( lambda x : x!=None and x!='' ,
+                                  map(removeAllEntities,retlist) )

        return retlist

-    def getLists(self):
-        lsts = {}
-        for ln in self.listables.keys():
-            lsts[ln] = self.getList(ln)
-        return lsts
+    def getSubjectTags(self, removeallentities=False):
+        # set to avoid duplicates subject tags.
+        subjectset = set()
+        
+        tags_list = self.getConfigList("include_subject_tags") + self.getConfigList("extra_subject_tags")
+            
+        # metadata all go into dc:subject tags, but only if they are configured.
+        for (name,value) in self.getAllMetadata(removeallentities=removeallentities,keeplists=True).iteritems():
+            if name in tags_list:
+                if isinstance(value,list):
+                    for tag in value:
+                        subjectset.add(tag)
+                else:
+                    subjectset.add(value)

-    def isList(self,listname):
-        return self.listables.has_key(listname)
-    
+        if None in subjectset:
+            subjectset.remove(None)
+            
+        return list(subjectset)
+            
    def addChapter(self, title, html):
+        if self.getConfig('strip_chapter_numbers') and \
+                self.getConfig('chapter_title_strip_pattern'):
+            title = re.sub(self.getConfig('chapter_title_strip_pattern'),"",title)
        self.chapters.append( (title,html) )

    def getChapters(self):
        "Chapters will be tuples of (title,html)"
-        return self.chapters
+        retval = []
+        if self.getConfig('add_chapter_numbers') and \
+                self.getConfig('chapter_title_add_pattern'):
+            for index, (title,html) in enumerate(self.chapters):
+                retval.append( (string.Template(self.getConfig('chapter_title_add_pattern')).substitute({'index':index+1,'title':title}),html) ) 
+        else:
+            retval = self.chapters
+            
+        return retval

    def formatFileName(self,template,allowunsafefilename=True):
        values = origvalues = self.getAllMetadata()
@@ -342,11 +402,11 @@ class Story:

    # pass fetch in from adapter in case we need the cookies collected
    # as well as it's a base_story class method.
-    def addImgUrl(self,configurable,parenturl,url,fetch,cover=False,coverexclusion=None):
+    def addImgUrl(self,parenturl,url,fetch,cover=False,coverexclusion=None):

        # otherwise it saves the image in the epub even though it
        # isn't used anywhere.
-        if cover and configurable.getConfig('never_make_cover'):
+        if cover and self.getConfig('never_make_cover'):
            return
        
        url = url.strip() # ran across an image with a space in the
@@ -384,20 +444,20 @@ class Story:
        if imgurl not in self.imgurls:
            parsedUrl = urlparse.urlparse(imgurl)
            try:
-                sizes = [ int(x) for x in configurable.getConfigList('image_max_size') ]
+                sizes = [ int(x) for x in self.getConfigList('image_max_size') ]
            except Exception, e:
-                raise exceptions.FailedToDownload("Failed to parse image_max_size from personal.ini:%s\nException: %s"%(configurable.getConfigList('image_max_size'),e))
+                raise exceptions.FailedToDownload("Failed to parse image_max_size from personal.ini:%s\nException: %s"%(self.getConfigList('image_max_size'),e))
            try:
                (data,ext,mime) = convert_image(imgurl,
                                                fetch(imgurl),
                                                sizes,
-                                                configurable.getConfig('grayscale_images'))
+                                                self.getConfig('grayscale_images'))
            except Exception, e:
                print("Failed to load or convert image, skipping:\n%s\nException: %s"%(imgurl,e))
                return "failedtoload"
            
            # explicit cover, make the first image.
-            if cover and not configurable.getConfig('never_make_cover'):
+            if cover and not self.getConfig('never_make_cover'):
                if len(self.imgtuples) > 0 and 'cover' in self.imgtuples[0]['newsrc']:
                    # remove existing cover, if there is one.
                    del self.imgurls[0]
@@ -414,8 +474,8 @@ class Story:
                #          NOT never_make_cover AND
                #          either no coverexclusion OR coverexclusion doesn't match
                if self.cover == None and \
-                        configurable.getConfig('make_firstimage_cover') and \
-                        not configurable.getConfig('never_make_cover') and \
+                        self.getConfig('make_firstimage_cover') and \
+                        not self.getConfig('never_make_cover') and \
                        (not coverexclusion or not re.search(coverexclusion,imgurl)):
                    newsrc = "images/cover.%s"%ext
                    self.cover=newsrc
@@ -444,7 +504,7 @@ class Story:
        return retlist
    
    def __str__(self):
-        return "Metadata: " +str(self.metadata) + "\nListables: " +str(self.listables) #+ "\nChapters: "+str(self.chapters)
+        return "Metadata: " +str(self.metadata) 

    def setReplace(self,replace):
        for line in replace.splitlines():
@@ -36,43 +36,12 @@ class BaseStoryWriter(Configurable):
    def getFormatExt():
        return '.bse'

-    def __init__(self, config, adapter):
-        Configurable.__init__(self, config)
-        self.setSectionOrder(adapter.getConfigSection(),self.getFormatName())
+    def __init__(self, configuration, adapter):
+        Configurable.__init__(self, configuration)
        
        self.adapter = adapter
        self.story = adapter.getStoryMetadataOnly() # only cache the metadata initially.
        
-        self.story.setReplace(self.getConfig('replace_metadata'))
-        
-        self.validEntries = [
-            'category',
-            'genre',
-            'language',
-            'characters',
-            'series',
-            'status',
-            'datePublished',
-            'dateUpdated',
-            'dateCreated',
-            'rating',
-            'warnings',
-            'numChapters',
-            'numWords',
-            'site',
-            'storyId',
-            'authorId',
-            'extratags',
-            'title',
-            'storyUrl',
-            'description',
-            'author',
-            'authorUrl',
-            'formatname',
-            'formatext',
-            'siteabbrev',
-            'version']
-
        # fall back labels.
        self.titleLabels = {
            'category':'Category',
@@ -81,6 +50,7 @@ class BaseStoryWriter(Configurable):
            'status':'Status',
            'series':'Series',
            'characters':'Characters',
+            'ships':'Relationships',
            'datePublished':'Published',
            'dateUpdated':'Updated',
            'dateCreated':'Packaged',
@@ -104,12 +74,19 @@ class BaseStoryWriter(Configurable):
            }
        self.story.setMetadata('formatname',self.getFormatName())
        self.story.setMetadata('formatext',self.getFormatExt())
-        
-        for tag in self.getConfigList("extratags"):
-            self.story.addToList("extratags",tag)

-    def getMetadata(self,key):
-        return stripHTML(self.story.getMetadata(key))
+        ## Look for config parameter, split and add each to metadata field.
+        for (config,metadata) in [("extratags","extratags"),
+                                  ("extracategories","category"),
+                                  ("extragenres","genre"),
+                                  ("extracharacters","characters"),
+                                  ("extraships","ships"),
+                                  ("extrawarnings","warnings")]:
+            for val in self.getConfigList(config):
+                self.story.addToList(metadata,val)
+
+    def getMetadata(self,key, removeallentities=False):
+        return stripHTML(self.story.getMetadata(key, removeallentities))

    def getOutputFileName(self):
        if self.getConfig('zip_output'):
@@ -139,11 +116,11 @@ class BaseStoryWriter(Configurable):
            if WIDE_ENTRY==None:
                WIDE_ENTRY=ENTRY

-            titleEntriesList = self.getConfigList("titlepage_entries")
+            titleEntriesList = self.getConfigList("titlepage_entries") + self.getConfigList("extra_titlepage_entries")
            wideTitleEntriesList = self.getConfigList("wide_titlepage_entries")

            for entry in titleEntriesList:
-                if entry in self.validEntries:
+                if self.isValidMetaEntry(entry):
                    if self.story.getMetadata(entry):
                        if entry in wideTitleEntriesList:
                            TEMPLATE=WIDE_ENTRY
@@ -152,9 +129,12 @@ class BaseStoryWriter(Configurable):
                            
                        if self.hasConfig(entry+"_label"):
                            label=self.getConfig(entry+"_label")
-                        else:
-                            print("Using fallback label for %s_label"%entry)
+                        elif entry in self.titleLabels:
+                            logging.debug("Using fallback label for %s_label"%entry)
                            label=self.titleLabels[entry]
+                        else:
+                            label="%s"%entry.title()
+                            logging.debug("No known label for %s, fallback to '%s'"%(entry,label))

                        # If the label for the title entry is empty, use the
                        # 'no title' option if there is one.
@@ -193,9 +173,11 @@ class BaseStoryWriter(Configurable):

        # minor cheat, tucking css into metadata.
        if self.getConfig("output_css"):
-            self.story.metadata["output_css"] = self.getConfig("output_css")
+            self.story.setMetadata("output_css",
+                                   self.getConfig("output_css"),
+                                   condremoveentities=False)
        else:
-            self.story.metadata["output_css"] = ''
+            self.story.setMetadata("output_css",'')
            
        if not outstream:
            close=True
@@ -252,29 +234,6 @@ class BaseStoryWriter(Configurable):
        if close:
            outstream.close()

-    def getTags(self):
-        # set to avoid duplicates subject tags.
-        subjectset = set()
-        
-        if self.story.getMetadataRaw('dateUpdated'):
-            # Last Update tags for Bill.
-            self.story.addToList('lastupdate',self.story.getMetadataRaw('dateUpdated').strftime("Last Update Year/Month: %Y/%m"))
-            self.story.addToList('lastupdate',self.story.getMetadataRaw('dateUpdated').strftime("Last Update: %Y/%m/%d"))
-        
-        for entry in self.validEntries:
-            if entry in self.getConfigList("include_subject_tags") and \
-                    entry not in self.story.getLists() and \
-                    self.story.getMetadata(entry):
-                subjectset.add(self.getMetadata(entry))
-                
-        # listables all go into dc:subject tags, but only if they are configured.
-        for (name,lst) in self.story.getLists().iteritems():
-            if name in self.getConfigList("include_subject_tags"):
-                for tag in lst:
-                    subjectset.add(tag)
-
-        return list(subjectset)
-            
    def writeStoryImpl(self, out):
        "Must be overriden by sub classes."
        pass
@@ -193,31 +193,33 @@ ${value}<br />
        Switch rindex to index to search from top instead of bottom.
        """
        values = {}
-        for entry in self.getConfigList("logpage_entries"):
-            if entry in self.validEntries:
-                try:
-                    # <span id="dateUpdated">1975-04-15</span>
-                    span = '<span id="%s">'%entry
-                    idx = logfile.rindex(span)+len(span)
-                    values[entry] = logfile[idx:logfile.index('</span>',idx)]
-                except Exception, e:
-                    #print("e:%s"%e)
-                    pass
+        for entry in self.getConfigList("logpage_entries") + self.getConfigList("extra_logpage_entries"):
+            try:
+                # <span id="dateUpdated">1975-04-15</span>
+                span = '<span id="%s">'%entry
+                idx = logfile.rindex(span)+len(span)
+                values[entry] = logfile[idx:logfile.index('</span>',idx)]
+            except Exception, e:
+                #print("e:%s"%e)
+                pass

        return values
        
    def _makeLogEntry(self, oldvalues={}):
        retval = "<p class='log_entry'>"

-        for entry in self.getConfigList("logpage_entries"):
-            if entry in self.validEntries:
+        for entry in self.getConfigList("logpage_entries") + self.getConfigList("extra_logpage_entries"):
+            if self.isValidMetaEntry(entry):
                val = self.story.getMetadata(entry)
                if val and ( entry not in oldvalues or val != oldvalues[entry] ):
                    if self.hasConfig(entry+"_label"):
                        label=self.getConfig(entry+"_label")
-                    else:
-                        print("Using fallback label for %s_label"%entry)
+                    elif entry in self.titleLabels:
+                        logging.debug("Using fallback label for %s_label"%entry)
                        label=self.titleLabels[entry]
+                    else:
+                        label="%s"%entry.title()
+                        logging.debug("No known label for %s, fallback to '%s'"%(entry,label))

                    retval = retval + self.EPUB_LOG_ENTRY.substitute({'id':entry,
                                                                      'label':label,
@@ -342,7 +344,7 @@ ${value}<br />
            metadata.appendChild(newTag(contentdom,"dc:description",text=
                                        self.getMetadata('description')))

-        for subject in self.getTags():
+        for subject in self.story.getSubjectTags():
            metadata.appendChild(newTag(contentdom,"dc:subject",text=subject))

                    
@@ -54,6 +54,27 @@
 	      much easier. </p>
 	  </div>
 	  <!-- put announcements here, h3 is a good title size. -->
+	  <h3>New Sites</h3>
+	  <p>
+	    New sites bloodties-fans.com and thehookupzone.net, thanks Marillapm!
+	  </p>
+	  <h3>New Features and Fixes</h3>
+	  <p>
+	    Added extra metadata feature for certain sites.  This one
+	    contains some metadata features that have been requested,
+	    plus quite a bit of internal refactoring to allow them.
+	    This is primarily intended for calibre plugin users, so
+	    it's <a href="">documented in the plugin forum</a>.
+	    
+	    <br /> Also added cover images for dramione.org, thanks
+	    ilovejedd.
+	  </p>
+	  <p>
+	    Fixes for: squidge.org/peja Rating  including '[' when
+	    there's no stars; loosen url check on AO3 to allow more
+	    copy/paste URLs; fix author on castlefans.org due to
+	    site/skin changes.
+	  </p>
 	  <p>
 	    Questions?  Check out our 
 	    <a href="http://code.google.com/p/fanficdownloader/wiki/FanFictionDownloaderFAQs">FAQs</a>.
@@ -62,7 +83,7 @@
 	    If you have any problems with this application, please
 	    report them in
 	    the <a href="http://groups.google.com/group/fanfic-downloader">FanFictionDownLoader Google Group</a>.  The
-	    <a href="http://4-4-21.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
+	    <a href="http://4-4-25.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
 	  </p>
 	  <div id='error'>
 	    {{ error_message }}
@@ -499,6 +520,46 @@
 	    Use the URL of the story's chapter list, such as
 	    <br /><a href="http://nha.magical-worlds.us/viewstory.php?sid=1234">http://nha.magical-worlds.us/viewstory.php?sid=1234</a>
 	  </dd>
+	  <dt>www.dracoandginny.com</dt>
+	  <dd>
+	    Use the URL of the story's chapter list, such as
+	    <br /><a href="http://www.dracoandginny.com/viewstory.php?sid=1234">http://www.dracoandginny.com/viewstory.php?sid=1234</a>
+	  </dd>
+	  <dt>www.scarvesandcoffee.net</dt>
+	  <dd>
+	    Use the URL of the story's chapter list, such as
+	    <br /><a href="http://www.scarvesandcoffee.net/viewstory.php?sid=1234">http://www.scarvesandcoffee.net/viewstory.php?sid=1234</a>
+	  </dd>
+	  <dt>www.thepetulantpoetess.com</dt>
+	  <dd>
+	    Use the URL of the story's chapter list, such as
+	    <br /><a href="http://www.thepetulantpoetess.com/viewstory.php?sid=1234">http://www.thepetulantpoetess.com/viewstory.php?sid=1234</a>
+	  </dd>
+	  <dt>www.wolverineandrogue.com</dt>
+	  <dd>
+	    Use the URL of the story's chapter list, such as
+	    <br /><a href="http://www.wolverineandrogue.com/wrfa/viewstory.php?sid=1234">http://www.wolverineandrogue.com/wrfa/viewstory.php?sid=1234</a>
+	  </dd>
+	  <dt>www.sinful-desire.org</dt>
+	  <dd>
+	    Use the URL of the story's chapter list, such as
+	    <br /><a href="http://www.sinful-desire.org/archive/viewstory.php?sid=1234">http://www.sinful-desire.org/archive/viewstory.php?sid=1234</a>
+	  </dd>
+	  <dt>merlinfic.dtwins.co.uk</dt>
+	  <dd>
+	    Use the URL of the story's chapter list, such as
+	    <br /><a href="http://merlinfic.dtwins.co.uk/viewstory.php?sid=1234">http://merlinfic.dtwins.co.uk/viewstory.php?sid=1234</a>
+	  </dd>
+	  <dt>bloodties-fans.com</dt>
+	  <dd>
+	    Use the URL of the story's chapter list, such as
+	    <br /><a href="http://bloodties-fans.com/fiction/viewstory.php?sid=1234">http://bloodties-fans.com/fiction/viewstory.php?sid=1234</a>
+	  </dd>
+	  <dt>thehookupzone.net</dt>
+	  <dd>
+	    Use the URL of the story's chapter list, such as
+	    <br /><a href="http://thehookupzone.net/CriminalMinds/viewstory.php?sid=1234">http://thehookupzone.net/CriminalMinds/viewstory.php?sid=1234</a>
+	  </dd>
 	</dl>
 	<p>
 	  A few additional things to know, which will make your life substantially easier:
@@ -29,7 +29,6 @@ import datetime

 import traceback
 from StringIO import StringIO
-import ConfigParser

 ## Just to shut up the appengine warning about "You are using the
 ## default Django version (0.96). The default Django version will
@@ -57,22 +56,25 @@ from google.appengine.runtime import DeadlineExceededError
 from ffstorage import *

 from fanficdownloader import adapters, writers, exceptions
+from fanficdownloader.configurable import Configuration

 class UserConfigServer(webapp2.RequestHandler):
-    def getUserConfig(self,user):
-        config = ConfigParser.SafeConfigParser()
+    
+    def getUserConfig(self,user,url,fileformat):

+        configuration = Configuration(adapters.getConfigSectionFor(url),fileformat)
+        
        logging.debug('reading defaults.ini config file')
-        config.read('defaults.ini')
+        configuration.read('defaults.ini')

        ## Pull user's config record.
        l = UserConfig.all().filter('user =', user).fetch(1)
        if l and l[0].config:
            uconfig=l[0]
            #logging.debug('reading config from UserConfig(%s)'%uconfig.config)
-            config.readfp(StringIO(uconfig.config))
+            configuration.readfp(StringIO(uconfig.config))

-        return config
+        return configuration

 class MainHandler(webapp2.RequestHandler):
    def get(self):
@@ -137,7 +139,8 @@ class EditConfigServer(UserConfigServer):
            uconfig.config = self.request.get('config').encode('utf8')[:10000] ## just in case.
            uconfig.put()
            try:
-                config = self.getUserConfig(user)
+                # just getting config for testing purposes.
+                configuration = self.getUserConfig(user,"test1.com","epub")
                self.redirect("/?error=configsaved")
            except Exception, e:
                logging.info("Saved Config Failed:%s"%e)
@@ -367,12 +370,12 @@ class FanfictionDownloader(UserConfigServer):
        adapter = None
        try:
            try:
-                config = self.getUserConfig(user)
+                configuration = self.getUserConfig(user,url,format)
            except Exception, e:
                self.redirect("/?error=custom&errtext=%s"%urlEscape("There's an error in your User Configuration: "+str(e)))
                return

-            adapter = adapters.getAdapter(config,url,format)
+            adapter = adapters.getAdapter(configuration,url)
            logging.info('Created an adaper: %s' % adapter)

            if len(login) > 1:
@@ -474,8 +477,8 @@ class FanfictionDownloaderTask(UserConfigServer):
        logging.info('Creating adapter...')

        try:
-            config = self.getUserConfig(user)
-            adapter = adapters.getAdapter(config,url,format)
+            configuration = self.getUserConfig(user,url,format)
+            adapter = adapters.getAdapter(configuration,url)

            logging.info('Created an adapter: %s' % adapter)

@@ -488,7 +491,7 @@ class FanfictionDownloaderTask(UserConfigServer):
            # adapter.getStoryMetadataOnly() only fetches enough to
            # get metadata.  writer.writeStory() will call
            # adapter.getStory(), too.
-            writer = writers.getWriter(format,config,adapter)
+            writer = writers.getWriter(format,configuration,adapter)
            download.name = writer.getOutputFileName()
            #logging.debug('output_filename:'+writer.getConfig('output_filename'))
            logging.debug('getOutputFileName:'+writer.getOutputFileName())
@@ -19,8 +19,7 @@
 ## overridden at several levels

 ## Some sites also require the user to confirm they are adult for
-## adult content.  In commandline version, this should go in your
-## personal.ini, not defaults.ini.
+## adult content.  Uncomment by removing '#' in front of is_adult.
 #is_adult:true

 ## All available titlepage_entries and the label used for them:
@@ -43,6 +42,7 @@ category_label:Category
 genre_label:Genre
 language_label:Language
 characters_label:Characters
+ships_label:Relationships
 series_label:Series
 ## Completed/In-Progress
 status_label:Status
@@ -67,15 +67,20 @@ authorId_label:Author ID
 ## show up in Calibre as tags.  Also carried into mobi when converted.
 extratags_label:Extra Tags
 ## The version of fanficdownloader
-##
 version_label:FFDL Version

+## Date formats used by FFDL.  Published and Update don't have time.
+## Note that ini format requires % to be escaped as %%.
+dateCreated_format:%%Y-%%m-%%d %%H:%%M:%%S
+datePublished_format:%%Y-%%m-%%d
+dateUpdated_format:%%Y-%%m-%%d
+
 ## items to include in the title page
 ## Empty metadata entries will *not* appear, even if in the list.
 ## You can include extra text or HTML that will be included as-is in
 ## the title page. Eg: titlepage_entries: ...,<br />,summary,<br />,...
 ## All current formats already include title and author.
-titlepage_entries: series,category,genre,language,characters,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,description
+titlepage_entries: series,category,genre,language,characters,ships,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,description

 ## Try to collect series name and number of this story in series.
 ## Some sites (ab)use 'series' for reading lists and personal
@@ -101,11 +106,20 @@ include_tocpage: true

 ## entries to make epub subjects and calibre tags
 ## lastupdate creates two tags: "Last Update Year/Month: %Y/%m" and "Last Update: %Y/%m/%d"
-include_subject_tags: extratags, genre, category, characters, status
+include_subject_tags: extratags, genre, category, characters, ships, status

 ## extra tags (comma separated) to include, primarily for epub.
 extratags: FanFiction

+## extra categories, genres, characters, ships and warnings can be
+## configured.  Used primarily for sites that are dedicated to a genre
+## or 'ship and so don't included it for every story.
+#extracategories:
+#extragenres:
+#extracharacters:
+#extraships:
+#extrawarnings:
+
 ## number of seconds to sleep between calls to the story site.  May by
 ## useful if pulling large numbers of stories or if the site is slow.
 #slow_down_sleep_time:0.5
@@ -143,6 +157,45 @@ extratags: FanFiction
 # ${category} => Buffy:? [tT]he Vampire Slayer => BuffyCover
 # ${category} => Star Trek => StarTrekCover

+## If set false, the summary will have all html stripped.
+## Both this and include_images must be true to get images in the
+## summary.
+keep_summary_html:true
+
+## Don't like the numbers at the start of chapter titles on some
+## sites?  You can use strip_chapter_numbers to strip them off.  Just
+## want to make them all look the same?  Strip them off, then add them
+## back on with add_chapter_numbers.  Don't like the way it strips
+## numbers or adds them back?  See chapter_title_strip_pattern and
+## chapter_title_add_pattern.
+strip_chapter_numbers:false
+add_chapter_numbers:false
+
+## (Two versions of chapter_title_strip_pattern are shown below.  You
+## should only have one uncommented.)
+## This version will remove the leading number from:
+## "1." => ""
+## "1. The Beginning" => "The Beginning"
+## "1: Start" => "Start"
+## "2, Chapter the second" => "Chapter the second"
+## etc
+chapter_title_strip_pattern:^[0-9]+[\.: -]+
+
+## This version will strip all of the above *plus* remove 'Chapter 1':
+## "Chapter 1" => ""
+## "1. Chapter 1" => ""
+## "1. Chapter 1, Bob's First Clue" => "Bob's First Clue"
+## "Chapter 2 - Pirates Place" => "Pirates Place"
+## etc
+#chapter_title_strip_pattern:^([0-9]+[\.: -]+)?(Chapter *[0-9]+[\.:, -]*)?
+
+## Uses a python template substitution.  The ${index} is the 'chapter'
+## number and ${title} is the chapter title, after applying
+## chapter_title_strip_pattern.  Those are the only variables available.
+## "The Beginning" => "1. The Beginning" 
+chapter_title_add_pattern:${index}. ${title}
+
+
 ## Each output format has a section that overrides [defaults]
 [html]

@@ -215,10 +268,12 @@ background_color: ffffff
 ## Allow customization of CSS.  Make sure to keep at least one space
 ## at the start of each line and to escape % to %%.  Also need
 ## background_color to be in the same section, if included in CSS.
+## 'adobe-text-layout: optimizeSpeed;' prevents hyphenation on newer Nooks
 output_css: 
 body { background-color: #%(background_color)s;
        text-align: justify;
-        margin: 2%%; }
+        margin: 2%%;
+	adobe-text-layout: optimizeSpeed; }
 pre { font-size: x-small; }
 sml { font-size: small; }
 h1 { text-align: center; }
@@ -245,11 +300,6 @@ output_css:
 ## stories.  Images will be converted to jpg for size if possible.
 #include_images:false

-## If not set, the summary will have all html stripped for safety.
-## Both this and include_images must be true to get images in the
-## summary.
-#keep_summary_html:false
-
 ## If set, the first image found will be made the cover image.  If
 ## keep_summary_html is true, any images in summary will be before any
 ## in chapters.
@@ -302,6 +352,11 @@ nook_img_fix:true
 ## URLs like: http://test1.com?sid=12345
 [test1.com]
 extratags: FanFiction,Testing
+extracategories:Fafner
+extragenres:Romance,Fluff
+extracharacters:Reginald Smythe-Smythe,Mokona
+extraships:Smythe-Smythe/Mokona
+extrawarnings:Extreme Bogosity

 ## If necessary, you can define [<site>:<format>] sections to
 ## customize the formats differently for the same site.  Overrides
@@ -328,6 +383,11 @@ extratags: FanFiction,Testing,HTML
 #is_adult:true

 [ashwinder.sycophanthex.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+extracharacters:Severus Snape,Hermione Granger
+extraships:Severus Snape/Hermione Granger
+
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -335,7 +395,26 @@ extratags: FanFiction,Testing,HTML
 #username:YourName
 #password:yourpassword

+[bloodties-fans.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Blood Ties
+
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
 [castlefans.org]
+## Site dedicated to these categories/characters/ships
+extracategories:Castle
+
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -359,6 +438,11 @@ extratags: FanFiction,Testing,HTML
 cover_exclusion_regexp:/images/.*?ribbon.gif

 [dark-solace.org]
+## Site dedicated to these categories/characters/ships
+extracategories:Buffy: The Vampire Slayer
+extracharacters:Buffy, Spike
+extraships:Spike/Buffy
+
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -367,6 +451,11 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 #password:yourpassword

 [dramione.org]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+extracharacters:Draco Malfoy,Hermione Granger
+extraships:Draco Malfoy/Hermione Granger
+
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -379,7 +468,59 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 ## personal.ini, not defaults.ini.
 #is_adult:true

+## Some adapters collect additional meta information beyond the
+## standard ones.  They need to be defined in extra_valid_entries to
+## tell the rest of the FFDL system about them.  They can be used in
+## include_subject_tags, titlepage_entries, extra_titlepage_entries,
+## logpage_entries, extra_logpage_entries, and include_in_* config
+## items.  You can also add additional entries here to build up
+## composite metadata entries.  dramione.org, for example, adds
+## 'cliches' and then defines as the composite of hermiones,dracos in
+## include_in_cliches.
+extra_valid_entries:themes,hermiones,dracos,timeline,cliches
+include_in_cliches:hermiones,dracos
+
+## For another example, you could, by uncommenting this line, include
+## themes in with genre metadata.
+#include_in_genre:genre, themes
+
+## You can give each new valid entry a specific label for use on
+## titlepage and logpage.  If not defined, it will simply be the 
+themes_label:Themes
+hermiones_label:Hermiones
+dracos_label:Dracos
+timeline_label:Timeline
+cliches_label:Character Cliches
+
+## extra_titlepage_entries (and extra_logpage_entries) *add* to
+## titlepage_entries (and logpage_entries) so you can add site
+## specific entries to titlepage/logpage without having to copy the
+## entire titlepage_entries line.  (But if you want them higher than
+## the end, you will need to copy titlepage_entries.)
+#extra_titlepage_entries: themes,timeline,cliches
+#extra_logpage_entries: themes,timeline,cliches
+#extra_subject_tags: themes,timeline,cliches
+
+## (Plugin Only) - You can also populate calibre custom columns with
+## the site specific metadata using custom_columns_settings (but only
+## if 'Allow custom_columns_settings from personal.ini' is checked in
+## the plugin GUI config.)  There are three parts, the entry name,
+## then the label of the calibre custom column, then (optionally) a
+## 'mode'.  'r' to Replace any existing values, 'a' to Add to existing
+## value (use with tag-like columns), and 'n' for setting on New books
+## only.
+## Make sure to keep at least one space at the start of each
+## line.
+
+#custom_columns_settings:
+# cliches=>#acolumn,r
+# themes=>#bcolumn,a
+# timeline=>#ccolumn,n
+
 [erosnsappho.sycophanthex.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
@@ -390,6 +531,9 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 cover_exclusion_regexp:/images/.*?ribbon.gif

 [fanfiction.mugglenet.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -403,6 +547,10 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 #is_adult:true

 [fanfiction.portkey.org]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+extraships:Harry Potter/Hermione Granger
+
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
@@ -425,6 +573,10 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 #password:yourpassword

 [grangerenchanted.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+extracharacters:Hermione Granger
+
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -438,26 +590,67 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 #is_adult:true

 [hlfiction.net]
+## Site dedicated to these categories/characters/ships
+extracategories:Highlander
+
+[ksarchive.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Star Trek
+extracharacters:Kirk,Spock
+extraships:Kirk/Spock

 [lumos.sycophanthex.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true

+[merlinfic.dtwins.co.uk]
+## Site dedicated to these categories/characters/ships
+extracategories:Merlin
+
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
 [national-library.net]
+## Site dedicated to these categories/characters/ships
+extracategories:West Wing

 [ncisfic.com]
+## Site dedicated to these categories/characters/ships
+extracategories:NCIS

 [nfacommunity.com]
+## Site dedicated to these categories/characters/ships
+extracategories:NCIS
+
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true

 [nha.magical-worlds.us]
+## Site dedicated to these categories/characters/ships
+extracategories:Buffy: The Vampire Slayer
+extracharacters:Willow

 [occlumency.sycophanthex.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+extracharacters:Severus Snape
+
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -466,6 +659,9 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 #password:yourpassword

 [onedirectionfanfiction.com]
+## Site dedicated to these categories/characters/ships
+extracategories:One Direction
+
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -479,18 +675,28 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 #is_adult:true

 [ponyfictionarchive.net]
+## Site dedicated to these categories/characters/ships
+extracategories:My Little Pony: Friendship is Magic
+
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true

 [pretendercentre.com]
+## Site dedicated to these categories/characters/ships
+extracategories:The Pretender
+
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true

 [samdean.archive.nu]
+## Site dedicated to these categories/characters/ships
+extracategories:Supernatural
+extracharacters:Sam,Dean
+extraships:Sam/Dean

 [sg1-heliopolis.com]
 ## Some sites do not require a login, but do require the user to
@@ -499,6 +705,8 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 #is_adult:true

 [stargate-atlantis.org]
+## Site dedicated to these categories/characters/ships
+extracategories:Stargate: Atlantis

 [svufiction.com]
 ## Some sites require login (or login for some rated stories) The
@@ -514,6 +722,26 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 #is_adult:true

 [thehexfiles.net]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+extracharacters:Draco Malfoy,Harry Potter
+extraships:Harry Potter/Draco Malfoy
+
+[thehookupzone.net]
+## Site dedicated to these categories/characters/ships
+extracategories:Criminal Minds
+
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true

 [themasque.net]
 ## Some sites require login (or login for some rated stories) The
@@ -529,6 +757,9 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 #is_adult:true

 [thequidditchpitch.org]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -542,14 +773,40 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 #is_adult:true

 [www.adastrafanfic.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Star Trek
+
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true

+[www.dracoandginny.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+extracharacters:Draco Malfoy,Ginny Weasley
+extraships:Draco Malfoy/Ginny Weasley
+
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
 [www.thealphagate.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Stargate: SG-1

 [www.checkmated.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -564,12 +821,20 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 #is_adult:true

 [www.dokuga.com]
+## Site dedicated to these categories/characters/ships
+extracategories:InuYasha
+extracharacters:Sesshoumaru,Kagome
+extraships:Sesshoumaru/Kagome

 [www.fanfiction.net]
 ## fanfiction.net's 'cover' images are really just tiny thumbnails.
-## Comment this out or change it to false to use them anyway.
+## Change this to false to use them anyway.
 never_make_cover: true

+## Extra metadata that this adapter knows about.  See [dramione.org]
+## for examples of how to use them.
+extra_valid_entries:reviews,favs,follows
+
 [www.fanfiktion.de]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -581,6 +846,9 @@ never_make_cover: true
 [www.ficbook.net]

 [www.fictionalley.org]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
@@ -599,6 +867,9 @@ extratags:
 #password:yourpassword

 [www.fimfiction.net]
+## Site dedicated to these categories/characters/ships
+extracategories:My Little Pony: Friendship is Magic
+
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
@@ -610,20 +881,33 @@ extratags:
 #fail_on_password: false

 [www.harrypotterfanfiction.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true

 [www.hpfandom.net]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true

 [www.hpfanficarchive.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter

 [www.ik-eternal.net]
+## Site dedicated to these categories/characters/ships
+extracategories:InuYasha
+extracharacters:InuYasha,Kagome
+extraships:InuYasha/Kagome
+
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -637,6 +921,9 @@ extratags:
 #is_adult:true

 [www.libraryofmoria.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Lord of the Rings
+
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
@@ -645,6 +932,9 @@ extratags:
 [www.mediaminer.org]

 [www.midnightwhispers.ca]
+## Site dedicated to these categories/characters/ships
+extracategories:Queer as Folk
+
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
@@ -655,8 +945,14 @@ extratags:
 cover_exclusion_regexp:/stories/999/images/.*?_trophy.png

 [www.ncisfiction.com]
+## Site dedicated to these categories/characters/ships
+extracategories:NCIS

 [www.phoenixsong.net]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+extraships:Harry Potter/Ginny Weasley
+
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -665,16 +961,43 @@ cover_exclusion_regexp:/stories/999/images/.*?_trophy.png
 #password:yourpassword

 [www.potionsandsnitches.net]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter

 [www.prisonbreakfic.net]
+## Site dedicated to these categories/characters/ships
+extracategories:Prison Break
+
+[www.scarvesandcoffee.net]
+## Site dedicated to these categories/characters/ships
+extracategories:Glee
+extracharacters:Kurt Hummel,Blaine Anderson
+
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+[www.sinful-desire.org]
+## Site dedicated to these categories/characters/ships
+extracategories:Supernatural
+
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true

 [www.siye.co.uk]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+extracharacters:Harry Potter,Ginny Weasley
+extraships:Harry Potter/Ginny Weasley

 [www.squidge.org/peja]
 # www.squidge.org/peja calls it Fandom <shrug>
 category_label:Fandom
 # Remove numWords -- www.squidge.org/peja word counts are inaccurate
-titlepage_entries: series,category,genre,language,characters,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,site,description
+titlepage_entries: series,category,genre,language,characters,ships,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,site,description

 [www.squidge.org/peja:txt]
 ## Add URLs since there aren't links.
@@ -682,6 +1005,16 @@ titlepage_entries: series,category,genre,language,characters,status,datePublishe
 titlepage_entries: series,category,genre,language,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,site,storyUrl, authorUrl, description

 [www.storiesofarda.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Lord of the Rings
+
+[www.thepetulantpoetess.com]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword

 [www.thewriterscoffeeshop.com]
 ## Some sites require login (or login for some rated stories) The
@@ -713,8 +1046,13 @@ collect_series: false
 #password:yourpassword

 [www.twilightarchives.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Twilight

 [www.twilighted.net]
+## Site dedicated to these categories/characters/ships
+extracategories:Twilight
+
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -726,6 +1064,9 @@ collect_series: false
 collect_series: false

 [www.twiwrite.net]
+## Site dedicated to these categories/characters/ships
+extracategories:Twilight
+
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -737,6 +1078,11 @@ collect_series: false
 collect_series: false

 [www.walkingtheplank.org]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+extracharacters:Severus Snape,Harry Potter
+extraships:Severus Snape/Harry Potter
+
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
@@ -757,7 +1103,15 @@ collect_series: false
 ## personal.ini, not defaults.ini.
 #is_adult:true

+[www.wolverineandrogue.com]
+## Site dedicated to these categories/characters/ships
+extracategories:X-Men Movie
+extracharacters:Wolverine,Rogue
+
 [www.wraithbait.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Stargate: Atlantis
+
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
@@ -1,14 +1,25 @@
 ## This is an example of what your personal configuration might look
-## like.
+## like.  Uncomment options by removing the '#' in front of them.

 [defaults]
 ## Some sites also require the user to confirm they are adult for
-## adult content.  In commandline version, this should go in your
-## personal.ini, not defaults.ini.
+## adult content.  Uncomment by removing '#' in front of is_adult.
 #is_adult:true

-## include images from img tags in the body and summary of
-## stories.  Images will be converted to jpg for size if possible.
+## Don't like the numbers at the start of chapter titles on some
+## sites?  You can use strip_chapter_numbers to strip them off.  Just
+## want to make them all look the same?  Strip them off, then add them
+## back on with add_chapter_numbers.  Don't like the way it strips
+## numbers or adds them back?  See chapter_title_strip_pattern and
+## chapter_title_add_pattern.
+#strip_chapter_numbers:true
+#add_chapter_numbers:true
+
+[epub]
+## include images from img tags in the body and summary of stories.
+## Images will be converted to jpg for size if possible.  Images work
+## in epub format only.  To get mobi or other format with images,
+## download as epub and use Calibre to convert.
 #include_images:true

 ## If not set, the summary will have all html stripped for safety.
@@ -29,18 +40,11 @@
 ## space.
 #grayscale_images: false

-## Try to collect series name and number of this story in series.
-## Some sites (ab)use 'series' for reading lists and personal
-## collections.  This lets us turn it on and off by site without
-## keeping a lengthy titlepage_entries per site and prevents it
-## updating in the plugin.
-## Turn off in [defaults] or [overrides] to prevent all sites from
-## updating series column.
-## default is true
-#collect_series: false

 ## Most common, I expect will be using this to save username/passwords
-## for different sites.
+## for different sites.  Here are a few examples.  See defaults.ini
+## for the full list.
+
 [www.twilighted.net]
 #username:YourPenname
 #password:YourPassword
@@ -1,3 +1,21 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Other code contributed by Pau Sanchez(bbcodeutils).
+
 To use, do:

 python downloader.py [-f (epub|html|txt)] <url>
Author	SHA1	Message	Date
Jim Miller	4bb91cd0c5	Extra metadata feature(refactoring), loosen url check on AO3, fix auth on castlefans.org.	2012-09-21 12:55:26 -05:00
Jim Miller	e5b0974d0e	Remove extra '[' from squidgeorgpeja Rating when no stars.	2012-09-17 12:29:12 -05:00
Jim Miller	0f77c2428b	Added cover images for dramione.org, thanks ilovejedd.	2012-09-14 17:43:48 -05:00
Jim Miller	bd57353601	Add/Integrate bloodtiesfancom, thehookupzonenet, thanks to marillapm67.	2012-09-14 17:08:22 -05:00
marillapm67	693863a28a	A Criminal Minds Fictions downloader.	2012-09-14 20:17:53 +00:00
Jim Miller	db4ca65bb8	Added tag FanFictionDownLoader-4.4.25 for changeset 8bf2a1c8e415	2012-09-14 12:12:15 -05:00
Jim Miller	686223aa6e	Added tag calibre-plugin-1.6.09 for changeset 8bf2a1c8e415	2012-09-14 12:12:02 -05:00
Jim Miller	c06a4ee9de	Integrate merlinfic.dtwins.co.uk, bump versions.	2012-09-14 12:11:41 -05:00
Ida	b66af0ef73	New adapter for merlinfic.dtwins.co.uk	2012-09-13 00:11:12 -04:00
Jim Miller	0ca71a6455	Add a BBCode lib for summaries on fimfic(needed to mod for unicode), Default keep_summary_html:true to now, Changes to the AO3, ficbook.net, ficwad.com adapters for summary HTML. Add options to strip and add chapter numbers to chapter titles.	2012-09-12 18:03:30 -05:00
Jim Miller	ab521ac093	No images for non-epub forms, show that clearly in example.ini.	2012-09-10 22:53:30 -05:00
Jim Miller	a85266e908	Added tag FanFictionDownLoader-4.4.24 for changeset e651eeeb1715	2012-09-06 17:24:02 -05:00
Jim Miller	2b3826d69b	Added tag calibre-plugin-1.6.08 for changeset e651eeeb1715	2012-09-06 17:23:41 -05:00
Jim Miller	079a0f5906	Scroll custom columns config, bump versions.	2012-09-06 17:23:25 -05:00
Jim Miller	f31fcc3341	Add extracategories, etc, Relationships, another Mac fix, fix for & in tags.	2012-09-05 12:20:35 -05:00
Ida	7eb1685be1	Allows different skins for stories that require login	2012-08-28 19:46:50 -04:00
Jim Miller	30997053ef	Added tag calibre-plugin-1.6.07 for changeset 9bad71f4988c	2012-08-27 12:14:01 -05:00
Jim Miller	347eb69044	Shortcuts for config/about for Mac. Handle tags inside metadata on ksarchivecom. Some Mac versions crash if there are menu items set to not have shortcuts. Also accept https, but change to http.	2012-08-27 12:13:31 -05:00
Jim Miller	86329a1fe3	Added tag FanFictionDownLoader-4.4.23a for changeset d9c3c28e545c	2012-08-25 21:50:32 -05:00
Jim Miller	3bcc31a404	Added tag calibre-plugin-1.6.06 for changeset d9c3c28e545c	2012-08-25 21:50:12 -05:00
Jim Miller	5e7c28d2dc	Integrate new site, add adobe-text-layout to epub CSS for no nook hyphenation.	2012-08-25 21:49:36 -05:00
Ida	574981931f	New adapter for www.sinful-desire.org	2012-08-24 13:29:30 -04:00
Jim Miller	a54765cf86	Added tag FanFictionDownLoader-4.4.23 for changeset 4437edd5f6fc	2012-08-23 11:57:46 -05:00
Jim Miller	7cd77c64d1	Added tag calibre-plugin-1.6.05 for changeset 4437edd5f6fc	2012-08-23 11:57:17 -05:00
Jim Miller	fd68b32111	Fix web site.	2012-08-23 11:57:07 -05:00
Jim Miller	eb969cfe2f	Integrate new sites.	2012-08-23 11:56:33 -05:00
Ida	337da79dbc	New adapters for www.wolverineandrogue.com/wrfa/ www.thepetulantpoetess.com www.scarvesandcoffee.net http://www.dracoandginny.com	2012-08-22 22:23:05 -04:00
Jim Miller	3176b4944b	Added tag FanFictionDownLoader-4.4.22a for changeset 7a2fb1721cc5	2012-08-18 11:18:35 -05:00
Jim Miller	a18c52e5a3	Added tag calibre-plugin-1.6.04 for changeset 7a2fb1721cc5	2012-08-18 11:18:19 -05:00
Jim Miller	1cfaff9d42	Workaround for poor HTML in summary(yourfanfictioncom), minor tweaks.	2012-08-18 11:18:04 -05:00
Jim Miller	793c65495f	Fix checkbox for Flesch-Kincaid Grade Level.	2012-08-07 12:45:53 -05:00
Jim Miller	efb908d438	Added tag FanFictionDownLoader-4.4.22 for changeset 09000ad9e797	2012-08-06 21:24:09 -05:00
Jim Miller	6ea015b111	Added tag calibre-plugin-1.6.03 for changeset 09000ad9e797	2012-08-06 21:23:54 -05:00