Bump versions.

Make finesotries.com adapter accept story detail pages as urls
Add optional call to 'Count Pages' plugin. Requires 'Count Pages' 1.6.0+
2026-09-15 11:36:30 +08:00 · 2012-07-15 10:42:37 -05:00 · 2012-07-14 18:33:23 -04:00 · 2012-07-14 13:52:54 -05:00 · 2012-07-13 22:53:59 -05:00 · 2012-07-13 23:22:20 -04:00
33 changed files with 2573 additions and 134 deletions
@@ -1,6 +1,6 @@
 # ffd-retief-hrd fanfictiondownloader
 application: fanfictiondownloader
-version: 4-4-18
+version: 4-4-20
 runtime: python27
 api_version: 1
 threadsafe: true
@@ -27,7 +27,7 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
    description         = 'UI plugin to download FanFiction stories from various sites.'
    supported_platforms = ['windows', 'osx', 'linux']
    author              = 'Jim Miller'
-    version             = (1, 5, 42)
+    version             = (1, 5, 44)
    minimum_calibre_version = (0, 8, 30)

    #: This field defines the GUI plugin class that contains all the code
@@ -61,6 +61,8 @@ all_prefs.defaults['gcnewonly'] = False
 all_prefs.defaults['gc_site_settings'] = {}
 all_prefs.defaults['allow_gc_from_ini'] = True

+all_prefs.defaults['countpagesstats'] = []
+
 all_prefs.defaults['errorcol'] = ''
 all_prefs.defaults['custom_cols'] = {}

@@ -165,6 +167,11 @@ class ConfigWidget(QWidget):
        if 'Generate Cover' not in plugin_action.gui.iactions:
            self.generatecover_tab.setEnabled(False)

+        self.countpages_tab = CountPagesTab(self, plugin_action)
+        tab_widget.addTab(self.countpages_tab, 'Count Pages')
+        if 'Count Pages' not in plugin_action.gui.iactions:
+            self.countpages_tab.setEnabled(False)
+
        self.columns_tab = CustomColumnsTab(self, plugin_action)
        tab_widget.addTab(self.columns_tab, 'Custom Columns')

@@ -218,8 +225,23 @@ class ConfigWidget(QWidget):
        prefs['gc_site_settings'] = gc_site_settings
        prefs['allow_gc_from_ini'] = self.generatecover_tab.allow_gc_from_ini.isChecked()

+        # Count Pages tab
+        countpagesstats = []
+        
+        if self.countpages_tab.pagecount.isChecked():
+            countpagesstats.append('PageCount')
+        if self.countpages_tab.wordcount.isChecked():
+            countpagesstats.append('WordCount')
+        if self.countpages_tab.fleschreading.isChecked():
+            countpagesstats.append('FleschReading')
+        if self.countpages_tab.fleschgrade.isChecked():
+            countpagesstats.append('FleschGrade')
+        if self.countpages_tab.gunningfog.isChecked():
+            countpagesstats.append('GunningFog')
+            
+        prefs['countpagesstats'] = countpagesstats
+        
        # Custom Columns tab
-
        # error column
        prefs['errorcol'] = unicode(self.columns_tab.errorcol.itemData(self.columns_tab.errorcol.currentIndex()).toString())

@@ -552,13 +574,61 @@ class GenerateCoverTab(QWidget):
        self.gcnewonly.setChecked(prefs['gcnewonly'])
        self.l.addWidget(self.gcnewonly)

-        self.allow_gc_from_ini = QCheckBox('Allow generate_cover_settings from personal.ini to override.',self)
+        self.allow_gc_from_ini = QCheckBox('Allow generate_cover_settings from personal.ini to override',self)
        self.allow_gc_from_ini.setToolTip("The personal.ini parameter generate_cover_settings allows you to choose a GC setting based on metadata rather than site, but it's much more complex.<br \>generate_cover_settings is ignored when this is off.")
        self.allow_gc_from_ini.setChecked(prefs['allow_gc_from_ini'])
        self.l.addWidget(self.allow_gc_from_ini)
            
        self.l.insertStretch(-1)
        
+class CountPagesTab(QWidget):
+
+    def __init__(self, parent_dialog, plugin_action):
+        self.parent_dialog = parent_dialog
+        self.plugin_action = plugin_action
+        QWidget.__init__(self)
+        
+        self.l = QVBoxLayout()
+        self.setLayout(self.l)
+
+        label = QLabel('These settings provide integration with the Count Pages Plugin.  Count Pages can automatically update custom columns with page, word and reading level statistics.  You have to create and configure the columns in Count Pages first.')
+        label.setWordWrap(True)
+        self.l.addWidget(label)
+        self.l.addSpacing(5)
+
+        label = QLabel('If any of the settings below are checked, when stories are added or updated, the Count Pages Plugin will be called to update the checked statistics.')
+        label.setWordWrap(True)
+        self.l.addWidget(label)
+        self.l.addSpacing(5)
+        
+        # 'PageCount', 'WordCount', 'FleschReading', 'FleschGrade', 'GunningFog'
+        self.pagecount = QCheckBox('PageCount',self)
+        self.pagecount.setToolTip('Which column and algorithm to use are configured in Count Pages.')
+        self.pagecount.setChecked('PageCount' in prefs['countpagesstats'])
+        self.l.addWidget(self.pagecount)
+            
+        self.wordcount = QCheckBox('WordCount',self)
+        self.wordcount.setToolTip('Which column and algorithm to use are configured in Count Words.\nWill overwrite word count from FFDL metadata if set to update the same custom column.')
+        self.wordcount.setChecked('WordCount' in prefs['countpagesstats'])
+        self.l.addWidget(self.wordcount)
+
+        self.fleschreading = QCheckBox('FleschReading',self)
+        self.fleschreading.setToolTip('Which column and algorithm to use are configured in Count Pages.')
+        self.fleschreading.setChecked('FleschReading' in prefs['countpagesstats'])
+        self.l.addWidget(self.fleschreading)
+        
+        self.fleschgrade = QCheckBox('Fleschgrade',self)
+        self.fleschgrade.setToolTip('Which column and algorithm to use are configured in Count Pages.')
+        self.fleschgrade.setChecked('Fleschgrade' in prefs['countpagesstats'])
+        self.l.addWidget(self.fleschgrade)
+        
+        self.gunningfog = QCheckBox('GunningFog',self)
+        self.gunningfog.setToolTip('Which column and algorithm to use are configured in Count Pages.')
+        self.gunningfog.setChecked('GunningFog' in prefs['countpagesstats'])
+        self.l.addWidget(self.gunningfog)
+        
+        self.l.insertStretch(-1)
+        
 class OtherTab(QWidget):

    def __init__(self, parent_dialog, plugin_action):
@@ -371,7 +371,7 @@ class AuthorTableWidgetItem(ReadOnlyTableWidgetItem):

    #Qt uses a simple < check for sorting items, override this to use the sortKey
    def __lt__(self, other):
-        return self.sort_key < other.sort_key
+        return self.sort_key.lower() < other.sort_key.lower()

 class UpdateExistingDialog(SizePersistedDialog):
    def __init__(self, gui, header, prefs, icon, books,
@@ -623,7 +623,7 @@ class StoryListTableWidget(QTableWidget):
        title_cell.setData(Qt.UserRole, QVariant(row))
        self.setItem(row, 1, title_cell)
        
-        self.setItem(row, 2, AuthorTableWidgetItem(book['author'], book['author_sort']))
+        self.setItem(row, 2, AuthorTableWidgetItem(", ".join(book['author']), ", ".join(book['author_sort'])))
        
        url_cell = ReadOnlyTableWidgetItem(book['url'])
        #url_cell.setData(Qt.UserRole, QVariant(book['url']))
@@ -19,6 +19,7 @@ from PyQt4.Qt import (QApplication, QMenu, QToolButton)
 from PyQt4.Qt import QPixmap, Qt
 from PyQt4.QtCore import QBuffer

+from calibre.constants import numeric_version as calibre_version

 from calibre.ptempfile import PersistentTemporaryFile, PersistentTemporaryDirectory, remove_dir
 from calibre.ebooks.metadata import MetaInformation, authors_to_string
@@ -193,10 +194,10 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                                                            shortcut_name='Get URLs from Selected Books',
                                                            triggered=self.get_list_urls)

-            self.get_list_action = self.create_menu_item_ex(self.menu, 'Get Story URLs from Web Page', image='view.png',
-                                                            unique_name='Get Story URLs from Web Page',
-                                                            shortcut_name='Get Story URLs from Web Page',
-                                                            triggered=self.get_urls_from_page)
+            self.get_list_url_action = self.create_menu_item_ex(self.menu, 'Get Story URLs from Web Page', image='view.png',
+                                                                unique_name='Get Story URLs from Web Page',
+                                                                shortcut_name='Get Story URLs from Web Page',
+                                                                triggered=self.get_urls_from_page)

            self.menu.addSeparator()
            self.config_action = create_menu_action_unique(self, self.menu, '&Configure Plugin', shortcut=False,
@@ -205,11 +206,11 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                                                           shortcut_name='Configure FanFictionDownLoader',
                                                           triggered=partial(do_user_config,parent=self.gui))
            
-            self.config_action = create_menu_action_unique(self, self.menu, '&About Plugin', shortcut=False,
-                                                           image= 'images/icon.png',
-                                                           unique_name='About FanFictionDownLoader',
-                                                           shortcut_name='About FanFictionDownLoader',
-                                                           triggered=self.about)
+            self.about_action = create_menu_action_unique(self, self.menu, '&About Plugin', shortcut=False,
+                                                          image= 'images/icon.png',
+                                                          unique_name='About FanFictionDownLoader',
+                                                          shortcut_name='About FanFictionDownLoader',
+                                                          triggered=self.about)
            
            # Before we finalize, make sure we delete any actions for menus that are no longer displayed
            for menu_id, unique_name in self.old_actions_unique_map.iteritems():
@@ -480,7 +481,7 @@ make_firstimage_cover:true

        book['all_metadata'] = story.getAllMetadata(removeallentities=True)
        book['title'] = story.getMetadata("title", removeallentities=True)
-        book['author_sort'] = book['author'] = story.getMetadata("author", removeallentities=True)
+        book['author_sort'] = book['author'] = story.getList("author", removeallentities=True)
        book['publisher'] = story.getMetadata("site")
        book['tags'] = writer.getTags() # getTags could be moved up into adapter now.  Adapter didn't used to know the fileform
        book['comments'] = stripHTML(story.getMetadata("description")) #, removeallentities=True) comments handles entities better.
@@ -520,20 +521,30 @@ make_firstimage_cover:true
        # only care about collisions when not ADDNEW
        elif collision != ADDNEW:
            # 'new' book from URL.  collision handling applies.
-            print("from URL")
+            print("from URL(%s)"%url)

-            # find dups
-            mi = MetaInformation(story.getMetadata("title", removeallentities=True),
-                                 [story.getMetadata("author", removeallentities=True)]) # author is a list.
-            identicalbooks = db.find_identical_books(mi)
-            ## removed for being overkill.
-            # for ib in identicalbooks:
-            #     # only *really* identical if URL matches, too.
-            #     # XXX make an option?
-            #     if self._get_story_url(db,ib) == url:
-            #         identicalbooks.append(ib)
-            #print("identicalbooks:%s"%identicalbooks)
+            # try to find by identifier url first.
+            searchstr = 'identifiers:"=url:%s"'%url.replace(":","|")
+            identicalbooks = db.search_getting_ids(searchstr, None)
+            if len(identicalbooks) < 1:
+                # find dups
+                authlist = story.getList("author", removeallentities=True)
+                if len(authlist) > 100 and calibre_version < (0, 8, 61):
+                    ## should be fixed from 0.8.61 on.  In the
+                    ## meantime, if it matches the title *and* first
+                    ## 100 authors, I'm prepared to assume it's a
+                    ## match.
+                    print("reduce author list to 100 only when calibre < 0.8.61")
+                    authlist = authlist[:100]
+                mi = MetaInformation(story.getMetadata("title", removeallentities=True),
+                                     authlist)
+                identicalbooks = db.find_identical_books(mi)
+                if len(identicalbooks) > 0:
+                    print("existing found by title/author(s)")

+            else:
+                print("existing found by identifier URL")
+                
            if collision == SKIP and identicalbooks:
                raise NotGoingToDownload("Skipping duplicate story.","list_remove.png")

@@ -544,7 +555,6 @@ make_firstimage_cover:true
            if collision == CALIBREONLY and not identicalbooks:
                collision = ADDNEW
                options['collision'] = ADDNEW
-            #     raise NotGoingToDownload("Not updating Calibre Metadata, no existing book to update.","search_delete_saved.png")

            if len(identicalbooks)>0:
                book_id = identicalbooks.pop()
@@ -658,7 +668,6 @@ make_firstimage_cover:true
            custom_columns = self.gui.library_view.model().custom_columns
            if prefs['errorcol'] != '' and prefs['errorcol'] in custom_columns:
                label = custom_columns[prefs['errorcol']]['label']
-                print("errorcol label:%s"%label)
                ## if error column and all bad.
                self.previous = self.gui.library_view.currentIndex()
                LoopProgressDialog(self.gui,
@@ -704,9 +713,9 @@ make_firstimage_cover:true
                                  'collision':ADDNEW,
                                  'updatemeta':True,
                                  'updateepubcover':True},):
-        
-        print("add/update bad %s %s %s"%(book['title'],book['url'],book['comment']))
-        db.set_custom(book['calibre_id'], book['comment'], label=label, commit=True)
+        if book['calibre_id']:
+            print("add/update bad %s %s %s"%(book['title'],book['url'],book['comment']))
+            db.set_custom(book['calibre_id'], book['comment'], label=label, commit=True)

    def _update_books_completed(self, book_list, options={}, showlist=True):
        
@@ -720,7 +729,7 @@ make_firstimage_cover:true
            self.gui.library_view.model().books_added(len(add_list))
            self.gui.library_view.model().refresh_ids(add_ids)

-        if update_ids:
+        if len(update_list):
            self.gui.library_view.model().refresh_ids(update_ids)

        current = self.gui.library_view.currentIndex()
@@ -744,7 +753,13 @@ make_firstimage_cover:true
    
        print("all done, remove temp dir.")
        remove_dir(options['tdir'])
-        
+
+        all_ids = add_ids
+        all_ids.extend(update_ids)
+        if 'Count Pages' in self.gui.iactions and len(prefs['countpagesstats']) and len(all_ids):
+            cp_plugin = self.gui.iactions['Count Pages']
+            cp_plugin.count_statistics(all_ids,prefs['countpagesstats'])
+
    def download_list_completed(self, job, options={}):
        if job.failed:
            self.gui.job_exception(job, dialog_title='Failed to Download Stories')
@@ -784,7 +799,6 @@ make_firstimage_cover:true
                if prefs['errorcol'] != '' and prefs['errorcol'] in custom_columns:
                    self.gui.status_bar.show_message(_('Adding/Updating %s BAD books.'%total_bad))
                    label = custom_columns[prefs['errorcol']]['label']
-                    print("errorcol label:%s"%label)
                    ## if error column and all bad.
                    LoopProgressDialog(self.gui,
                                       bad_list,
@@ -857,9 +871,11 @@ make_firstimage_cover:true

        # set author link if found.  All current adapters have authorUrl, except anonymous on AO3.
        if 'authorUrl' in book['all_metadata']:
-            autid=db.get_author_id(book['author'])
-            db.set_link_field_for_author(autid, unicode(book['all_metadata']['authorUrl']),
-                                         commit=False, notify=False)
+            authurls = book['all_metadata']['authorUrl'].split(", ")
+            for i, auth in enumerate(book['author']):
+                autid=db.get_author_id(auth)
+                db.set_link_field_for_author(autid, unicode(authurls[i]),
+                                             commit=False, notify=False)
            
        db.set_metadata(book_id,mi)

@@ -895,8 +911,13 @@ make_firstimage_cover:true
                db.set_custom(book_id, val, label=label, commit=False)

        db.commit()
-        
+
        if 'Generate Cover' in self.gui.iactions and (book['added'] or not prefs['gcnewonly']):
+            
+            # force a refresh if generating cover so complex composite
+            # custom columns are current and correct
+            db.refresh_ids([book_id])
+                
            gc_plugin = self.gui.iactions['Generate Cover']
            setting_name = None
            if prefs['allow_gc_from_ini']:
@@ -911,7 +932,7 @@ make_firstimage_cover:true
                for line in adapter.getConfig('generate_cover_settings').splitlines():
                    if "=>" in line:
                        (template,regexp,setting) = map( lambda x: x.strip(), line.split("=>") )
-                        value = Template(template).substitute(book['all_metadata']).encode('utf8')
+                        value = Template(template).safe_substitute(book['all_metadata']).encode('utf8')
                        print("%s(%s) => %s => %s"%(template,value,regexp,setting))
                        if re.search(regexp,value):
                            setting_name = setting
@@ -937,7 +958,6 @@ make_firstimage_cover:true
        ## if error column set.
        if prefs['errorcol'] != '' and prefs['errorcol'] in custom_columns:
            label = custom_columns[prefs['errorcol']]['label']
-            print("_update_book label:%s"%label)
            db.set_custom(book['calibre_id'], '', label=label, commit=True) # book['comment']

    def _get_clean_reading_lists(self,lists):
@@ -1001,19 +1021,19 @@ make_firstimage_cover:true
                        message="<p>You configured FanFictionDownLoader to automatically update Reading List '%s', but you don't have a list of that name?</p>"%l
                        confirm(message,'fanfictiondownloader_no_reading_list_%s'%l, self.gui)

-    def _find_existing_book_id(self,db,book,matchurl=True):
-        mi = MetaInformation(book["title"],[book["author"]]) # author is a list.
-        identicalbooks = db.find_identical_books(mi)
-        if matchurl: # only *really* identical if URL matches, too.
-            for ib in identicalbooks:
-                if self._get_story_url(db,ib) == book['url']:
-                    return ib
-        if identicalbooks:
-            return identicalbooks.pop()
-        return None
+    # def _find_existing_book_id(self,db,book,matchurl=True):
+    #     mi = MetaInformation(book["title"],book["author"]) # author is a list.
+    #     identicalbooks = db.find_identical_books(mi)
+    #     if matchurl: # only *really* identical if URL matches, too.
+    #         for ib in identicalbooks:
+    #             if self._get_story_url(db,ib) == book['url']:
+    #                 return ib
+    #     if identicalbooks:
+    #         return identicalbooks.pop()
+    #     return None
    
    def _make_mi_from_book(self,book):
-        mi = MetaInformation(book['title'],[book['author']]) # author is a list.
+        mi = MetaInformation(book['title'],book['author']) # author is a list.
        mi.set_identifiers({'url':book['url']})
        mi.publisher = book['publisher']
        mi.tags = book['tags']
@@ -1050,8 +1070,7 @@ make_firstimage_cover:true
        book['good'] = True
        book['calibre_id'] = None
        book['title'] = 'Unknown'
-        book['author'] = 'Unknown'
-        book['author_sort'] = 'Unknown'
+        book['author_sort'] = book['author'] = ['Unknown'] # list
        book['begin'] = None
        book['end'] = None
            
@@ -1067,8 +1086,7 @@ make_firstimage_cover:true
        book['good'] = good
        book['calibre_id'] = idval
        book['title'] = 'Unknown'
-        book['author'] = 'Unknown'
-        book['author_sort'] = 'Unknown'
+        book['author_sort'] = book['author'] = ['Unknown'] # list
        book['begin'] = None
        book['end'] = None
            
@@ -1084,7 +1102,7 @@ make_firstimage_cover:true
        book['good'] = True
        book['calibre_id'] = mi.id
        book['title'] = mi.title
-        book['author'] = authors_to_string(mi.authors)
+        book['author'] = mi.authors
        book['author_sort'] = mi.author_sort
        book['comment'] = ''
        book['url'] = ""
@@ -357,6 +357,14 @@ extratags: FanFiction,Testing,HTML
 ## cover image.  This lets you exclude them.
 cover_exclusion_regexp:/images/.*?ribbon.gif

+[dark-solace.org]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
 [dramione.org]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -407,6 +415,14 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 #username:YourName
 #password:yourpassword

+[finestories.com]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
 [lumos.sycophanthex.com]
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
@@ -450,6 +466,12 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true

+[pretendercentre.com]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
 [samdean.archive.nu]

 [sg1-heliopolis.com]
@@ -460,8 +482,34 @@ cover_exclusion_regexp:/images/.*?ribbon.gif

 [stargate-atlantis.org]

+[svufiction.com]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
 [thehexfiles.net]

+[themasque.net]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
 [thequidditchpitch.org]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -558,6 +606,8 @@ extratags:
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true

+[www.hpfanficarchive.com]
+
 [www.ik-eternal.net]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -647,6 +697,8 @@ collect_series: false
 #username:YourName
 #password:yourpassword

+[www.twilightarchives.com]
+
 [www.twilighted.net]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -677,6 +729,19 @@ collect_series: false

 [www.whofic.com]

+[www.wizardtales.net]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
 [www.wraithbait.com]
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
@@ -16,8 +16,6 @@
 #

 import logging
-## XXX cli option for logging level.
-logging.basicConfig(level=logging.INFO,format="%(levelname)s:%(filename)s(%(lineno)d):%(message)s")

 import sys, os
 from os.path import normpath, expanduser, isfile, join
@@ -74,9 +72,18 @@ def main():
   parser.add_option("-l", "--list",
                     action="store_true", dest="list",
                     help="Get list of valid story URLs from page given.",)
+   parser.add_option("-d", "--debug",
+                     action="store_true", dest="debug",
+                     help="Show debug output while downloading.",)
   
   (options, args) = parser.parse_args()

+   if options.debug:
+       logging.basicConfig(level=logging.DEBUG,format="%(levelname)s:%(filename)s(%(lineno)d):%(message)s")
+   else:
+       logging.basicConfig(level=logging.INFO,format="%(levelname)s:%(filename)s(%(lineno)d):%(message)s")
+
+   
   if len(args) != 1:
       parser.error("incorrect number of arguments")

@@ -141,11 +148,16 @@ def main():
       ## Check for include_images and absence of PIL, give warning.
       if adapter.getConfig('include_images'):
           try:
-               import Image
+               from calibre.utils.magick import Image
+               logging.debug("Using calibre.utils.magick")
           except:
-               print "You have include_images enabled, but Python Image Library(PIL) isn't found.\nImages will be included full size in original format.\nContinue? (y/n)?"
-               if not sys.stdin.readline().strip().lower().startswith('y'):
-                   return
+               try:
+                   import Image
+                   logging.debug("Using PIL")
+               except:
+                   print "You have include_images enabled, but Python Image Library(PIL) isn't found.\nImages will be included full size in original format.\nContinue? (y/n)?"
+                   if not sys.stdin.readline().strip().lower().startswith('y'):
+                       return
               

       ## three tries, that's enough if both user/pass & is_adult needed,
@@ -81,6 +81,14 @@ import adapter_ponyfictionarchivenet
 import adapter_sg1heliopoliscom
 import adapter_ncisficcom
 import adapter_nationallibrarynet
+import adapter_themasquenet
+import adapter_pretendercentrecom
+import adapter_darksolaceorg
+import adapter_finestoriescom
+import adapter_hpfanficarchivecom
+import adapter_svufictioncom
+import adapter_twilightarchivescom
+import adapter_wizardtalesnet

 ## This bit of complexity allows adapters to be added by just adding
 ## importing.  It eliminates the long if/else clauses we used to need
@@ -118,6 +126,11 @@ def getAdapter(config,url,fileform=None):

    logging.debug("site:"+domain)
    cls = getClassFor(domain)
+    if not cls and domain.startswith("www."):
+        domain = domain.replace("www.","")
+        logging.debug("trying site:without www: "+domain)
+        cls = getClassFor(domain)
+        fixedurl = fixedurl.replace("http://www.","http://")
    if not cls:
        logging.debug("trying site:www."+domain)
        cls = getClassFor("www."+domain)
@@ -155,14 +155,15 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
        self.story.setMetadata('title',a.string)
 		
        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"^/users/\w+/pseuds/\w+"))
-        if a == None: # ao3 allows for author 'Anonymous' with no author link.
+        alist = soup.findAll('a', href=re.compile(r"^/users/\w+/pseuds/\w+"))
+        if len(alist) < 1: # ao3 allows for author 'Anonymous' with no author link.
            self.story.setMetadata('author','Anonymous')
            self.story.setMetadata('authorUrl',self.url)
        else:
-            self.story.setMetadata('authorId',a['href'].split('/')[2])
-            self.story.setMetadata('authorUrl','http://'+self.host+a['href'])
-            self.story.setMetadata('author',a.text)
+            for a in alist:
+                self.story.addToList('authorId',a['href'].split('/')[2])
+                self.story.addToList('authorUrl','http://'+self.host+a['href'])
+                self.story.addToList('author',a.text)

        # Find the chapters:
        chapters=soup.findAll('a', href=re.compile(r'/works/'+self.story.getMetadata('storyId')+"/chapters/\d+$"))
@@ -0,0 +1,279 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return DarkSolaceOrgAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class DarkSolaceOrgAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+		
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/elysian/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','dksl')
+
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","Buffy the Vampire Slayer")
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%d %B %Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'dark-solace.org'
+        
+    @classmethod
+    def getAcceptDomains(cls):
+        return ['www.dark-solace.org','dark-solace.org']
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/elysian/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://")+"(www\.)?"+re.escape(self.getSiteDomain()+"/elysian/viewstory.php?sid=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'This story contains adult content not suitable for children' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['rememberme'] = '1'
+        params['sid'] = ''
+        params['intent'] = ''
+        params['submit'] = 'Submit'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/elysian/user.php'
+        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "User Account Page" not in d : #Member Account
+            logging.info("Failed to login to URL %s as %s, or have no authorization to access the story" % (loginUrl, params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            addurl="&ageconsent=ok"
+            self.performLogin(url)
+            data = self._fetchUrl(url+addurl)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title and author
+        a = soup.find('div', {'id' : 'pagetitle'})
+        
+        aut = a.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',aut['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/elysian/'+aut['href'])
+        self.story.setMetadata('author',aut.string)
+        aut.extract()
+        
+        self.story.setMetadata('title',a.string[:(len(a.string)-3)])
+
+        # Find the chapters:
+        chapters=soup.find('select', {'name' : 'chapter'})
+        if chapters != None:
+            for chapter in chapters.findAll('option'):
+                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/elysian/viewstory.php?sid='+self.story.getMetadata('storyId')+'&chapter='+chapter['value']))
+        else:
+            self.chapterUrls.append((self.story.getMetadata('title'),url))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+        
+        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
+        
+        for list in asoup.findAll('div', {'class' : re.compile('listbox\s+')}):
+            a = list.find('a', href=re.compile(r'viewstory.php\?sid='))
+            if a != None:
+                if 'viewstory.php?sid='+self.story.getMetadata('storyId') in a['href']:
+                    break
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+                
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = list.findAll('span', {'class' : 'classification'})
+        for labelspan in labels:
+            label = labelspan.text
+            value = labelspan.nextSibling
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not (defaultGetattr(value,'class') == 'classification' or "Chapters: " in stripHTML(value)):
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value[:len(value)-2])
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'categories.php\?catid=\d+'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                for char in value.string.split(', '):
+                    if not 'None' in char:
+                        self.story.addToList('characters',char)
+
+            if 'Genre' in label:
+                for genre in value.string.split(', '):
+                    if not 'None' in genre:
+                        self.story.addToList('genre',genre)
+
+            if 'Warnings' in label:
+                for warning in value.string.split(', '):
+                    if not 'None' in warning:
+                        self.story.addToList('warnings',warning)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = list.find('a', href=re.compile(r"series.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/elysian/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            i=1
+            for a in storyas:
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if ('viewstory.php?sid='+self.story.getMetadata('storyId')) in a['href']:
+                        self.setSeries(series_name, i)
+                        break
+                    i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -62,7 +62,7 @@ class DramioneOrgAdapter(BaseSiteAdapter):

        # The date format will vary from site to site.
        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%dth %B %Y"
+        self.dateformat = "%d %B %Y"
            
    @staticmethod # must be @staticmethod, don't remove it.
    def getSiteDomain():
@@ -228,11 +228,11 @@ class DramioneOrgAdapter(BaseSiteAdapter):
                    self.story.setMetadata('status', 'In-Progress')

            if 'Published' in label:
-                value=value.replace('st','th').replace('nd','th').replace('rd','th')
+                value=re.sub(r"(\d+)(st|nd|rd|th)",r"\1",value)
                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
            
            if 'Updated' in label:
-                value=value.replace('st','th').replace('nd','th').replace('rd','th')
+                value=re.sub(r"(\d+)(st|nd|rd|th)",r"\1",value)
                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))

        try:
@@ -88,7 +88,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
            else:
                raise e
            
-        if "Unable to locate story with id of " in data:
+        if "Unable to locate story" in data:
            raise exceptions.StoryDoesNotExist(url)

        # some times "Chapter not found...", sometimes "Chapter text not found..."
@@ -184,7 +184,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):

        donechars = False
        while len(metalist) > 0:
-            if  metalist[0].startswith('Reviews') or metalist[0].startswith('Chapters') or metalist[0].startswith('Status') or metalist[0].startswith('id:'):
+            if  metalist[0].startswith('Reviews') or metalist[0].startswith('Chapters') or metalist[0].startswith('Status') or metalist[0].startswith('id:') or metalist[0].startswith('Favs:') or metalist[0].startswith('Follows:'):
                pass
            elif  metalist[0].startswith('Updated'):
                self.story.setMetadata('dateUpdated',makeDate(metalist[0].split(':')[1].strip(), '%m-%d-%y'))
@@ -161,9 +161,9 @@ class FanFiktionDeAdapter(BaseSiteAdapter):
            self.story.addToList('genre',genre)
            
        if 'fertiggestellt' in head.text:
-            self.story.setMetadata('status', 'Fertiggestellt')
+            self.story.setMetadata('status', 'Completed')
        else:
-            self.story.setMetadata('status', 'In Arbeit')
+            self.story.setMetadata('status', 'In Progress')

        #find metadata on the author's page
        asoup = bs.BeautifulSoup(self._fetchUrl("http://"+self.getSiteDomain()+"?a=q&a1=v&t=nickdetailsstories&lbi=stories&ar=0&nick="+self.story.getMetadata('authorId')))
@@ -0,0 +1,284 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return FineStoriesComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class FineStoriesComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url
+        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2].split(':')[0])
+        if 'storyInfo' in self.story.getMetadata('storyId'):
+            self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/s/storyInfo.php?id='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','fnst')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%Y-%m-%d"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'finestories.com'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/s/10537 http://"+self.getSiteDomain()+"/s/10537:4010 http://"+self.getSiteDomain()+"/s/toryInfo.php?id=10537"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/s/")+r"(storyInfo.php\?id=)?\d+(:\d+)?(;\d+)?$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Free Registration' in data \
+                or "Invalid Password!" in data \
+                or "Invalid User Name!" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['theusername'] = self.username
+            params['thepassword'] = self.password
+        else:
+            params['theusername'] = self.getConfig("username")
+            params['thepassword'] = self.getConfig("password")
+        params['rememberMe'] = '1'
+        params['page'] = 'http://finestories.com/'
+        params['submit'] = 'Login'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/login.php'
+        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['theusername']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "My Account" not in d : #Member Account
+            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['theusername']))
+            raise exceptions.FailedToLogin(url,params['theusername'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'/s/'+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.text)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"/a/\w+"))
+        self.story.setMetadata('authorId',a['href'].split('/')[2])
+        self.story.setMetadata('authorUrl','http://'+self.host+a['href'])
+        self.story.setMetadata('author',a.text)
+
+        # Find the chapters:
+        chapters = soup.findAll('a', href=re.compile(r'/s/'+self.story.getMetadata('storyId')+":\d+$"))
+        if len(chapters) != 0:
+            for chapter in chapters:
+                # just in case there's tags, like <i> in chapter titles.
+                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+chapter['href']))
+        else:
+            self.chapterUrls.append((self.story.getMetadata('title'),'http://'+self.host+'/s/'+self.story.getMetadata('storyId')))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # surprisingly, the detailed page does not give enough details, so go to author's page
+        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
+        
+        for lc2 in asoup.findAll('td', {'class' : 'lc2'}):
+            if lc2.find('a')['href'] == '/s/'+self.story.getMetadata('storyId'):
+                break
+        
+        self.story.addToList('category',lc2.find('div', {'class' : 'typediv'}).text)
+        
+        self.story.setMetadata('numWords', lc2.findNext('td', {'class' : 'num'}).text)
+        
+        lc4 = lc2.findNext('td', {'class' : 'lc4'})
+        
+        
+        try:
+            a = lc4.find('a', href=re.compile(r"/library/show_series.php\?id=\d+"))
+            i = a.parent.text.split('(')[1].split(')')[0]
+            self.setSeries(a.text, i)
+        except:
+            pass
+        try:
+            a = lc4.find('a', href=re.compile(r"/library/universe.php\?id=\d+"))
+            self.story.addToList("category",a.text)
+        except:
+            pass
+            
+        for a in lc4.findAll('span', {'class' : 'help'}):
+            a.extract()
+            
+        self.setDescription('http://'+self.host+'/s/'+self.story.getMetadata('storyId'),lc4.text.split('[More Info')[0])
+            
+        for b in lc4.findAll('b'):
+            label = b.text
+            value = b.nextSibling
+            
+            if 'For Age' in label:
+                self.story.setMetadata('rating', value)
+                
+            if 'Tags' in label:
+                for genre in value.split(', '):
+                    self.story.addToList('genre',genre)
+                    
+            if 'Posted' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+                
+            if 'Concluded' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+                
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+                
+        status = lc4.find('span', {'class' : 'ab'})
+        if  status != None:
+            self.story.setMetadata('status', 'In-Progress')
+            if "Last Activity" in status.text:
+                self.story.setMetadata('dateUpdated', makeDate(status.text.split('Activity: ')[1].split(')')[0], self.dateformat))
+        else:
+            self.story.setMetadata('status', 'Completed')
+
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+        
+        # some big chapters are split over several pages
+        pager = div.find('span', {'class' : 'pager'})
+        if pager != None:
+            urls=pager.findAll('a')
+            urls=urls[:len(urls)-1]
+            
+            
+            for ur in urls:
+                soup = bs.BeautifulSoup(self._fetchUrl("http://"+self.getSiteDomain()+ur['href']),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+                div1 = soup.find('div', {'id' : 'story'})
+                
+                # appending next section
+                last=div.findAll('p')
+                next=div1.find('span', {'class' : 'conTag'}).nextSibling
+            
+                last[len(last)-1]=last[len(last)-1].append(next)
+                div.append(div1)
+            
+            
+            
+        # removing all the left-over stuff    
+        for a in div.findAll('span'):
+            a.extract()
+            
+        for a in div.findAll('h1'):
+            a.extract()
+        for a in div.findAll('h2'):
+            a.extract()
+        for a in div.findAll('h3'):
+            a.extract()
+        for a in div.findAll('h4'):
+            a.extract()
+        for a in div.findAll('br'):
+            a.extract()
+        for a in div.findAll('div', {'class' : 'date'}):
+            a.extract()
+            
+        a = div.find('form')
+        if a != None:
+            b = a.nextSibling
+            while b != None:
+                a.extract()
+                a=b
+                b=b.nextSibling
+        
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -0,0 +1,221 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return HPFanficArchiveComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class HPFanficArchiveComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/stories/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','hpffa')
+
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","Harry Potter")
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%B %d, %Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.hpfanficarchive.com'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/stories/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/stories/viewstory.php?sid=")+r"\d+$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/stories/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/stories/'+chapter['href']))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) # XXX
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/stories/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            i=1
+            for a in storyas:
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        break
+                    i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+        
+
+        soup = bs.BeautifulSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -197,7 +197,13 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
                svalue = ""
                while not defaultGetattr(value,'class') == 'label':
                    svalue += str(value)
-                    value = value.nextSibling
+                    # poor HTML(unclosed <p> for one) can cause run on
+                    # over the next label.
+                    if '<span class="label">' in svalue:
+                        svalue = svalue[0:svalue.find('<span class="label">')]
+                        break
+                    else:
+                        value = value.nextSibling
                self.setDescription(url,svalue)
                #self.story.setMetadata('description',stripHTML(svalue))

@@ -211,7 +217,9 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
                catstext = [cat.string for cat in cats]
                for cat in catstext:
-                    if cat.string.strip() in ('Poetry','Essays'):
+                    # ran across one story with an empty <a href="browse.php?type=categories&amp;catid=1"></a>
+                    # tag in the desc once.
+                    if cat and cat.strip() in ('Poetry','Essays'):
                        self.story.addToList('category',cat.string)

            if 'Characters' in label:
@@ -0,0 +1,255 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return PretenderCenterComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class PretenderCenterComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/missingpieces/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','ptdc')
+
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","The Pretender")
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%d/%m/%y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'pretendercentre.com'
+        
+    @classmethod
+    def getAcceptDomains(cls):
+        return ['www.pretendercentre.com','pretendercentre.com']
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/missingpieces/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://")+"(www\.)?"+re.escape(self.getSiteDomain()+"/missingpieces/viewstory.php?sid=")+r"\d+$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=4"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&amp;ageconsent=ok)?&amp;warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid &amp; error in url.
+                addurl = addurl.replace("&amp;","&")
+                url = self.url+'&index=1'+addurl
+                logging.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.text)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/missingpieces/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/missingpieces/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+                
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) # XXX
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/missingpieces/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            i=1
+            for a in storyas:
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        break
+                    i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.findAll('div', {'id' : 'story'})[1]
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -0,0 +1,271 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return SVUFictionComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class SVUFictionComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','svuf')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%b %d, %Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'svufiction.com'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Registered Users Only' in data \
+                or 'There is no such account on our website' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['cookiecheck'] = '1'
+        params['submit'] = 'Submit'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
+        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "Member Account" not in d : #Member Account
+            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+    
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=6"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+                
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+                
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&amp;ageconsent=ok)?&amp;warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid &amp; error in url.
+                addurl = addurl.replace("&amp;","&")
+                url = self.url+'&index=1'+addurl
+                logging.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        pt = soup.find('div', {'class' : 'title'})
+        a = pt.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        if a.text != "":
+            self.story.setMetadata('title',a.text)
+        else:
+            self.story.setMetadata('title','Banner')
+        
+        # Find authorid and URL from... author url.
+        a = pt.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        rating=pt.text.split('[')[1].split(']')[0]
+        self.story.setMetadata('rating', rating)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        bottom=soup.find('div', {'class' : 'bottom'}).text
+        self.story.setMetadata('numWords', bottom.split('Word count: ')[1].split(' Read:')[0])
+        self.story.setMetadata('datePublished', makeDate(bottom.split('Published: ')[1].split(' Updated:')[0], self.dateformat))
+        self.story.setMetadata('dateUpdated', makeDate(bottom.split('Updated: ')[1], self.dateformat))
+
+        
+        content=soup.find('div', {'class' : 'content'})
+        value=content.find('h1').nextSibling
+        svalue = ""
+        while not defaultGetattr(value,'class') == 'smaller':
+            svalue += str(value)
+            value = value.nextSibling
+        self.setDescription(url,svalue)
+        
+        status = content.text.split('Completed: ')[1]
+        if 'Yes' in status:
+            self.story.setMetadata('status', 'Completed')
+        else:
+            self.story.setMetadata('status', 'In-Progress')
+        
+        cats = content.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+        for cat in cats:
+            self.story.addToList('category',cat.string)
+            
+        chars = content.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+        for char in chars:
+            self.story.addToList('characters',char.string)
+            
+        genres = content.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+        for genre in genres:
+            self.story.addToList('genre',genre.string)
+            
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            i=1
+            for a in storyas:
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        break
+                    i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -103,8 +103,19 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
        
        self.story.setMetadata('rating','Tweenie')
        
-        self.story.setMetadata('authorId','98765')
-        self.story.setMetadata('authorUrl','http://author/url')
+
+        if self.story.getMetadata('storyId') == '673':
+            self.story.addToList('author','Author From List')
+            self.story.addToList('author','Author From List 2')
+        
+            self.story.addToList('authorId','98765')
+            self.story.addToList('authorId','98765-2')
+        
+            self.story.addToList('authorUrl','http://author/url')
+            self.story.addToList('authorUrl','http://author/url-2')
+        else:
+            self.story.setMetadata('authorId','98765')
+            self.story.setMetadata('authorUrl','http://author/url')

        self.story.addToList('warnings','Swearing')
        self.story.addToList('warnings','Violence')
@@ -114,10 +125,15 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
        self.story.addToList('category','Crossover')
        self.story.addToList('category',u'Puella Magi Madoka Magica/魔法少女まどか★マギカ')
        self.story.addToList('category',u'Magical Girl Lyrical Nanoha')
+
        self.story.addToList('genre','Fantasy')
        self.story.addToList('genre','SF')
        self.story.addToList('genre','Noir')
        
+        self.story.addToList('characters','Bob Smith')
+        self.story.addToList('characters','George Johnson')
+        self.story.addToList('characters','Fred Smythe')
+        
        self.chapterUrls = [(u'Prologue '+self.crazystring,self.url+"&chapter=1"),
                            ('Chapter 1, Xenos on Cinnabar',self.url+"&chapter=2"),
                            ('Chapter 2, Sinmay on Kintikin',self.url+"&chapter=3"),
@@ -170,6 +186,7 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
 <p>http://test1.com?sid=670 - Succeeds, but sleeps 2sec on each chapter</p>
 <p>http://test1.com?sid=671 - Succeeds, but sleeps 2sec metadata only</p>
 <p>http://test1.com?sid=672 - Succeeds, quick meta, sleeps 2sec chapters only</p>
+<p>http://test1.com?sid=673 - Succeeds, multiple authors</p>
 <p>Odd sid's will be In-Progress, evens complete.  sid&lt;10 will be assigned one of four languages and included in a series.</p>
 </div>
 '''
@@ -0,0 +1,271 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return TheMasqueNetAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class TheMasqueNetAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        if self.parsedUrl.path.split('/',)[1] == 'wiktt':
+            self.story.addToList("category","Harry Potter")
+            self.story.setMetadata('section','/wiktt/efiction/')
+            self.dateformat = "%m/%d/%Y"
+        else:
+            self.story.addToList("category","Originals")
+            self.story.setMetadata('section','/efiction/')
+            self.dateformat = "%b %d, %Y"
+            
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + self.story.getMetadata('section') + 'viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','msq')
+        
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'themasque.net'
+        
+    def getSiteExampleURLs(self):
+        return "http://themasque.net/wiktt/efiction/viewstory.php?sid=1234 http://themasque.net/efiction/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain())+"(/wiktt)?/efiction"+re.escape("/viewstory.php?sid=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Registered Users Only' in data \
+                or 'There is no such account on our website' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['cookiecheck'] = '1'
+        params['submit'] = 'Submit'
+    
+        loginUrl = 'http://' + self.getSiteDomain()  + self.story.getMetadata('section') + 'user.php?action=login'
+        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "Member Account" not in d : #Member Account
+            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=4"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&amp;ageconsent=ok)?&amp;warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid &amp; error in url.
+                addurl = addurl.replace("&amp;","&")
+                url = self.url+'&index=1'+addurl
+                logging.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host + self.story.getMetadata('section') + chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+                
+# summary, rated, word count, categories, characters, genre, warnings, completed, published, updated, seires
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.text
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -151,12 +151,33 @@ class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter):
        self.story.setMetadata('authorId',a['href'].split('/')[1].split('-')[1])
        self.story.setMetadata('authorUrl','http://'+self.host+a['href'])
        self.story.setMetadata('author',stripHTML(a))
+        authorurl = 'http://'+self.host+a['href']

+        ainfo = soup.find('a', href='/StoryInfo-%s-1'%self.story.getMetadata('storyId'))
+        if ainfo != None: # indicates multiple authors/contributors.
+            try:
+                # going to pull part of the meta data from author list page.
+                infourl = 'http://'+self.host+ainfo['href']
+                logging.debug("**StoryInfo** URL: "+infourl)
+                infodata = self._fetchUrl(infourl)
+                infosoup = bs.BeautifulSoup(infodata)
+
+                for a in infosoup.findAll('a',href=re.compile(r"^/Author-\d+")):
+                    self.story.addToList('authorId',a['href'].split('/')[1].split('-')[1])
+                    self.story.addToList('authorUrl','http://'+self.host+a['href'].replace("/Author-","/AuthorStories-"))
+                    self.story.addToList('author',stripHTML(a))                
+                                            
+            except urllib2.HTTPError, e:
+                if e.code == 404:
+                    raise exceptions.StoryDoesNotExist(url)
+                else:
+                    raise e
+    
        try:
-            # going to pull part of the meta data from author list page.
-            logging.debug("**AUTHOR** URL: "+self.story.getMetadata('authorUrl'))
-            authordata = self._fetchUrl(self.story.getMetadata('authorUrl'))
-            descurl=self.story.getMetadata('authorUrl')
+            # going to pull part of the meta data from *primary* author list page.
+            logging.debug("**AUTHOR** URL: "+authorurl)
+            authordata = self._fetchUrl(authorurl)
+            descurl=authorurl
            authorsoup = bs.BeautifulSoup(authordata)
            # author can have several pages, scan until we find it.
            while( not authorsoup.find('a', href=re.compile(r"^/Story-"+self.story.getMetadata('storyId'))) ):
@@ -209,7 +230,7 @@ class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter):
                if not BtVSNonX:
                    BtVS = False # Don't add BtVS if Non-Crossover, unless it's a BtVS/AtS Non-Crossover

-        print("BtVS: %s BtVSNonX: %s"%(BtVS,BtVSNonX))
+        #print("BtVS: %s BtVSNonX: %s"%(BtVS,BtVSNonX))
        if BtVS:
            self.story.addToList('category','Buffy: The Vampire Slayer')
            
@@ -0,0 +1,190 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return TwilightArchivesComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class TwilightArchivesComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL. http://www.twilightarchives.com/read/9353
+        self._setURL('http://' + self.getSiteDomain() + '/read/'+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','twa')
+
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","Twilight")
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%d %b %y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.twilightarchives.com'
+       
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/read/1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://" + self.getSiteDomain()+"/read/")+r"\d+(/d+)?$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('h1')
+        self.story.setMetadata('title',a.string)
+        
+        # Find the chapters:
+        chapters=soup.find('ol', {'class' : 'chapters'})
+        if chapters != None:
+            for chapter in chapters.findAll('a', href=re.compile(r'/read/'+self.story.getMetadata('storyId')+"/\d+$")):
+                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+chapter['href']))
+        else:
+            self.chapterUrls.append((self.story.getMetadata('title'),url))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+                
+# rated, genre, warnings, seires
+
+        summary = soup.find('p', {'class' : 'images'})
+        self.setDescription(url,summary)
+
+        for c in soup.findAll('h2', {'class' : 'title'}):
+            div = c.nextSibling.nextSibling
+            
+            if 'Information' in c.text:
+                for dt in div.findAll('dt'):
+                    dd=dt.nextSibling.nextSibling
+                    
+                    if 'Author' in dt.text:
+                        a=dd.find('a')
+                        self.story.setMetadata('authorId',a['href'].split('/')[2])
+                        self.story.setMetadata('authorUrl','http://'+self.host+a['href'])
+                        self.story.setMetadata('author',a.text)
+                        
+                    if 'Words' in dt.text:
+                        self.story.setMetadata('numWords', dd.text)
+                        
+                    if 'Published' in dt.text:
+                        self.story.setMetadata('datePublished', makeDate(stripHTML(dd.text), self.dateformat))
+            
+                    if 'Updated' in dt.text:
+                        self.story.setMetadata('dateUpdated', makeDate(stripHTML(dd.text), self.dateformat))
+                    
+                    if 'Status' in dt.text:
+                        if 'Complete' in dd.text:
+                            self.story.setMetadata('status', 'Completed')
+                        else:
+                            self.story.setMetadata('status', 'In-Progress')
+
+            if 'Categories' in c.text:
+                for a in div.findAll('a'):
+                    self.story.addToList('category',a.text)
+                    
+            if 'Characters' in c.text:
+                for a in div.findAll('a'):
+                    self.story.addToList('category',a.text)
+
+            if 'Series' in c.text:
+                a=div.find('a')
+                series_name = a.text
+                series_url = 'http://'+self.host+a['href']
+                
+                seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+                storyas = seriessoup.find('tbody').findAll('a', href=re.compile(r'^/read/\d+$'))
+                i=1
+                for a in storyas:
+                    if a['href'] == ('/read/'+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        break
+                    i+=1
+                    
+        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
+        a=asoup.find('tbody').find('a', href=re.compile(r'^/read/'+self.story.getMetadata('storyId')))
+        self.story.setMetadata('rating',a.parent.nextSibling.nextSibling.text)
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'class' : 'size images medium'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -0,0 +1,299 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return WizardTalesNetAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class WizardTalesNetAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','wzt')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%m/%d/%Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.wizardtales.net'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Registered Users Only' in data \
+                or 'There is no such account on our website' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['cookiecheck'] = '1'
+        params['submit'] = 'Submit'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
+        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "Member Account" not in d : #Member Account
+            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=4"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&amp;ageconsent=ok)?&amp;warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid &amp; error in url.
+                addurl = addurl.replace("&amp;","&")
+                url = self.url+'&index=1'+addurl
+                logging.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        pt = soup.find('div', {'id' : 'pagetitle'})
+        a = pt.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.text)
+        
+        # Find authorid and URL from... author url.
+        a = pt.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        rating=pt.text.split('[')[1].split(']')[0]
+        self.story.setMetadata('rating', rating)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        content=soup.find('div',{'class' : 'content'})
+        
+        for genre in content.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')):
+            self.story.addToList('genre',genre.string)
+
+        for warning in content.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')):
+            self.story.addToList('warnings',warning.string)
+        
+        labels = content.findAll('b')
+        
+        value = labels[0].previousSibling
+        svalue = ""
+        while value != None:
+            val = value
+            value = value.previousSibling
+        while "Categories" not in val:
+            svalue += str(val)
+            val = val.nextSibling
+        self.setDescription(url,svalue)
+        
+        
+        
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', stripHTML(value).split(': ')[1].split(';')[0])
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value).split(': ')[1].split(';')[0], self.dateformat))
+            
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value).split(': ')[1], self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            i=1
+            for a in storyas:
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        break
+                    i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.findAll('div', {'id' : 'story'})[1]
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)
@@ -119,10 +119,11 @@ class WraithBaitComAdapter(BaseSiteAdapter):
        self.story.setMetadata('title',a.string)
        
        # Find authorid and URL from... author url.
-        a = pt.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
+        alist = pt.findAll('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        for a in alist:
+            self.story.addToList('authorId',a['href'].split('=')[1])
+            self.story.addToList('authorUrl','http://'+self.host+'/'+a['href'])
+            self.story.addToList('author',a.string)

        rating=pt.text.split('[')[1].split(']')[0]
        self.story.setMetadata('rating', rating)
@@ -215,8 +216,7 @@ class WraithBaitComAdapter(BaseSiteAdapter):

        logging.debug('Getting chapter text from: %s' % url)

-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        soup = bs.BeautifulSoup(self._fetchUrl(url))
        
        div = soup.find('div', {'id' : 'story'})

@@ -37,11 +37,15 @@ def get_urls_from_page(url):
    for a in soup.findAll('a'):
        if a.has_key('href'):
            href = form_url(url,a['href'])
-            # lots of eFiction sites use similar 'are you old enough' javascript links.
-            if 'javascript' in a['href'] and 'viewstory.php' in a['href']:
-                m = re.search(r"'(?P<sid>(view)?story\.php\?(sid|psid|no|story|stid)=\d+)",a['href'])
+            # this (should) catch normal story links, some javascript
+            # 'are you old enough' links, and 'Report This' links.
+            # The 'normalized' set prevents duplicates.
+            if 'story.php' in a['href']:
+                #print("trying:%s"%a['href'])
+                m = re.search(r"(?P<sid>(view)?story\.php\?(sid|psid|no|story|stid)=\d+)",a['href'])
                if m != None:
                    href = form_url(url,m.group('sid'))
+                    
            try:
                href = href.replace('&index=1','')
                adapter = adapters.getAdapter(config,href,"EPUB")
@@ -223,9 +223,9 @@ class Story:
        
    def getMetadata(self, key, removeallentities=False, doreplacements=True):
        value = None
-        if self.getLists().has_key(key):
+        if self.isList(key):
            value = ', '.join(self.getList(key))
-        if self.metadata.has_key(key):
+        elif self.metadata.has_key(key):
            value = self.metadata[key]
            if value:
                if key == "numWords":
@@ -247,11 +247,33 @@ class Story:
        All single value *and* list value metadata as strings.
        '''
        allmetadata = {}
+        
+        # special handling for authors/authorUrls
+        authlinkhtml="<a class='authorlink' href='%s'>%s</a>"
+        if 'author' in self.listables.keys(): # more than one author, assume multiple authorUrl too.
+            htmllist=[]
+            for i, v in enumerate(self.listables['author']):
+                aurl = self.listables['authorUrl'][i]                    
+                auth = v
+                # make sure doreplacements & removeallentities are honored.
+                if doreplacements:
+                    aurl=self.doReplacments(aurl)
+                    auth=self.doReplacments(auth)
+                if removeallentities:
+                    aurl=removeAllEntities(aurl)
+                    auth=removeAllEntities(auth)
+                
+                htmllist.append(authlinkhtml%(aurl,auth))
+            self.setMetadata('authorHTML',', '.join(htmllist))
+        else:
+           self.setMetadata('authorHTML',authlinkhtml%(self.getMetadata('authorUrl', removeallentities, doreplacements),
+                                                       self.getMetadata('author', removeallentities, doreplacements)))
+            
        for k in self.metadata.keys():
            allmetadata[k] = self.getMetadata(k, removeallentities, doreplacements)
        for l in self.listables.keys():
            allmetadata[l] = self.getMetadata(l, removeallentities, doreplacements)
-
+                
        return allmetadata

    # just for less clutter in adapters.
@@ -263,23 +285,36 @@ class Story:
        if value==None:
            return
        value = conditionalRemoveEntities(value)
-        if not self.listables.has_key(listname):
+        if not self.isList(listname):
            self.listables[listname]=[]
        # prevent duplicates.
        if not value in self.listables[listname]:
            self.listables[listname].append(value)

-    def getList(self,listname):
-        if not self.listables.has_key(listname):
-            return []
-        return filter( lambda x : x!=None and x!='' ,
-                       map(self.doReplacments,self.listables[listname]) )
+    def getList(self,listname, removeallentities=False, doreplacements=True):
+        retlist = []
+        if not self.isList(listname):
+            retlist = [self.getMetadata(listname,removeallentities=removeallentities)]
+        else:
+            retlist = self.listables[listname]
+
+        if doreplacements:
+            retlist = filter( lambda x : x!=None and x!='' ,
+                              map(self.doReplacments,retlist) )
+        if removeallentities:
+            retlist = filter( lambda x : x!=None and x!='' ,
+                              map(removeAllEntities,retlist) )
+
+        return retlist

    def getLists(self):
        lsts = {}
        for ln in self.listables.keys():
            lsts[ln] = self.getList(ln)
        return lsts
+
+    def isList(self,listname):
+        return self.listables.has_key(listname)
    
    def addChapter(self, title, html):
        self.chapters.append( (title,html) )
@@ -134,7 +134,7 @@ class BaseStoryWriter(Configurable):
        names as Story.metadata, but ENTRY should use label and value.
        """
        if self.getConfig("include_titlepage"):
-            self._write(out,START.substitute(self.story.metadata))
+            self._write(out,START.substitute(self.story.getAllMetadata()))

            if WIDE_ENTRY==None:
                WIDE_ENTRY=ENTRY
@@ -166,7 +166,7 @@ class BaseStoryWriter(Configurable):
                else:
                    self._write(out, entry)

-            self._write(out,END.substitute(self.story.metadata))
+            self._write(out,END.substitute(self.story.getAllMetadata()))

    def writeTOCPage(self, out, START, ENTRY, END):
        """
@@ -176,13 +176,13 @@ class BaseStoryWriter(Configurable):
        """
        # Only do TOC if there's more than one chapter and it's configured.
        if len(self.story.getChapters()) > 1 and self.getConfig("include_tocpage") and not self.metaonly :
-            self._write(out,START.substitute(self.story.metadata))
+            self._write(out,START.substitute(self.story.getAllMetadata()))

            for index, (title,html) in enumerate(self.story.getChapters()):
                if html:
                    self._write(out,ENTRY.substitute({'chapter':title, 'index':"%04d"%(index+1)}))

-            self._write(out,END.substitute(self.story.metadata))
+            self._write(out,END.substitute(self.story.getAllMetadata()))

    # if no outstream is given, write to file.
    def writeStory(self,outstream=None, metaonly=False, outfilename=None, forceOverwrite=False):
@@ -52,7 +52,7 @@ class EpubWriter(BaseStoryWriter):
 <link href="stylesheet.css" type="text/css" charset="UTF-8" rel="stylesheet"/>
 </head>
 <body>
-<h3><a href="${storyUrl}">${title}</a> by <a href="${authorUrl}">${author}</a></h3>
+<h3><a href="${storyUrl}">${title}</a> by ${authorHTML}</h3>
 <div>
 ''')

@@ -79,7 +79,7 @@ ${value}<br />
 <link href="stylesheet.css" type="text/css" charset="UTF-8" rel="stylesheet"/>
 </head>
 <body>
-<h3><a href="${storyUrl}">${title}</a> by <a href="${authorUrl}">${author}</a></h3>
+<h3><a href="${storyUrl}">${title}</a> by ${authorHTML}</h3>
 <table class="full">
 ''')

@@ -185,7 +185,7 @@ ${value}<br />

        uniqueid= 'fanficdownloader-uid:%s-u%s-s%s' % (
            self.getMetadata('site'),
-            self.getMetadata('authorId'),
+            self.story.getList('authorId')[0],
            self.getMetadata('storyId'))
        
        contentdom = getDOMImplementation().createDocument(None, "package", None)
@@ -206,9 +206,15 @@ ${value}<br />
            metadata.appendChild(newTag(contentdom,"dc:title",text=self.getMetadata('title')))

        if self.getMetadata('author'):
-            metadata.appendChild(newTag(contentdom,"dc:creator",
-                                        attrs={"opf:role":"aut"},
-                                        text=self.getMetadata('author')))
+            if self.story.isList('author'):
+                for auth in self.story.getList('author'):
+                    metadata.appendChild(newTag(contentdom,"dc:creator",
+                                                attrs={"opf:role":"aut"},
+                                                text=auth))
+            else:
+                metadata.appendChild(newTag(contentdom,"dc:creator",
+                                            attrs={"opf:role":"aut"},
+                                            text=self.getMetadata('author')))

        metadata.appendChild(newTag(contentdom,"dc:contributor",text="fanficdownloader [http://fanficdownloader.googlecode.com]",attrs={"opf:role":"bkp"}))
        metadata.appendChild(newTag(contentdom,"dc:rights",text=""))
@@ -433,7 +439,7 @@ div { margin: 0pt; padding: 0pt; }
        del tocncxdom

        # write stylesheet.css file.
-        outputepub.writestr("OEBPS/stylesheet.css",self.EPUB_CSS.substitute(self.story.metadata)) 
+        outputepub.writestr("OEBPS/stylesheet.css",self.EPUB_CSS.substitute(self.story.getAllMetadata())) 

        # write title page.
        if self.getConfig("titlepage_use_table"):
@@ -43,7 +43,7 @@ ${output_css}
 </style>
 </head>
 <body>
-<h1><a href="${storyUrl}">${title}</a> by <a href="${authorUrl}">${author}</a></h1>
+<h1><a href="${storyUrl}">${title}</a> by ${authorHTML}</h1>
 ''')

        self.HTML_TITLE_PAGE_START = string.Template('''
@@ -82,7 +82,7 @@ ${output_css}

    def writeStoryImpl(self, out):

-        self._write(out,self.HTML_FILE_START.substitute(self.story.metadata))
+        self._write(out,self.HTML_FILE_START.substitute(self.story.getAllMetadata()))

        self.writeTitlePage(out,
                            self.HTML_TITLE_PAGE_START,
@@ -100,4 +100,4 @@ ${output_css}
                self._write(out,self.HTML_CHAPTER_START.substitute({'chapter':title, 'index':"%04d"%(index+1)}))
                self._write(out,html)

-        self._write(out,self.HTML_FILE_END.substitute(self.story.metadata))
+        self._write(out,self.HTML_FILE_END.substitute(self.story.getAllMetadata()))
@@ -43,7 +43,7 @@ class MobiWriter(BaseStoryWriter):
 <title>${title} by ${author}</title>
 </head>
 <body>
-<h3><a href="${storyUrl}">${title}</a> by <a href="${authorUrl}">${author}</a></h3>
+<h3><a href="${storyUrl}">${title}</a> by ${authorHTML}</h3>
 <div>
 ''')

@@ -69,7 +69,7 @@ ${value}<br />
 <title>${title} by ${author}</title>
 </head>
 <body>
-<h3><a href="${storyUrl}">${title}</a> by <a href="${authorUrl}">${author}</a></h3>
+<h3><a href="${storyUrl}">${title}</a> by ${authorHTML}</h3>
 <table class="full">
 ''')

@@ -114,7 +114,7 @@ End file.
        
        wrapout = KludgeStringIO()
        
-        wrapout.write(self.TEXT_FILE_START.substitute(self.story.metadata))
+        wrapout.write(self.TEXT_FILE_START.substitute(self.story.getAllMetadata()))

        self.writeTitlePage(wrapout,
                            self.TEXT_TITLE_PAGE_START,
@@ -139,7 +139,7 @@ End file.
                self._write(out,self.lineends(self.wraplines(removeAllEntities(self.TEXT_CHAPTER_START.substitute({'chapter':title, 'index':index+1})))))
                self._write(out,self.lineends(html2text(html,wrap_width=self.wrap_width)))

-        self._write(out,self.lineends(self.wraplines(self.TEXT_FILE_END.substitute(self.story.metadata))))
+        self._write(out,self.lineends(self.wraplines(self.TEXT_FILE_END.substitute(self.story.getAllMetadata()))))

    def wraplines(self, text):
        
@@ -57,23 +57,17 @@
 	  <h3>New Sites</h3>
 	  <p>
 	    
-	    New sites ncisfic.com and national-library.net added.
+	    New sites finestories.com, www.hpfanficarchive.com, svufiction.com, www.twilightarchives.com and www.wizardtales.net.
 	    <br />Thanks, Ida!
 	    
 	  </p>
-	  <h3>New Fixes</h3>
+	  <h3>New Feature</h3>
 	  <p>
-
-	    Fixes to support www.fanfiktion.de and archiveofourown.org username and password.
 	    
-	  </p>
-	  <p>
-
-	    <b>Note:</b> As part of the archiveofourown.org fix, the
-	    configuration section for AO3 is now
-	    [archiveofourown.org], not [www.archiveofourown.org].
-	    Please change your personal configuration accordingly, if
-	    you have customized your settings for archiveofourown.org.
+	    FFDL now supports multiple authors for AO3, TtH and
+	    wraithbait.com.  If you know of other sites we support
+	    that can have more than one author per story, please let
+	    us know.
 	    
 	  </p>
 	  <p>
@@ -84,7 +78,7 @@
 	    If you have any problems with this application, please
 	    report them in
 	    the <a href="http://groups.google.com/group/fanfic-downloader">FanFictionDownLoader Google Group</a>.  The
-	    <a href="http://4-4-17.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
+	    <a href="http://4-4-19.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
 	  </p>
 	  <div id='error'>
 	    {{ error_message }}
@@ -465,6 +459,46 @@
 	    Use the URL of the story's first chapter, such as
 	    <br /><a href="http://national-library.net/viewstory.php?storyid=1234">http://national-library.net/viewstory.php?storyid=1234</a>
 	  </dd>
+	  <dt>dark-solace.org</dt>
+	  <dd>
+	    Use the URL of the story's first chapter, such as
+	    <br /><a href="http://dark-solace.org/elysian/viewstory.php?sid=1234">http://dark-solace.org/elysian/viewstory.php?sid=1234</a>
+	  </dd>
+	  <dt>pretendercentre.com</dt>
+	  <dd>
+	    Use the URL of the story's first chapter, such as
+	    <br /><a href="http://pretendercentre.com/missingpieces/viewstory.php?sid=1234">http://pretendercentre.com/missingpieces/viewstory.php?sid=1234</a>
+	  </dd>
+	  <dt>themasque.net</dt>
+	  <dd>
+	    Use the URL of the story's first chapter, such as
+	    <br /><a href="http://themasque.net/wiktt/efiction/viewstory.php?sid=1234">http://themasque.net/wiktt/efiction/viewstory.php?sid=1234</a>
+	  </dd>
+	  <dt>finestories.com</dt>
+	  <dd>
+	    Use the URL of the story's first chapter, such as
+	    <br /><a href="http://finestories.com/s/1234">http://finestories.com/s/1234</a>
+	  </dd>
+	  <dt>www.hpfanficarchive.com</dt>
+	  <dd>
+	    Use the URL of the story's first chapter, such as
+	    <br /><a href="http://www.hpfanficarchive.com/stories/viewstory.php?sid=1234">http://www.hpfanficarchive.com/stories/viewstory.php?sid=1234</a>
+	  </dd>
+	  <dt>svufiction.com</dt>
+	  <dd>
+	    Use the URL of the story's first chapter, such as
+	    <br /><a href="http://svufiction.com/viewstory.php?sid=1234">http://svufiction.com/viewstory.php?sid=1234</a>
+	  </dd>
+	  <dt>www.twilightarchives.com</dt>
+	  <dd>
+	    Use the URL of the story's first chapter, such as
+	    <br /><a href="http://www.twilightarchives.com/read/1234">http://www.twilightarchives.com/read/1234</a>
+	  </dd>
+	  <dt>www.wizardtales.net</dt>
+	  <dd>
+	    Use the URL of the story's first chapter, such as
+	    <br /><a href="http://www.wizardtales.net/viewstory.php?sid=1234">http://www.wizardtales.net/viewstory.php?sid=1234</a>
+	  </dd>
 	</dl>
 	<p>
 	  A few additional things to know, which will make your life substantially easier:
@@ -343,6 +343,14 @@ extratags: FanFiction,Testing,HTML
 ## cover image.  This lets you exclude them.
 cover_exclusion_regexp:/images/.*?ribbon.gif

+[dark-solace.org]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
 [dramione.org]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -393,6 +401,14 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 #username:YourName
 #password:yourpassword

+[finestories.com]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
 [lumos.sycophanthex.com]
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
@@ -436,6 +452,12 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true

+[pretendercentre.com]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
 [samdean.archive.nu]

 [sg1-heliopolis.com]
@@ -446,8 +468,34 @@ cover_exclusion_regexp:/images/.*?ribbon.gif

 [stargate-atlantis.org]

+[svufiction.com]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
 [thehexfiles.net]

+[themasque.net]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
 [thequidditchpitch.org]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -541,6 +589,8 @@ extratags:
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true

+[www.hpfanficarchive.com]
+
 [www.ik-eternal.net]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -630,6 +680,8 @@ collect_series: false
 #username:YourName
 #password:yourpassword

+[www.twilightarchives.com]
+
 [www.twilighted.net]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -660,6 +712,19 @@ collect_series: false

 [www.whofic.com]

+[www.wizardtales.net]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
 [www.wraithbait.com]
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
Author	SHA1	Message	Date
Jim Miller	b76407ad6b	Bump versions.	2012-07-15 10:42:37 -05:00
Ida	cf13afbfe6	Make finesotries.com adapter accept story detail pages as urls	2012-07-14 18:33:23 -04:00
Jim Miller	9a5fc945a4	Add optional call to 'Count Pages' plugin. Requires 'Count Pages' 1.6.0+	2012-07-14 13:52:54 -05:00
Jim Miller	7fdf893672	Integrate new adapters.	2012-07-13 22:53:59 -05:00
Ida	edf557e7e4	New adapters for finestories.com, hpfanficarchive.com, svufiction.com, twilightarchives.com, and wizardtales.net	2012-07-13 23:22:20 -04:00
Jim Miller	817aef40dd	Yet another change to ffnet metadata.	2012-07-13 15:56:10 -05:00
Jim Miller	9f5a39d713	Keep dc:identifier from getting out of hand.	2012-07-13 13:52:38 -05:00
Jim Miller	624bc609a0	CLI: Use calibre.utils.magick image library when run with calibre-debug -e	2012-07-13 12:12:16 -05:00
Jim Miller	41e6400851	CLI option for DEBUG level logging.	2012-07-13 12:06:16 -05:00
Jim Miller	a2e8bcac49	Add support for multiple authors per story. AO3, TtH and wraithbait so far. Existing check now looks for identifier url first, then title/author match.	2012-07-13 11:37:39 -05:00
Jim Miller	f91ed265ba	Fix for August dates in dramioneorg.	2012-07-12 15:18:19 -05:00
Jim Miller	1b72bbfba9	Change wraithbaitcom to use BeautifulSoup, not Stone for blockquote's in text.	2012-07-12 12:33:09 -05:00
Jim Miller	312b6b2c3e	Tweak fixes to ffn and fanfiktionde.	2012-07-12 09:48:25 -05:00
Jim Miller	fcdd799d10	Automatically try site without "www.".	2012-07-12 09:48:00 -05:00
Jim Miller	615f7b2bc2	Fix ksarchive for poor summary html (copied from twcs).	2012-07-09 15:27:06 -05:00
Jim Miller	f073c2543f	Handle empty category tags in ksarchive.com, more forgiving 'list from url'.	2012-07-09 13:12:21 -05:00
Jim Miller	6c0b31b2ea	Added tag calibre-plugin-1.5.43 for changeset e0b0b635af11	2012-07-08 21:54:17 -05:00
Jim Miller	ddc4d9424a	Added tag FanFictionDownLoader-4.4.19 for changeset e0b0b635af11	2012-07-08 21:53:57 -05:00
Jim Miller	27fca9eb05	Integrate new adapters, bump versions.	2012-07-08 21:53:27 -05:00
Jim Miller	b408cef83f	Fixes for a couple obscure Generate Cover issues.	2012-07-08 21:51:38 -05:00
Ida	e78cf1d7a7	New adapters for dark-solace.org, pretendercentre.com and themasque.net	2012-07-08 17:06:21 -04:00
Jim Miller	050b01509f	Added tag calibre-plugin-1.5.42 for changeset ea845594e14a	2012-07-06 21:13:39 -05:00
Jim Miller	18070b1437	Added tag FanFictionDownLoader-4.4.18b for changeset ea845594e14a	2012-07-06 21:13:29 -05:00