Remove extra code for unchanged author link for calibre 1.0.0, bump version.

Attempted debuging for another Mac issue.
Feature: add_genre_when_multi_category setting
2026-09-15 11:36:30 +08:00 · 2013-08-31 19:50:18 -05:00 · 2013-08-31 19:42:58 -05:00 · 2013-08-31 15:38:44 -05:00 · 2013-08-29 19:23:23 -05:00 · 2013-08-27 17:11:43 -05:00
103 changed files with 461 additions and 285 deletions
@@ -1,6 +1,6 @@
 # ffd-retief-hrd fanfictiondownloader
 application: fanfictiondownloader
-version: 4-4-65
+version: 4-4-68
 runtime: python27
 api_version: 1
 threadsafe: true
@@ -26,7 +26,7 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
    description         = 'UI plugin to download FanFiction stories from various sites.'
    supported_platforms = ['windows', 'osx', 'linux']
    author              = 'Jim Miller'
-    version             = (1, 7, 31)
+    version             = (1, 7, 39)
    minimum_calibre_version = (0, 8, 57)

    #: This field defines the GUI plugin class that contains all the code
@@ -169,6 +169,8 @@ class ConfigWidget(QWidget):
        prefs['updatecover'] = self.basic_tab.updatecover.isChecked()
        prefs['updateepubcover'] = self.basic_tab.updateepubcover.isChecked()
        prefs['keeptags'] = self.basic_tab.keeptags.isChecked()
+        prefs['suppressauthorsort'] = self.basic_tab.suppressauthorsort.isChecked()
+        prefs['suppresstitlesort'] = self.basic_tab.suppresstitlesort.isChecked()
        prefs['showmarked'] = self.basic_tab.showmarked.isChecked()
        prefs['urlsfromclip'] = self.basic_tab.urlsfromclip.isChecked()
        prefs['updatedefault'] = self.basic_tab.updatedefault.isChecked()
@@ -177,6 +179,7 @@ class ConfigWidget(QWidget):
        prefs['includeimages'] = self.basic_tab.includeimages.isChecked()
        prefs['lookforurlinhtml'] = self.basic_tab.lookforurlinhtml.isChecked()
        prefs['checkforseriesurlid'] = self.basic_tab.checkforseriesurlid.isChecked()
+        prefs['checkforurlchange'] = self.basic_tab.checkforurlchange.isChecked()
        prefs['injectseries'] = self.basic_tab.injectseries.isChecked()
        prefs['smarten_punctuation'] = self.basic_tab.smarten_punctuation.isChecked()

@@ -322,10 +325,10 @@ class BasicTab(QWidget):
        self.updateepubcover.setChecked(prefs['updateepubcover'])
        self.l.addWidget(self.updateepubcover)

+        self.smarten_punctuation = QCheckBox('Smarten Punctuation (EPUB only)',self)
+        self.smarten_punctuation.setToolTip("Run Smarten Punctuation from Calibre's Polish Book feature on each EPUB download and update.")
+        self.smarten_punctuation.setChecked(prefs['smarten_punctuation'])
        if calibre_version >= (0, 9, 39):
-            self.smarten_punctuation = QCheckBox('Smarten Punctuation (EPUB only)',self)
-            self.smarten_punctuation.setToolTip("Run Smarten Punctuation from Calibre's Polish Book feature on each EPUB download and update.")
-            self.smarten_punctuation.setChecked(prefs['smarten_punctuation'])
            self.l.addWidget(self.smarten_punctuation)

        cali_gb = groupbox = QGroupBox("Updating Calibre Options")
@@ -347,11 +350,26 @@ class BasicTab(QWidget):
        self.keeptags.setChecked(prefs['keeptags'])
        self.l.addWidget(self.keeptags)

+        self.suppressauthorsort = QCheckBox('Force Author into Author Sort?',self)
+        self.suppressauthorsort.setToolTip("If checked, the author(s) as given will be used for the Author Sort, too.\nIf not checked, calibre will apply it's built in algorithm which makes 'Bob Smith' sort as 'Smith, Bob', etc.")
+        self.suppressauthorsort.setChecked(prefs['suppressauthorsort'])
+        self.l.addWidget(self.suppressauthorsort)
+
+        self.suppresstitlesort = QCheckBox('Force Title into Title Sort?',self)
+        self.suppresstitlesort.setToolTip("If checked, the title as given will be used for the Title Sort, too.\nIf not checked, calibre will apply it's built in algorithm which makes 'The Title' sort as 'Title, The', etc.")
+        self.suppresstitlesort.setChecked(prefs['suppresstitlesort'])
+        self.l.addWidget(self.suppresstitlesort)
+
        self.checkforseriesurlid = QCheckBox("Check for existing Series Anthology books?",self)
        self.checkforseriesurlid.setToolTip("Check for existings Series Anthology books using each new story's series URL before downloading.\nOffer to skip downloading if a Series Anthology is found.")
        self.checkforseriesurlid.setChecked(prefs['checkforseriesurlid'])
        self.l.addWidget(self.checkforseriesurlid)

+        self.checkforurlchange = QCheckBox("Check for changed Story URL?",self)
+        self.checkforurlchange.setToolTip("Warn you if an update will change the URL of an existing book.")
+        self.checkforurlchange.setChecked(prefs['checkforurlchange'])
+        self.l.addWidget(self.checkforurlchange)
+
        self.lookforurlinhtml = QCheckBox("Search EPUB text for Story URL?",self)
        self.lookforurlinhtml.setToolTip("Look for first valid story URL inside EPUB text if not found in metadata.\nSomewhat risky, could find wrong URL depending on EPUB content.\nAlso finds and corrects bad ffnet URLs from ficsaver.com files.")
        self.lookforurlinhtml.setChecked(prefs['lookforurlinhtml'])
@@ -503,7 +521,7 @@ class PersonalIniTab(QWidget):
        self.ini = QTextEdit(self)
        try:
            self.ini.setFont(QFont("Courier",
-                                   self.plugin_action.gui.font().pointSize()+1));
+                                   self.plugin_action.gui.font().pointSize()+1))
        except Exception as e:
            print("Couldn't get font: %s"%e)
        self.ini.setLineWrapMode(QTextEdit.NoWrap)
@@ -539,7 +557,7 @@ class ShowDefaultsIniDialog(QDialog):
        self.ini.setToolTip("These are all of the plugin's configurable options\nand their default settings.")
        try:
            self.ini.setFont(QFont("Courier",
-                                   get_gui().font().pointSize()+1));
+                                   get_gui().font().pointSize()+1))
        except Exception as e:
            print("Couldn't get font: %s"%e)
        self.ini.setLineWrapMode(QTextEdit.NoWrap)
@@ -164,7 +164,7 @@ class DroppableQTextEdit(QTextEdit):
        
    def canInsertFromMimeData(self, source):
        if source.hasUrls():
-            return True;
+            return True
        else:
            return QTextEdit.canInsertFromMimeData(self,source)

@@ -208,7 +208,7 @@ class AddNewDialog(SizePersistedDialog):
        self.groupbox.setChecked(False)
        self.groupbox.setFlat(True)
        print("style:%s"%self.groupbox.styleSheet())
-        self.groupbox.setStyleSheet(gpstyle);
+        self.groupbox.setStyleSheet(gpstyle)

        self.gbf = QFrame()
        self.gbl = QVBoxLayout()
@@ -7,13 +7,14 @@ __license__   = 'GPL v3'
 __copyright__ = '2012, Jim Miller'
 __docformat__ = 'restructuredtext en'

-import time, os, copy, threading, re, platform
+import time, os, copy, threading, re, platform, sys
 from StringIO import StringIO
 from functools import partial
 from datetime import datetime
 from string import Template
 import urllib
 import email
+import traceback

 from PyQt4.Qt import (QApplication, QMenu, QToolButton, QTimer)

@@ -23,7 +24,7 @@ from PyQt4.QtCore import QBuffer
 from calibre.constants import numeric_version as calibre_version

 from calibre.ptempfile import PersistentTemporaryFile, PersistentTemporaryDirectory, remove_dir
-from calibre.ebooks.metadata import MetaInformation, authors_to_string
+from calibre.ebooks.metadata import MetaInformation
 from calibre.ebooks.metadata.meta import get_metadata
 from calibre.gui2 import error_dialog, warning_dialog, question_dialog, info_dialog
 from calibre.gui2.dialogs.message_box import ViewLog
@@ -628,7 +629,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
            return

        if not id_list:
-            id_list = self.gui.library_view.get_selected_ids();
+            id_list = self.gui.library_view.get_selected_ids()
        
        if len(id_list) == 0:
            self.gui.status_bar.show_message(_('No Selected Books to Update'), 3000)
@@ -918,7 +919,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
    
                if book_id and mi: # book_id and mi only set if matched by title/author.
                    liburl = self.get_story_url(db,book_id)
-                    if book['url'] != liburl:
+                    if book['url'] != liburl and prefs['checkforurlchange']:
                        if collision in (OVERWRITE,OVERWRITEALWAYS):
                            updat="overwrit"
                        else:
@@ -976,6 +977,8 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                                skip_date_update = True
                        elif chaptercount > urlchaptercount:
                            raise NotGoingToDownload("Existing epub contains %d chapters, web site only has %d. Use Overwrite to force update." % (chaptercount,urlchaptercount),'dialog_error.png')
+                        elif chaptercount == 0:
+                            raise NotGoingToDownload("FFDL doesn't recognize chapters in existing epub, epub is probably from a different source. Use Overwrite to force update.",'dialog_error.png')
        
                if collision == OVERWRITE and \
                        db.has_format(book_id,formmapping[fileform],index_is_id=True):
@@ -1001,7 +1004,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                mi = db.get_metadata(book_id,index_is_id=True)
                if not book['series'] and mi.series != None:
                    book['calibre_series'] = (mi.series,mi.series_index)
-                    print("calibre_series:%s [%s]"%book['calibre_series'])
+                    #print("calibre_series:%s [%s]"%book['calibre_series'])
                
        if book['good']: # there shouldn't be any !'good' books at this point.
            # if still 'good', make a temp file to write the output to.
@@ -1106,7 +1109,17 @@ class FanFictionDownLoaderPlugin(InterfaceAction):

        if options['collision'] == CALIBREONLY or \
                ( (options['updatemeta'] or book['added']) and book['good'] ):
-            self.update_metadata(db, book['calibre_id'], book, mi, options)
+            try:
+                self.update_metadata(db, book['calibre_id'], book, mi, options)
+            except:
+                det_msg = "".join(traceback.format_exception(*sys.exc_info()))+"\nStory Details:\n%s"%pretty_book(book)
+                print("Error Updating Metadata:\n%s"%det_msg)
+                error_dialog(self.gui,
+                             "Error Updating Metadata",
+                             "<p>An error has occurred while FFDL was updating calibre's metadata for <a href='%s'>%s</a>.</p>"%(book['url'],book['title'])+
+                             "The ebook has been updated, but the metadata has not.",
+                             det_msg=det_msg,
+                             show=True)

    def update_books_finish(self, book_list, options={}, showlist=True):
        '''Notify calibre about updated rows, update external plugins
@@ -1157,10 +1170,20 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                self.gui.library_view.sort_by_named_field('marked', True)
                
        self.gui.status_bar.show_message(_('Finished Adding/Updating %d books.'%(len(update_list) + len(add_list))), 3000)
-            
+
+        1+1
+        
        print("all done, remove temp dir.")
+
+        1+1
+        
        remove_dir(options['tdir'])

+        1+1
+        
+        print("removed temp dir.")
+
+        
        if 'Count Pages' in self.gui.iactions and len(prefs['countpagesstats']) and len(all_ids):
            cp_plugin = self.gui.iactions['Count Pages']
            cp_plugin.count_statistics(all_ids,prefs['countpagesstats'])
@@ -1368,14 +1391,23 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
        oldmi = db.get_metadata(book_id,index_is_id=True)
        if prefs['keeptags']:
            old_tags = db.get_tags(book_id)
+            #print("old_tags:%s"%old_tags)
+            #print("mi.tags:%s"%mi.tags)
            # remove old Completed/In-Progress only if there's a new one.
            if 'Completed' in mi.tags or 'In-Progress' in mi.tags:
                old_tags = filter( lambda x : x not in ('Completed', 'In-Progress'), old_tags)
                # remove old Last Update tags if there are new ones.
            if len(filter( lambda x : not x.startswith("Last Update"), mi.tags)) > 0:
                old_tags = filter( lambda x : not x.startswith("Last Update"), old_tags)
+                
            # mi.tags needs to be list, but set kills dups.
-            mi.tags = list(set(list(old_tags)+mi.tags))
+            # this way also removes case-mismatched dups, keeping old_tags version.
+            foldedcase_tags = dict()
+            for t in list(mi.tags) + list(old_tags):
+                foldedcase_tags[t.lower()] = t
+
+            mi.tags = foldedcase_tags.values()   
+            #print("mi.tags:%s"%mi.tags)

        if book['all_metadata']['langcode']:
            mi.languages=[book['all_metadata']['langcode']]
@@ -1408,6 +1440,8 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                            pass

        db.set_metadata(book_id,mi)
+        # mi.authors gets run through the string_to_authors and split on '&' ',' 'and' and 'with'
+        db.set_authors(book_id,book['author']) # author is a list.

        # do configured column updates here.
        #print("all_metadata: %s"%book['all_metadata'])
@@ -1432,7 +1466,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                continue
            label = coldef['label']
            if coldef['datatype'] in ('enumeration','text','comments','datetime','series'):
-                db.set_custom(book_id, book['all_metadata'][meta], label=label, commit=False)
+                db.set_custom(book_id, book['all_metadata'][meta], label, commit=False)
            elif coldef['datatype'] in ('int','float'):
                num = unicode(book['all_metadata'][meta]).replace(",","")
                db.set_custom(book_id, num, label=label, commit=False)
@@ -1455,14 +1489,11 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                    if "," in custcol:
                        (custcol,flag) = map( lambda x: x.strip(), custcol.split(",") )

-                    print("meta:(%s) => custcol:(%s), flag(%s) "%(meta,custcol,flag))
-                    
                    if meta not in book['all_metadata']:
                        print("No value for %s, skipping custom column(%s) update."%(meta,custcol))
                        continue
                    
                    if custcol not in custom_columns:
-                        print("No custom column(%s), skipping."%(custcol))
                        continue
                    else:
                        coldef = custom_columns[custcol]
@@ -1499,12 +1530,26 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
        # Moved down so author's already in the DB.
        if 'authorUrl' in book['all_metadata']:
            authurls = book['all_metadata']['authorUrl'].split(", ")
-            for i, auth in enumerate(book['author']):
-                #print("===Update author url for %s to %s"%(auth,authurls[i]))
-                autid=db.get_author_id(auth)
-                db.set_link_field_for_author(autid, unicode(authurls[i]),
-                                             commit=False, notify=False)
+            if hasattr(db, 'new_api'): # new_api starts in calibre 1.0.0
+                authorlist = [ a.replace('&',';') for a in book['author'] ]
+                authorids = db.new_api.get_item_ids('authors',authorlist)
+                authordata = db.new_api.author_data(authorids.values())
+                # print("\n\nauthorids:%s"%authorids)
+                # print("authordata:%s"%authordata)

+                author_id_to_link_map = dict()
+                for i, author in enumerate(authorlist):
+                    author_id_to_link_map[authorids[author]] = authurls[i]
+
+                # print("author_id_to_link_map:%s\n\n"%author_id_to_link_map)
+                db.new_api.set_link_for_authors(author_id_to_link_map)
+            else:
+                # keep for pre-calibre 1.0.0
+                for i, auth in enumerate(authorlist):
+                    #print("===Update author url for %s to %s"%(auth,authurls[i]))
+                    autid=db.get_author_id(auth)
+                    db.set_link_field_for_author(autid, unicode(authurls[i]),
+                                             commit=False, notify=False)
        db.commit()

        if 'Generate Cover' in self.gui.iactions and (book['added'] or not prefs['gcnewonly']):
@@ -1604,6 +1649,14 @@ class FanFictionDownLoaderPlugin(InterfaceAction):

    def make_mi_from_book(self,book):
        mi = MetaInformation(book['title'],book['author']) # author is a list.
+        if prefs['suppressauthorsort']:
+            # otherwise author names will have calibre's sort algs
+            # applied automatically.
+            mi.author_sort = ' & '.join(book['author'])
+        if prefs['suppresstitlesort']:
+            # otherwise titles will have calibre's sort algs applied
+            # automatically.
+            mi.title_sort = book['title']
        mi.set_identifiers({'url':book['url']})
        mi.publisher = book['publisher']
        mi.tags = book['tags']
@@ -1850,7 +1903,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                book['title'] = series
                for sr in serieslist:
                    if series != sr:
-                        book['title'] = deftitle;
+                        book['title'] = deftitle
                        break
            
            configuration = get_ffdl_config(book['url'],fileform)
@@ -1889,3 +1942,17 @@ def split_text_to_urls(urls):
 def escapehtml(txt):
    return txt.replace("&","&amp;").replace(">","&gt;").replace("<","&lt;")
        
+def pretty_book(d, indent=0, spacer='     '):
+    kindent = spacer * indent
+
+    # if isinstance(d, list):
+    #     return '\n'.join([(pretty_book(v, indent, spacer)) for v in d])
+
+    if isinstance(d, dict):
+        for k in ('password','username'):
+            if k in d and d[k]:
+                d[k]='<was set, removed for security>'
+        return '\n'.join(['%s%s:\n%s' % (kindent, k, pretty_book(v, indent + 1, spacer)) 
+                          for k, v in d.items()])
+    return "%s%s"%(kindent, d)
+
@@ -30,6 +30,8 @@ default_prefs['updatemeta'] = True
 default_prefs['updatecover'] = False
 default_prefs['updateepubcover'] = False
 default_prefs['keeptags'] = False
+default_prefs['suppressauthorsort'] = False
+default_prefs['suppresstitlesort'] = False
 default_prefs['showmarked'] = False
 default_prefs['urlsfromclip'] = True
 default_prefs['updatedefault'] = True
@@ -40,6 +42,7 @@ default_prefs['adddialogstaysontop'] = False
 default_prefs['includeimages'] = False
 default_prefs['lookforurlinhtml'] = False
 default_prefs['checkforseriesurlid'] = True
+default_prefs['checkforurlchange'] = True
 default_prefs['injectseries'] = False
 default_prefs['smarten_punctuation'] = False

@@ -152,6 +152,9 @@ extratags: FanFiction
 #extraships:
 #extrawarnings:

+## Add this to genre if there's more than one category.
+#add_genre_when_multi_category: Crossover
+
 ## number of seconds to sleep between calls to the story site.  May by
 ## useful if pulling large numbers of stories or if the site is slow.
 #slow_down_sleep_time:0.5
@@ -1355,6 +1358,11 @@ titlepage_entries: series,seriesUrl,category,genre,language,status,datePublished
 ## Site dedicated to these categories/characters/ships
 extracategories:Lord of the Rings

+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
 [www.thepetulantpoetess.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -1476,7 +1484,8 @@ extracategories:Stargate: Atlantis
 [teststory:defaults]
 valid_entries:title,author_list,authorId_list,authorUrl_list,storyUrl,
 datePublished,dateUpdated,numWords,status,language,series,seriesUrl,
- rating,category_list,genre_list,warnings_list,characters_list
+ rating,category_list,genre_list,warnings_list,characters_list,ships_list,
+ description,site,extratags

 # {{storyId}} is a special case--it's the only one that works.
 title:Test Story Title {{storyId}}
@@ -137,6 +137,9 @@ def main(argv,
   if options.update:
       try:
           (url,chaptercount) = get_dcsource_chaptercount(args[0])
+           if not url:
+               print "No story URL found in epub to update."
+               return
           print "Updating %s, URL: %s" % (args[0],url)
           output_filename = args[0]
       except:
@@ -145,8 +148,15 @@ def main(argv,
           url = args[0]
   else:
       url = args[0]
-           
-   configuration = Configuration(adapters.getConfigSectionFor(url),options.format)
+
+   try:
+       configuration = Configuration(adapters.getConfigSectionFor(url),options.format)
+   except exceptions.UnknownSite, e:
+       if options.list or options.normalize:
+           # list for page doesn't have to be a supported site.
+           configuration = Configuration("test1.com",options.format)
+       else:
+           raise e

   conflist = []
   homepath = join(expanduser("~"),".fanficdownloader")
@@ -199,8 +209,7 @@ def main(argv,

   if options.list or options.normalize:
       retlist = get_urls_from_page(args[0], configuration, normalize=options.normalize)
-       print "\n".join(retlist)
-               
+       print "\n".join(retlist)               
       return

   try:
@@ -258,6 +267,8 @@ def main(argv,
               print "%s already contains %d chapters." % (output_filename,chaptercount)
           elif chaptercount > urlchaptercount:
               print "%s contains %d chapters, more than source: %d." % (output_filename,chaptercount,urlchaptercount)
+           elif chaptercount == 0:
+               print "%s doesn't contain any recognizable chapters, probably from a different source.  Not updating." % (output_filename)
           else:
               print "Do update - epub(%d) vs url(%d)" % (chaptercount, urlchaptercount)
               if not options.metaonly:
@@ -126,6 +126,7 @@ import adapter_scarheadnet
    
 ## List of registered site adapters.
 __class_list = []
+__domain_map = {}

 def imports():
    for name, val in globals().items():
@@ -135,7 +136,10 @@ def imports():
 for x in imports():
    if "fanficdownloader.adapters.adapter_" in x:
        #print x
-        __class_list.append(sys.modules[x].getClass())
+        cls = sys.modules[x].getClass()
+        __class_list.append(cls)
+        for site in cls.getAcceptDomains():
+            __domain_map[site]=cls

 def getNormalStoryURL(url):
    if not getNormalStoryURL.__dummyconfig:
@@ -153,12 +157,14 @@ def getNormalStoryURL(url):
 # kludgey function static/singleton
 getNormalStoryURL.__dummyconfig = None

-def getAdapter(config,url):
+def getAdapter(config,url,anyurl=False):

    #logger.debug("trying url:"+url)
    (cls,fixedurl) = getClassFor(url)
    #logger.debug("fixedurl:"+fixedurl)
    if cls:
+        if anyurl:
+            fixedurl = cls.getSiteExampleURLs().split()[0]
        adapter = cls(config,fixedurl) # raises InvalidStoryURL
        return adapter
    # No adapter found.
@@ -204,12 +210,14 @@ def getClassFor(url):
        #logger.debug("trying site:www."+domain)
        cls = getClassFromList("www."+domain)
        fixedurl = fixedurl.replace("http://","http://www.")
-        
-    fixedurl = cls.stripURLParameters(fixedurl)
+
+    if cls:
+        fixedurl = cls.stripURLParameters(fixedurl)
    
    return (cls,fixedurl)
    
 def getClassFromList(domain):
-    for cls in __class_list:
-        if cls.matchesSite(domain):
-            return cls
+    try:
+        return __domain_map[domain]
+    except KeyError:
+        pass # return none.
@@ -42,7 +42,7 @@ class AdAstraFanficComSiteAdapter(BaseSiteAdapter):
        
        # get storyId from url--url validation guarantees query is only sid=1234
        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
        
        # normalized story URL.
        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -90,7 +90,7 @@ class AdAstraFanficComSiteAdapter(BaseSiteAdapter):

        ## Title
        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = soup.find('a', href=re.compile(r"viewuser.php"))
@@ -51,13 +51,13 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
        
        # get storyId from url--url validation guarantees query is only sid=1234
        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
        
        # get storyId from url--url validation guarantees query correct
        m = re.match(self.getSiteURLPattern(),url)
        if m:
            self.story.setMetadata('storyId',m.group('id'))
-            logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+            
            # normalized story URL.
            self._setURL('http://' + self.getSiteDomain() + '/works/'+self.story.getMetadata('storyId'))
        else:
@@ -163,7 +163,7 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
        
        ## Title
        a = soup.find('a', href=re.compile(r"^/works/\d+$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
 		
        # Find authorid and URL from... author url.
        alist = soup.findAll('a', href=re.compile(r"^/users/\w+/pseuds/\w+"))
@@ -49,7 +49,7 @@ class ArchiveSkyeHawkeComAdapter(BaseSiteAdapter):
        
        # get storyId from url--url validation guarantees query is only sid=1234
        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
        
        # normalized story URL.
        self._setURL('http://' + self.getSiteDomain() + '/story.php?no='+self.story.getMetadata('storyId'))
@@ -101,7 +101,7 @@ class ArchiveSkyeHawkeComAdapter(BaseSiteAdapter):
        
        ## Title
        a = soup.find('div', {'class':"story border"}).find('span',{'class':'left'})
-        title=a.text.split('"')[1]
+        title=stripHTML(a).split('"')[1]
        self.story.setMetadata('title',title)
        
        # Find authorid and URL from... author url.
@@ -48,7 +48,7 @@ class AshwinderSycophantHexComAdapter(BaseSiteAdapter):
        
        # get storyId from url--url validation guarantees query is only sid=1234
        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
 		
        
        # normalized story URL.
@@ -157,7 +157,7 @@ class AshwinderSycophantHexComAdapter(BaseSiteAdapter):
        for info in asoup.findAll('table', {'width' : '100%', 'bordercolor' : re.compile(r'#')}):
            a = info.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
            if a != None:
-                self.story.setMetadata('title',a.string)
+                self.story.setMetadata('title',stripHTML(a))
                break
 		

@@ -46,7 +46,7 @@ class Asr3SlashzoneOrgAdapter(BaseSiteAdapter):
        
        # get storyId from url--url validation guarantees query is only sid=1234
        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
        
        # normalized story URL.
        self._setURL('http://' + self.getSiteDomain() + '/archive/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -128,7 +128,7 @@ class Asr3SlashzoneOrgAdapter(BaseSiteAdapter):
        
        ## Title
        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -70,7 +70,7 @@ class BloodTiesFansComAdapter(BaseSiteAdapter): # XXX
        
        # get storyId from url--url validation guarantees query is only sid=1234
        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
        
        # normalized story URL.
        # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
@@ -207,7 +207,7 @@ class BloodTiesFansComAdapter(BaseSiteAdapter): # XXX
        
        ## Title
        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -58,7 +58,7 @@ class BuffyNFaithNetAdapter(BaseSiteAdapter):
        m = re.match(self.getSiteURLPattern(),url)
        if m:
            self.story.setMetadata('storyId',m.group('id'))
-            logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+            
            # normalized story URL. gets rid of chapter if there, left with ch 1 URL on this site
            nurl = "http://"+self.getSiteDomain()+"/fanfictions/index.php?act=vie&id="+self.story.getMetadata('storyId')
            self._setURL(nurl)
@@ -69,7 +69,7 @@ class BuffyNFaithNetAdapter(BaseSiteAdapter):
            raise exceptions.InvalidStoryURL(url,
                                             self.getSiteDomain(),
                                             self.getSiteExampleURLs())
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
        
        # Each adapter needs to have a unique site abbreviation.
        self.story.setMetadata('siteabbrev','bnfnet')
@@ -72,7 +72,7 @@ class CastleFansOrgAdapter(BaseSiteAdapter): # XXX
        
        # get storyId from url--url validation guarantees query is only sid=1234
        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
        
        # normalized story URL.
        # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
@@ -180,7 +180,7 @@ class CastleFansOrgAdapter(BaseSiteAdapter): # XXX
        pagetitle = soup.find('div',{'id':'pagetitle'})
        ## Title
        a = pagetitle.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = pagetitle.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -48,7 +48,7 @@ class ChaosSycophantHexComAdapter(BaseSiteAdapter):
        
        # get storyId from url--url validation guarantees query is only sid=1234
        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
 		
        
        # normalized story URL.
@@ -116,7 +116,7 @@ class ChaosSycophantHexComAdapter(BaseSiteAdapter):
        ## Title
        pt = soup.find('div', {'id' : 'pagetitle'})
        a = pt.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.text)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = pt.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -48,7 +48,7 @@ class CheckmatedComAdapter(BaseSiteAdapter):

        # get storyId from url--url validation guarantees query is only sid=1234
        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        

        self._setURL('http://' + self.getSiteDomain() + '/story.php?story='+self.story.getMetadata('storyId'))

@@ -143,11 +143,11 @@ class CheckmatedComAdapter(BaseSiteAdapter):
        # Now go hunting for all the meta data and the chapter list.

        ## Title
-        a = soup.findAll('span', {'class' : 'storytitle'})
-        self.story.setMetadata('title',a[0].string)
+        a = soup.find('span', {'class' : 'storytitle'})
+        self.story.setMetadata('title',stripHTML(a))

        # Find authorid and URL from... author url.
-        a = a[1].find('a', href=re.compile(r"authors.php\?name\=\w+"))
+        a = a.parent.find('a', href=re.compile(r"authors.php\?name\=\w+"))
        self.story.setMetadata('authorId',a['href'].split('=')[1])
        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
        self.story.setMetadata('author',a.string)
@@ -48,7 +48,7 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
        
        # get storyId from url--url validation guarantees query is only sid=1234
        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
        
        # normalized story URL.
        self._setURL('http://' + self.getSiteDomain() + '/elysian/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -184,31 +184,31 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
        # first a tag in pagetitle is title
        self.story.setMetadata('title',stripHTML(div.find('a')))

-        # Find the chapters:
-        # chapters=soup.find('select', {'name' : 'chapter'})
-        # if chapters != None:
-        #     for chapter in chapters.findAll('option'):
-        #         self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/elysian/viewstory.php?sid='+self.story.getMetadata('storyId')+'&chapter='+chapter['value']))
-        # else:
-        #     self.chapterUrls.append((self.story.getMetadata('title'),url+"&chapter=1"))
-
        for chapa in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+
                                                       self.story.getMetadata('storyId')+'&chapter=\d+')):
            self.chapterUrls.append((stripHTML(chapa),'http://'+self.host+'/elysian/'+chapa['href']))
        
        self.story.setMetadata('numChapters',len(self.chapterUrls))
-        
+
        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
+        storylink = asoup.find('a', href=re.compile(r'viewstory.php\?sid='+
+                                                    self.story.getMetadata('storyId')+'($|[^\d])'))
+        # author's story list is paginated if there's a pagelinks div.
+        # Only need to look in it if the story wasn't on the first page.
+        pagelinks = asoup.find('div',{'id':'pagelinks'})
+        if pagelinks and storylink==None:
+            authpageslist = pagelinks.findAll('a',href=re.compile(r'action=storiesby'))
+            for page in authpageslist[1:]: # skip first, already checked above.
+                asoup = bs.BeautifulSoup(self._fetchUrl('http://'+self.host+'/elysian/'+page['href']))
+                storylink = asoup.find('a', href=re.compile(r'viewstory.php\?sid='+
+                                                            self.story.getMetadata('storyId')+'($|[^\d])'))
+                if storylink:
+                    break
+
+        if not storylink:
+            raise exceptions.FailedToDownload("Unable to find story metadata on author's page(s)")
        
-        # for metalist in asoup.findAll('div', {'class' : re.compile('listbox\s+')}):
-        #     a = metalist.find('a', href=re.compile(r'viewstory.php\?sid='))
-        #     if a != None:
-        #         if 'viewstory.php?sid='+self.story.getMetadata('storyId') in a['href']:
-        #             break
-
-        metalist = asoup.find('a', href=re.compile(r'viewstory.php\?sid='+
-                                                   self.story.getMetadata('storyId')+'($|[^\d])')).parent.parent
-
+        metalist = storylink.parent.parent
        # eFiction sites don't help us out a lot with their meta data
        # formating, so it's a little ugly.

@@ -229,7 +229,7 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
            if 'Summary' in label:
                ## Everything until the next span class='label'
                svalue = ""
-                while not (defaultGetattr(value,'class') == 'label' or "Chapters: " in stripHTML(value)):
+                while value and not (defaultGetattr(value,'class') == 'label' or "Chapters: " in stripHTML(value)):
                    svalue += str(value)
                    value = value.nextSibling
                self.setDescription(url,svalue)
@@ -284,20 +284,40 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
            # use BeautifulSoup HTML parser to make everything easier to find.
            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
-            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
-            i=1
-            for a in storyas:
-                # skip 'report this' and 'TOC' links
-                if 'contact.php' not in a['href'] and 'index' not in a['href']:
-                    if ('viewstory.php?sid='+self.story.getMetadata('storyId')) in a['href']:
-                        self.setSeries(series_name, i)
-                        self.story.setMetadata('seriesUrl',series_url)
+            storylink = seriessoup.find('a', href=re.compile(r'viewstory.php\?sid='+
+                                                             self.story.getMetadata('storyId')+'($|[^\d])'))
+            if storylink and storylink.parent and storylink.parent['class'] != 'title': # in case of links inside story summaries.
+                storylink = None
+
+            offset = 0
+            # series story list is paginated if there's a pagelinks div.
+            # Only need to look in it if the story wasn't on the first page.
+            pagelinks = seriessoup.find('div',{'id':'pagelinks'})
+            if pagelinks and storylink==None:
+                authpageslist = pagelinks.findAll('a',href=re.compile(r'offset='))
+                for page in authpageslist[1:]: # skip first, already checked above.
+                    seriessoup = bs.BeautifulSoup(self._fetchUrl('http://'+self.host+'/elysian/'+page['href']))
+                    storylink = seriessoup.find('a', href=re.compile(r'viewstory.php\?sid='+
+                                                                     self.story.getMetadata('storyId')+'($|[^\d])'))
+                    if storylink and storylink.parent and storylink.parent['class'] != 'title': # in case of links inside story summaries.
+                        storylink = None
+                    if storylink:
+                        offset = int(page['href'].split('=')[-1]) # offset is last.
                        break
-                    i+=1
+
+            # for reasons I don't understand, searching for story
+            # links by regex wasn't working reliably.  It was missing
+            # the javascript links sometimes.  This is cleaner anyway.
+            for i, div in enumerate(seriessoup.findAll('div', {'class':'title'})):
+                a = div.find('a') # first a is story link.
+                # skip 'report this' and 'TOC' links
+                if a == storylink:
+                    self.setSeries(series_name, 1+i+offset)
+                    self.story.setMetadata('seriesUrl',series_url)
+                    break
            
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
+        except Exception, e:
+            print("Series parsing failed: %s"%e)
            
    # grab the text for an individual chapter.
    def getChapterText(self, url):
@@ -48,7 +48,7 @@ class DestinysGatewayComAdapter(BaseSiteAdapter):
        
        # get storyId from url--url validation guarantees query is only sid=1234
        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
        
        # normalized story URL.
        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -131,7 +131,7 @@ class DestinysGatewayComAdapter(BaseSiteAdapter):
        
        ## Title
        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -48,7 +48,7 @@ class DokugaComAdapter(BaseSiteAdapter):
        
        # get storyId from url--url validation guarantees query is only sid=1234
        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[3])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        

        # www.dokuga.com has two 'sections', shown in URL as
        # 'fanfiction' and 'spark' that change how things should be
@@ -164,7 +164,7 @@ class DokugaComAdapter(BaseSiteAdapter):
        aut.extract()
        
        a = a.string[:(len(a.string)-4)]
-        self.story.setMetadata('title',a)
+        self.story.setMetadata('title',stripHTML(a))

        # Find the chapters:
        chapters = soup.find('select').findAll('option')
@@ -205,7 +205,6 @@ class DokugaComAdapter(BaseSiteAdapter):
            
            a=div.text.split('Category: ')
            if len(a) == 2: self.story.addToList('category', a[1].split('&')[0])
-            self.story.addToList('category', 'Fanfiction')
        
            a=div.text.split('Created: ')
            if len(a) == 2: self.story.setMetadata('datePublished', makeDate(stripHTML(a[1].split('&')[0]), self.dateformat))
@@ -48,7 +48,7 @@ class DotMoonNetAdapter(BaseSiteAdapter):
        
        # get storyId from url--url validation guarantees query is only sid=1234
        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
        
        # normalized story URL. www.dotmoon.net/library_view.php?storyid=3
        self._setURL('http://' + self.getSiteDomain() + '/library_view.php?storyid='+self.story.getMetadata('storyId'))
@@ -141,7 +141,7 @@ class DotMoonNetAdapter(BaseSiteAdapter):
        
        ## Title
        a = body.find('b')
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url. http://www.dotmoon.net/board/index.php?action=profile;u=1'
        a = body.find('a', href=re.compile(r"index.php\?action=profile;u=\d+"))
@@ -48,7 +48,7 @@ class DracoAndGinnyComAdapter(BaseSiteAdapter):
        
        # get storyId from url--url validation guarantees query is only sid=1234
        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
        
        # normalized story URL.
        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -170,7 +170,7 @@ class DracoAndGinnyComAdapter(BaseSiteAdapter):
        
        ## Title
        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -48,7 +48,7 @@ class DramioneOrgAdapter(BaseSiteAdapter):
        
        # get storyId from url--url validation guarantees query is only sid=1234
        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
        
        # normalized story URL.
        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -154,7 +154,7 @@ class DramioneOrgAdapter(BaseSiteAdapter):
        
        ## Title
        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -48,7 +48,7 @@ class EfictionEstelielDeAdapter(BaseSiteAdapter):
        
        # get storyId from url--url validation guarantees query is only sid=1234
        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
        
        # normalized story URL.
        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -104,7 +104,7 @@ class EfictionEstelielDeAdapter(BaseSiteAdapter):
        pagetitle = soup.find('div',{'id':'pagetitle'})
        ## Title
        a = pagetitle.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = pagetitle.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -48,7 +48,7 @@ class EFPFanFicNet(BaseSiteAdapter):
        
        # get storyId from url--url validation guarantees query is only sid=1234
        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
        
        # normalized story URL.
        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -166,7 +166,7 @@ class EFPFanFicNet(BaseSiteAdapter):
        logger.debug("Normalizing to URL: "+url)
        self._setURL(url)
        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        

        # eFiction sites don't help us out a lot with their meta data
        # formating, so it's a little ugly.
@@ -48,7 +48,7 @@ class ErosnSapphoSycophantHexComAdapter(BaseSiteAdapter):
        
        # get storyId from url--url validation guarantees query is only sid=1234
        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
 		
        
        # normalized story URL.
@@ -132,7 +132,7 @@ class ErosnSapphoSycophantHexComAdapter(BaseSiteAdapter):
        ## Title
        pt = soup.find('div', {'id' : 'pagetitle'})
        a = pt.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.text)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = pt.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -50,7 +50,7 @@ class FanFiktionDeAdapter(BaseSiteAdapter):
        
        # get storyId from url--url validation guarantees query is only sid=1234
        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
        
        # normalized story URL.
        self._setURL('http://' + self.getSiteDomain() + '/s/'+self.story.getMetadata('storyId') + '/1')
@@ -69,10 +69,10 @@ class FanFiktionDeAdapter(BaseSiteAdapter):

    @classmethod
    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/s/46ccbef30000616306614050"
+        return "http://"+self.getSiteDomain()+"/s/46ccbef30000616306614050 http://"+self.getSiteDomain()+"/s/46ccbef30000616306614050/1 http://"+self.getSiteDomain()+"/s/46ccbef30000616306614050/1/story-name"

    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/s/")+r"\w+(/\d+)?$"
+        return re.escape("http://"+self.getSiteDomain()+"/s/")+r"\w+(/\d+)?"
        
        ## Login seems to be reasonably standard across eFiction sites.
    def needToLoginCheck(self, data):
@@ -138,15 +138,15 @@ class FanFiktionDeAdapter(BaseSiteAdapter):
        # Now go hunting for all the meta data and the chapter list.
        
        ## Title
-        a = soup.find('a', href=re.compile(r'/s/'+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        a = soup.find('a', href=re.compile(r'/s/'+self.story.getMetadata('storyId')+"/"))
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        head = soup.find('div', {'class' : 'story-metadata-left-top'})
        a = head.find('a')
        self.story.setMetadata('authorId',a['href'].split('/')[2])
        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
+        self.story.setMetadata('author',stripHTML(a))

        # Find the chapters:
        for chapter in soup.find('select').findAll('option'):
@@ -173,14 +173,14 @@ class FanFiktionDeAdapter(BaseSiteAdapter):
        tr=asoup.findAll('tr')
        for i in range(1,len(tr)):
            a = tr[i].find('a')
-            if a['href'] == '/s/'+self.story.getMetadata('storyId'):
+            if '/s/'+self.story.getMetadata('storyId')+'/1/' in a['href']:
                break
        self.setDescription(url,a['onmouseover'].split("', '")[1])
        
        td = tr[i].findAll('td')
        self.story.addToList('category',stripHTML(td[1]))
-        self.story.setMetadata('rating', td[4].string)
-        self.story.setMetadata('numWords', td[5].string)
+        self.story.setMetadata('rating', stripHTML(td[4]))
+        self.story.setMetadata('numWords', stripHTML(td[5]))
 
            
    # grab the text for an individual chapter.
@@ -52,7 +52,7 @@ class FicBookNetAdapter(BaseSiteAdapter):
        
        # get storyId from url--url validation guarantees query is only sid=1234
        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
        
        # normalized story URL.
        self._setURL('http://' + self.getSiteDomain() + '/readfic/'+self.story.getMetadata('storyId'))
@@ -45,7 +45,7 @@ class FictionAlleyOrgSiteAdapter(BaseSiteAdapter):
        if m:
            self.story.setMetadata('authorId',m.group('auth'))
            self.story.setMetadata('storyId',m.group('id'))
-            logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+            
            # normalized story URL.
            self._setURL(url)
        else:
@@ -111,7 +111,7 @@ class FictionAlleyOrgSiteAdapter(BaseSiteAdapter):
            url=self.url
            logger.debug("Normalizing to URL: "+url)
            ## title's right there...
-            self.story.setMetadata('title',storya.string)
+            self.story.setMetadata('title',stripHTML(storya))
            data = self._fetchUrl(url)
            soup = bs.BeautifulSoup(data)
            chapterlinklist = soup.findAll('a',{'class':'chapterlink'})
@@ -121,7 +121,7 @@ class FictionAlleyOrgSiteAdapter(BaseSiteAdapter):
            ## same way.
            chapsoup = bs.BeautifulSoup(chapterdata)
            storya = chapsoup.find('div',{'class':'breadcrumbs'}).findAll('a')[1]
-            self.story.setMetadata('title',storya.string)
+            self.story.setMetadata('title',stripHTML(storya))
            del chapsoup

        del chapterdata
@@ -116,7 +116,7 @@ class FicwadComSiteAdapter(BaseSiteAdapter):

        # title - first h4 tag will be title.
        titleh4 = soup.find('h4')
-        self.story.setMetadata('title', titleh4.a.string)
+        self.story.setMetadata('title', stripHTML(titleh4.a))

        # Find authorid and URL from... author url.
        a = soup.find('a', href=re.compile(r"^/author/\d+"))
@@ -117,7 +117,7 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
            
        ## Title
        a = soup.find('a', href=re.compile(r'^/story/'+self.story.getMetadata('storyId')))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # self.story.setMetadata("title", storyMetadata["title"])
        # if not storyMetadata["title"]:
@@ -50,7 +50,7 @@ class FineStoriesComAdapter(BaseSiteAdapter):
        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2].split(':')[0])
        if 'storyInfo' in self.story.getMetadata('storyId'):
            self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
        
        # normalized story URL.
        self._setURL('http://' + self.getSiteDomain() + '/s/storyInfo.php?id='+self.story.getMetadata('storyId'))
@@ -142,7 +142,7 @@ class FineStoriesComAdapter(BaseSiteAdapter):
        
        ## Title
        a = soup.find('a', href=re.compile(r'/s/'+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.text)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = soup.find('a', href=re.compile(r"/a/\w+"))
@@ -48,7 +48,7 @@ class GrangerEnchantedCom(BaseSiteAdapter):
        
        # get storyId from url--url validation guarantees query is only sid=1234
        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
        self.section=self.parsedUrl.path.split('/',)[1]
             
        # normalized story URL.
@@ -182,7 +182,7 @@ class GrangerEnchantedCom(BaseSiteAdapter):
        
        ## Title
        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -42,7 +42,7 @@ class HarryPotterFanFictionComSiteAdapter(BaseSiteAdapter):
        
        # get storyId from url--url validation guarantees query is only psid=1234
        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
        
        # normalized story URL.
        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?psid='+self.story.getMetadata('storyId'))
@@ -92,7 +92,7 @@ class HarryPotterFanFictionComSiteAdapter(BaseSiteAdapter):

        ## Title
        a = soup.find('a', href=re.compile(r'\?psid='+self.story.getMetadata('storyId')))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        ## javascript:if (confirm('Please note. This story may contain adult themes. By clicking here you are stating that you are over 17. Click cancel if you do not meet this requirement.')) location = '?psid=290995'
        if "This story may contain adult themes." in a['href'] and not (self.is_adult or self.getConfig("is_adult")):
            raise exceptions.AdultCheckRequired(self.url)
@@ -48,7 +48,7 @@ class HennethAnnunNetAdapter(BaseSiteAdapter):
        
        # get storyId from url--url validation guarantees query is only sid=1234
        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
        
        # normalized story URL.
        self._setURL('http://' + self.getSiteDomain() + '/stories/chapter.cfm?stid='+self.story.getMetadata('storyId'))
@@ -101,7 +101,7 @@ class HennethAnnunNetAdapter(BaseSiteAdapter):
        
        ## Title
        a = soup.find('h2', {'id':'page_heading'})
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))

        # Find the chapters: chapter_view.cfm?stid=6663&amp;spordinal=1" 
        for chapter in soup.findAll('a', href=re.compile(r'chapter_view.cfm\?stid='+self.story.getMetadata('storyId')+"&spordinal=\d+$")):
@@ -48,7 +48,7 @@ class HLFictionNetAdapter(BaseSiteAdapter):
        
        # get storyId from url--url validation guarantees query is only sid=1234
        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
        
        # normalized story URL.
        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -106,7 +106,7 @@ class HLFictionNetAdapter(BaseSiteAdapter):
        self.story.setMetadata('author',aut.string)
        aut.extract()
        
-        self.story.setMetadata('title',a.string[:(len(a.string)-3)])
+        self.story.setMetadata('title',stripHTML(a)[:(len(a.string)-3)])

        # Find the chapters:
        chapters=soup.find('select')
@@ -52,7 +52,7 @@ class HPFandomNetAdapterAdapter(BaseSiteAdapter): # XXX
        
        # get storyId from url--url validation guarantees query is only sid=1234
        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
        
        # normalized story URL.
        # XXX Most sites don't have the /eff part.  Replace all to remove it usually.
@@ -37,6 +37,11 @@ class HPFanficArchiveComAdapter(BaseSiteAdapter):
    def __init__(self, config, url):
        BaseSiteAdapter.__init__(self, config, url)

+        # hpfanficarchive.com blocks the default user-agent.  However,
+        # when asked, they said it was just general anti-spam, not
+        # targeted at us.  That lets me do this in good conscience:
+        self.opener.addheaders = [('User-agent', 'FFDL/1.7')]
+        
        self.decode = ["Windows-1252",
                       "utf8"] # 1252 is a superset of iso-8859-1.
                               # Most sites that claim to be
@@ -48,7 +53,7 @@ class HPFanficArchiveComAdapter(BaseSiteAdapter):
        
        # get storyId from url--url validation guarantees query is only sid=1234
        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
        
        # normalized story URL.
        self._setURL('http://' + self.getSiteDomain() + '/stories/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -100,7 +105,7 @@ class HPFanficArchiveComAdapter(BaseSiteAdapter):
        
        ## Title
        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -48,7 +48,7 @@ class IkEternalNetAdapter(BaseSiteAdapter):
        
        # get storyId from url--url validation guarantees query is only sid=1234
        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
        
        # normalized story URL.
        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -183,7 +183,7 @@ class IkEternalNetAdapter(BaseSiteAdapter):
        
        ## Title
        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -48,7 +48,7 @@ class ImagineEFicComAdapter(BaseSiteAdapter):
        
        # get storyId from url--url validation guarantees query is only sid=1234
        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
        
        # normalized story URL.
        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -170,7 +170,7 @@ class ImagineEFicComAdapter(BaseSiteAdapter):
        
        ## Title
        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -51,7 +51,7 @@ class InDeathNetAdapter(BaseSiteAdapter):
        m = re.match(self.getSiteURLPattern(),url)
        if m:
            self.story.setMetadata('storyId',m.group('id'))
-            logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+            
            # normalized story URL.
            self._setURL('http://www.' + self.getSiteDomain() + '/blog/archive/'+self.story.getMetadata('storyId')+'-'+m.group('name')+'/')
        else:
@@ -109,7 +109,7 @@ class InDeathNetAdapter(BaseSiteAdapter):
        ## Title
        h = soup.find('a', id="blog_title")
        t = h.find('span')
-        self.story.setMetadata('title',t.contents[0].string.strip())
+        self.story.setMetadata('title',stripHTML(t.contents[0]).strip())
 		
        s = t.find('div')        
        if s != None:
@@ -122,9 +122,6 @@ class InDeathNetAdapter(BaseSiteAdapter):
        self.story.setMetadata('authorUrl',a['href'])
        self.story.setMetadata('author',m.group('name'))

-
-
-
        # Find the chapters:
        chapters=soup.findAll('a', title="View entry", href=re.compile(r'http://www.indeath.net/blog/'+self.story.getMetadata('storyId')+"/entry\-(\d+)\-([^/]*)/$"))

@@ -52,7 +52,7 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
        
        # get storyId from url--url validation guarantees query is only sid=1234
        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
        
        # normalized story URL.
        # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
@@ -49,7 +49,7 @@ class LibraryOfMoriaComAdapter(BaseSiteAdapter):
        
        # get storyId from url--url validation guarantees query is only sid=1234
        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
        
        # normalized story URL.
        self._setURL('http://' + self.getSiteDomain() + '/a/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -129,7 +129,7 @@ class LibraryOfMoriaComAdapter(BaseSiteAdapter):
        
        ## Title
        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -48,7 +48,7 @@ class LumosSycophantHexComAdapter(BaseSiteAdapter):
        
        # get storyId from url--url validation guarantees query is only sid=1234
        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
 		
        
        # normalized story URL.
@@ -116,7 +116,7 @@ class LumosSycophantHexComAdapter(BaseSiteAdapter):
        ## Title
        pt = soup.find('div', {'id' : 'pagetitle'})
        a = pt.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.text)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = pt.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -43,7 +43,7 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
        m = re.match(self.getSiteURLPattern(),url)
        if m:
            self.story.setMetadata('storyId',m.group('id'))
-            logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+            
            # normalized story URL.
            self._setURL('http://' + self.getSiteDomain() + '/fanfic/view_st.php/'+self.story.getMetadata('storyId'))
        else:
@@ -107,10 +107,10 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
        for font in title.findAll('font'):
            font.extract() # removes 'font' tags from inside the td.        
        if title.has_key('colspan'):
-            titlet = title.text
+            titlet = stripHTML(title)
        else:
            ## No colspan, it's part chapter title--even if it's a one-shot.
-            titlet = ':'.join(title.text.split(':')[:-1]) # strip trailing 'Chapter X' or chapter title
+            titlet = ':'.join(stripHTML(title).split(':')[:-1]) # strip trailing 'Chapter X' or chapter title
        self.story.setMetadata('title',titlet)
        ## The story title is difficult to reliably parse from the
        ## story pages.  Getting it from the author page is, but costs
@@ -48,7 +48,7 @@ class MerlinFicDtwinsCoUk(BaseSiteAdapter):
        
        # get storyId from url--url validation guarantees query is only sid=1234
        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
        
        # normalized story URL.
        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -170,7 +170,7 @@ class MerlinFicDtwinsCoUk(BaseSiteAdapter):
        
        ## Title
        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -52,7 +52,7 @@ class MidnightwhispersCaAdapter(BaseSiteAdapter): # XXX
        
        # get storyId from url--url validation guarantees query is only sid=1234
        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
        
        # normalized story URL.
        # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
@@ -52,7 +52,7 @@ class MuggleNetComAdapter(BaseSiteAdapter): # XXX
        
        # get storyId from url--url validation guarantees query is only sid=1234
        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
        
        # normalized story URL.
        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -193,7 +193,7 @@ class MuggleNetComAdapter(BaseSiteAdapter): # XXX
        
        ## Title
        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -48,7 +48,7 @@ class NationalLibraryNetAdapter(BaseSiteAdapter):
        
        # get storyId from url--url validation guarantees query is only storyid=1234
        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
        
        # normalized story URL.
        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?storyid='+self.story.getMetadata('storyId'))
@@ -104,7 +104,7 @@ class NationalLibraryNetAdapter(BaseSiteAdapter):
        
        ## Title
        a = soup.find('h1')
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = soup.find('a', href=re.compile(r"authorresults.php\?author=\d+"))
@@ -48,7 +48,7 @@ class NCISFicComAdapter(BaseSiteAdapter):
        
        # get storyId from url--url validation guarantees query is only storyid=1234
        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
        
        # normalized story URL.
        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?storyid='+self.story.getMetadata('storyId'))
@@ -103,7 +103,7 @@ class NCISFicComAdapter(BaseSiteAdapter):
        
        ## Title
        a = soup.find('h1')
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = soup.find('a', href=re.compile(r"authorresults.php\?author=\d+"))
@@ -37,10 +37,10 @@ class NCISFictionNetAdapter(BaseSiteAdapter):
    def __init__(self, config, url):
        BaseSiteAdapter.__init__(self, config, url)

-        # ncisfiction.net blocks IPs the default user-agent.  However,
+        # ncisfiction.net blocks the default user-agent.  However,
        # when asked, they said it was just general anti-spam, not
        # targeted at us.  That lets me do this in good conscience:
-        self.opener.addheaders = [('User-agent', 'FFDL/1.6')]
+        self.opener.addheaders = [('User-agent', 'FFDL/1.7')]
        
        self.decode = ["iso-8859-1",
                       "Windows-1252"] # 1252 is a superset of iso-8859-1.
@@ -53,7 +53,7 @@ class NCISFictionNetAdapter(BaseSiteAdapter):
        
        # get storyId from url--url validation guarantees query is only sid=1234
        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
        
        # normalized story URL.
        self._setURL("http://"+self.getSiteDomain()\
@@ -46,7 +46,7 @@ class NetRaptorOrgAdapter(BaseSiteAdapter):
        
        # get storyId from url--url validation guarantees query is only sid=1234
        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
        
        # normalized story URL.
        self._setURL('http://' + self.getSiteDomain() + '/fanfiction/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -97,7 +97,7 @@ class NetRaptorOrgAdapter(BaseSiteAdapter):
        ## Title
        pagetitle = soup.find('div',{'id':'pagetitle'})
        a = pagetitle.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = pagetitle.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -52,7 +52,7 @@ class NfaCommunityComAdapter(BaseSiteAdapter): # XXX
        
        # get storyId from url--url validation guarantees query is only sid=1234
        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
        
        # normalized story URL.
        # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
@@ -156,7 +156,7 @@ class NfaCommunityComAdapter(BaseSiteAdapter): # XXX
        
        ## Title
        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -48,7 +48,7 @@ class NHAMagicalWorldsUsAdapter(BaseSiteAdapter):
        
        # get storyId from url--url validation guarantees query is only sid=1234
        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
 		
        
        # normalized story URL.
@@ -115,7 +115,7 @@ class NHAMagicalWorldsUsAdapter(BaseSiteAdapter):
            a = info.find('a')
            if 'viewstory.php?sid='+self.story.getMetadata('storyId') == a['href'] or \
                    ('viewstory.php?sid='+self.story.getMetadata('storyId')+'&') in a['href']:
-                self.story.setMetadata('title',a.string)
+                self.story.setMetadata('title',stripHTML(a))
                break
 		

@@ -48,7 +48,7 @@ class NickAndGregNetAdapter(BaseSiteAdapter):
        
        # get storyId from url--url validation guarantees query is only sid=1234
        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
        
        # normalized story URL.
        # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
@@ -101,7 +101,7 @@ class NickAndGregNetAdapter(BaseSiteAdapter):

        ## Title
        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -48,7 +48,7 @@ class OcclumencySycophantHexComAdapter(BaseSiteAdapter):
        
        # get storyId from url--url validation guarantees query is only sid=1234
        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
 		
        
        # normalized story URL.
@@ -158,7 +158,7 @@ class OcclumencySycophantHexComAdapter(BaseSiteAdapter):
        for info in asoup.findAll('table', {'class' : 'border'}):
            a = info.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
            if a != None:
-                self.story.setMetadata('title',a.text)
+                self.story.setMetadata('title',stripHTML(a))
                break
 		

@@ -48,7 +48,7 @@ class OneDirectionFanfictionComAdapter(BaseSiteAdapter):
        
        # get storyId from url--url validation guarantees query is only sid=1234
        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
        
        # normalized story URL.
        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -158,7 +158,7 @@ class OneDirectionFanfictionComAdapter(BaseSiteAdapter):
        
        ## Title
        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -48,7 +48,7 @@ class PhoenixSongNetAdapter(BaseSiteAdapter):
        
        # get storyId from url--url validation guarantees query is only sid=1234
        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[3])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
        
        # normalized story URL.
        self._setURL('http://' + self.getSiteDomain() + '/fanfiction/story/' +self.story.getMetadata('storyId')+'/')
@@ -136,7 +136,7 @@ class PhoenixSongNetAdapter(BaseSiteAdapter):
        ## Title
        b = soup.find('div', {'id' : 'nav25'})
        a = b.find('a', href=re.compile(r'fanfiction/story/'+self.story.getMetadata('storyId')+"/$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.  /fanfiction/stories.php?psid=125
        a = b.find('a', href=re.compile(r"/fanfiction/stories.php\?psid=\d+"))
@@ -48,7 +48,7 @@ class PommeDeSangComAdapter(BaseSiteAdapter):
        
        # get storyId from url--url validation guarantees query is only sid=1234
        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
        
        # pommedesang.com has two 'sections', shown in URL as
        # 'efiction' and 'sds' that change how things should be
@@ -180,7 +180,7 @@ class PommeDeSangComAdapter(BaseSiteAdapter):
        
        ## Title
        a = soup.find('a', href=re.compile('viewstory.php\?sid=\d+'))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -48,8 +48,7 @@ class PonyFictionArchiveNetAdapter(BaseSiteAdapter):
        
        # get storyId from url--url validation guarantees query is only sid=1234
        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-        logger.info(self.parsedUrl.netloc)
+        
        # normalized story URL.
        if "explicit" in self.parsedUrl.netloc:
            self._setURL('http://explicit.' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -137,7 +136,7 @@ class PonyFictionArchiveNetAdapter(BaseSiteAdapter):
        
        ## Title
        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -55,7 +55,7 @@ class PortkeyOrgAdapter(BaseSiteAdapter): # XXX
        
        # get storyId from url--url validation guarantees query is only sid=1234
        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
        
        # normalized story URL.
        self._setURL('http://' + self.getSiteDomain() + '/story/'+self.story.getMetadata('storyId'))
@@ -41,7 +41,7 @@ class PotionsAndSnitchesNetSiteAdapter(BaseSiteAdapter):
        
        # get storyId from url--url validation guarantees query is only sid=1234
        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
        
        # normalized story URL.
        self._setURL('http://' + self.getSiteDomain() + '/fanfiction/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -83,7 +83,7 @@ class PotionsAndSnitchesNetSiteAdapter(BaseSiteAdapter):

        ## Title
        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -54,7 +54,7 @@ class PotterFicsComAdapter(BaseSiteAdapter):
        m = re.match(self.getSiteURLPattern(),url)
        if m:
            self.story.setMetadata('storyId',m.group('id'))
-            logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+            
            # normalized story URL. gets rid of chapter if there, left with chapter index URL
            nurl = "http://"+self.getSiteDomain()+"/historias/"+self.story.getMetadata('storyId')
            self._setURL(nurl)
@@ -62,7 +62,7 @@ class PotterFicsComAdapter(BaseSiteAdapter):
            raise exceptions.InvalidStoryURL(url,
                                             self.getSiteDomain(),
                                             self.getSiteExampleURLs())
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
        
        # Each adapter needs to have a unique site abbreviation.
        self.story.setMetadata('siteabbrev','potficscom')
@@ -147,7 +147,7 @@ class PotterFicsComAdapter(BaseSiteAdapter):
        #find first metadata block
        mb = cell.div.findNextSibling('div')
        #Get meta...
-        self.story.setMetadata('title', mb.b.string)
+        self.story.setMetadata('title', stripHTML(mb.b))
        #strip out brackets on rating
        self.story.setMetadata('rating', mb.span.string[1:-1])
        #Completion status is denoted by the presence of this image:
@@ -48,7 +48,7 @@ class PotterHeadsAnonymousComAdapter(BaseSiteAdapter):
        
        # get storyId from url--url validation guarantees query is only sid=1234
        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
        
        # normalized story URL.
        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -181,7 +181,7 @@ class PotterHeadsAnonymousComAdapter(BaseSiteAdapter):
        
        ## Title
        a = pagetitle.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = pagetitle.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -48,7 +48,7 @@ class PretenderCenterComAdapter(BaseSiteAdapter):
        
        # get storyId from url--url validation guarantees query is only sid=1234
        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
        
        # normalized story URL.
        self._setURL('http://' + self.getSiteDomain() + '/missingpieces/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -134,7 +134,7 @@ class PretenderCenterComAdapter(BaseSiteAdapter):
        
        ## Title
        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.text)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -48,7 +48,7 @@ class PsychFicComAdapter(BaseSiteAdapter):
        
        # get storyId from url--url validation guarantees query is only sid=1234
        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
        
        # normalized story URL.
        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -130,7 +130,7 @@ class PsychFicComAdapter(BaseSiteAdapter):
        
        ## Title
        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -48,7 +48,7 @@ class QafFicComAdapter(BaseSiteAdapter):
        
        # get storyId from url--url validation guarantees query is only sid=1234
        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
        
        # normalized story URL.
        self._setURL('http://' + self.getSiteDomain() + '/atp/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -138,7 +138,7 @@ class QafFicComAdapter(BaseSiteAdapter):
        self.story.setMetadata('author',aut.string)
        aut.extract()
        
-        self.story.setMetadata('title',a.string[:(len(a.string)-3)])
+        self.story.setMetadata('title',stripHTML(a)[:(len(a.string)-3)])

        # Find the chapters:
        chapters=soup.find('select')
@@ -52,10 +52,9 @@ class RestrictedSectionOrgSiteAdapter(BaseSiteAdapter):
        m = re.match(self.getSiteURLPattern(),url)
        if m:
            self.story.setMetadata('storyId',m.group('id'))
-            logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+            
            # normalized story URL.
            self._setURL('http://' + self.getSiteDomain() + '/' + m.group('filestory') + '.php?' + m.group('filestory') + '=' + self.story.getMetadata('storyId'))
-            logger.debug("storyUrl: (%s)"%self.story.getMetadata('storyUrl'))
        else:
            raise exceptions.InvalidStoryURL(url,
                                             self.getSiteDomain(),
@@ -127,10 +126,12 @@ class RestrictedSectionOrgSiteAdapter(BaseSiteAdapter):
        
        self.story.setMetadata('authorId',ahref.split('=')[1])
        self.story.setMetadata('authorUrl','http://'+self.host+'/'+ahref)
-        self.story.setMetadata('author',a.string)
+        self.story.setMetadata('author',stripHTML(a))

        # title, remove byauthorname.
-        self.story.setMetadata('title',h2.text[:h2.text.index("by"+a.string)])
+        auth=stripHTML(a)
+        title=stripHTML(h2)
+        self.story.setMetadata('title',title[:title.index(" by "+auth)])

        dates = soup.findAll('span', {'class':'date'})
        if dates: # only for multi-chapter
@@ -48,7 +48,7 @@ class SamDeanArchiveNuAdapter(BaseSiteAdapter):
        
        # get storyId from url--url validation guarantees query is only sid=1234
        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
        
        # normalized story URL.
        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -110,7 +110,7 @@ class SamDeanArchiveNuAdapter(BaseSiteAdapter):
        self.story.setMetadata('author',aut.string)
        aut.extract()
        
-        self.story.setMetadata('title',a.string[:(len(a.string)-3)])
+        self.story.setMetadata('title',stripHTML(a)[:(len(stripHTML(a))-3)])

        # Find the chapters:
        chapters=soup.find('select')
@@ -48,7 +48,7 @@ class ScarHeadNetAdapter(BaseSiteAdapter):
        
        # get storyId from url--url validation guarantees query is only sid=1234
        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
        
        # normalized story URL.
        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -181,7 +181,7 @@ class ScarHeadNetAdapter(BaseSiteAdapter):
        
        ## Title
        a = pagetitle.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = pagetitle.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -48,7 +48,7 @@ class ScarvesAndCoffeeNetAdapter(BaseSiteAdapter):
        
        # get storyId from url--url validation guarantees query is only sid=1234
        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
        
        # normalized story URL.
        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -139,7 +139,7 @@ class ScarvesAndCoffeeNetAdapter(BaseSiteAdapter):
        
        ## Title
        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = soup.find('div',{"id":"pagetitle"}).find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -48,7 +48,7 @@ class SG1HeliopolisComAdapter(BaseSiteAdapter):
        
        # get storyId from url--url validation guarantees query is only sid=1234
        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
        self.section=self.parsedUrl.path.split('/',)[1]
        
        # normalized story URL.
@@ -140,7 +140,7 @@ class SG1HeliopolisComAdapter(BaseSiteAdapter):
        
        ## Title
        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -48,7 +48,7 @@ class SimplyUndeniableComAdapter(BaseSiteAdapter):
        
        # get storyId from url--url validation guarantees query is only sid=1234
        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
        
        # normalized story URL.
        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -143,7 +143,7 @@ class SimplyUndeniableComAdapter(BaseSiteAdapter):
        
        ## Title
        a = soup.find('h1')
-        self.story.setMetadata('title',a.text)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -155,7 +155,7 @@ class SimplyUndeniableComAdapter(BaseSiteAdapter):
        for info in asoup.findAll('table', {'cellpadding' : '5'}):
            a = info.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
            if a != None:
-                self.story.setMetadata('title',a.string)
+                self.story.setMetadata('title',stripHTML(a))
                break
                
        # Find the chapters:
@@ -48,7 +48,7 @@ class SinfulDesireOrgAdapter(BaseSiteAdapter):
        
        # get storyId from url--url validation guarantees query is only sid=1234
        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
        
        # normalized story URL.
        self._setURL('http://' + self.getSiteDomain() + '/archive/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -130,7 +130,7 @@ class SinfulDesireOrgAdapter(BaseSiteAdapter):
        
        ## Title
        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -49,7 +49,7 @@ class SiyeCoUkAdapter(BaseSiteAdapter): # XXX
        
        # get storyId from url--url validation guarantees query is only sid=1234
        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
        
        # normalized story URL.
        self._setURL('http://' + self.getSiteDomain() + '/siye/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -59,7 +59,7 @@ class SquidgeOrgPejaAdapter(BaseSiteAdapter):
        
        # get storyId from url--url validation guarantees query is only sid=1234
        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
        
        # normalized story URL.
        self._setURL('http://' + self.getSiteDomain() + '/peja/cgi-bin/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -122,7 +122,7 @@ class SquidgeOrgPejaAdapter(BaseSiteAdapter):
        # formating, so it's a little ugly.

        title = authorSoup.find('a',{'href':'viewstory.php?sid='+self.story.getMetadata('storyId')})
-        self.story.setMetadata('title',title.string)
+        self.story.setMetadata('title',stripHTML(title))
        titleblock=title.parent.parent
        
        chapterselect=soup.find('select',{'name':'chapter'})
@@ -48,7 +48,7 @@ class StargateAtlantisOrgAdapter(BaseSiteAdapter):
        
        # get storyId from url--url validation guarantees query is only sid=1234
        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
        
        # normalized story URL.
        self._setURL('http://' + self.getSiteDomain() + '/fanfics/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -106,7 +106,7 @@ class StargateAtlantisOrgAdapter(BaseSiteAdapter):
        self.story.setMetadata('author',aut.string)
        aut.extract()
        
-        self.story.setMetadata('title',a.string[:(len(a.string)-3)])
+        self.story.setMetadata('title',stripHTML(a)[:(len(stripHTML(a))-3)])

        # Find the chapters:
        chapters=soup.findAll('div', {'class' : 'content'})
@@ -48,7 +48,7 @@ class StoriesOfArdaComAdapter(BaseSiteAdapter):
        
        # get storyId from url--url validation guarantees query is only sid=1234
        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
        
        # normalized story URL.
        self._setURL('http://' + self.getSiteDomain() + '/chapterlistview.asp?SID='+self.story.getMetadata('storyId'))
@@ -104,7 +104,7 @@ class StoriesOfArdaComAdapter(BaseSiteAdapter):
        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
        
        a.find('em').extract()
-        self.story.setMetadata('title',a.text)
+        self.story.setMetadata('title',stripHTML(a))

        # Find the chapters: chapterview.asp?sid=7000&cid=30919
        chapters=soup.findAll('a', href=re.compile(r'chapterview.asp\?sid='+self.story.getMetadata('storyId')+"&cid=\d+$"))
@@ -137,8 +137,20 @@ class StoriesOfArdaComAdapter(BaseSiteAdapter):

        logger.debug('Getting chapter text from: %s' % url)

-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+        if self.getConfig('is_adult'):
+            params = {'confirmAge':'1'}
+            data = self._postUrl(url,params)
+        else:
+            data = self._fetchUrl(url)
+
+        data = data[data.index('<table width="90%" align="center">'):]
+        data.replace("<body","<notbody").replace("<BODY","<NOTBODY")
+            
+        soup = bs.BeautifulStoneSoup(data,
                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+
+        if "Please indicate that you are an adult by selecting the appropriate choice below" in data:
+            raise exceptions.FailedToDownload("Chapter requires you be an adult.  Set is_adult in personal.ini (chapter url:%s)" % url)
        
        div = soup.find('table', {'width' : '90%'}).find('td')
        div.name='div'
@@ -44,7 +44,7 @@ class TenhawkPresentsComSiteAdapter(BaseSiteAdapter):
        
        # get storyId from url--url validation guarantees query is only sid=1234
        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
        
        # normalized story URL.
        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -133,7 +133,7 @@ class TenhawkPresentsComSiteAdapter(BaseSiteAdapter):

        ## Title
        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -48,7 +48,7 @@ class TheAlphaGateComAdapter(BaseSiteAdapter):
        
        # get storyId from url--url validation guarantees query is only sid=1234
        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
        
        # normalized story URL.
        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -99,7 +99,7 @@ class TheAlphaGateComAdapter(BaseSiteAdapter):
        
        ## Title
        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -48,7 +48,7 @@ class TheHexFilesNetAdapter(BaseSiteAdapter):
        
        # get storyId from url--url validation guarantees query is only sid=1234
        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
        
        # normalized story URL.
        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -103,7 +103,7 @@ class TheHexFilesNetAdapter(BaseSiteAdapter):
        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
        self.story.setMetadata('authorId',a['href'].split('=')[1])
        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
+        self.story.setMetadata('author',stripHTML(a))
        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
 		
        try:
@@ -118,7 +118,7 @@ class TheHexFilesNetAdapter(BaseSiteAdapter):
        for info in asoup.findAll('table', {'cellspacing' : '4'}):
            a = info.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
            if a != None:
-                self.story.setMetadata('title',a.string)
+                self.story.setMetadata('title',stripHTML(a))
                break
 		

@@ -72,7 +72,7 @@ class TheHookupZoneNetAdapter(BaseSiteAdapter): # XXX
        
        # get storyId from url--url validation guarantees query is only sid=1234
        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
        
        # normalized story URL.
        # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
@@ -179,7 +179,7 @@ class TheHookupZoneNetAdapter(BaseSiteAdapter): # XXX
        
        ## Title
        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -48,7 +48,7 @@ class TheMasqueNetAdapter(BaseSiteAdapter):
        
        # get storyId from url--url validation guarantees query is only sid=1234
        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
        
        if self.parsedUrl.path.split('/',)[1] == 'wiktt':
            self.story.addToList("category","Harry Potter")
@@ -177,7 +177,7 @@ class TheMasqueNetAdapter(BaseSiteAdapter):
        
        ## Title
        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -48,7 +48,7 @@ class ThePetulantPoetessComAdapter(BaseSiteAdapter):
        
        # get storyId from url--url validation guarantees query is only sid=1234
        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
        
        # normalized story URL.
        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId') +'&i=1')
@@ -165,7 +165,7 @@ class ThePetulantPoetessComAdapter(BaseSiteAdapter):
            for info in asoup.findAll('td', {'class' : 'highlightcolor1'}):
                a = info.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
                if a != None:
-                    self.story.setMetadata('title',a.text)                
+                    self.story.setMetadata('title',stripHTML(a))
                    found = 1
                    break
            index=index+1
@@ -52,7 +52,7 @@ class TheQuidditchPitchOrgAdapter(BaseSiteAdapter): # XXX
        
        # get storyId from url--url validation guarantees query is only sid=1234
        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
        
        # normalized story URL.
        # XXX Most sites don't have the  part.  Replace all to remove it usually.
@@ -165,7 +165,7 @@ class TheQuidditchPitchOrgAdapter(BaseSiteAdapter): # XXX
        
        ## Title
        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -44,7 +44,7 @@ class TheWritersCoffeeShopComSiteAdapter(BaseSiteAdapter):
        
        # get storyId from url--url validation guarantees query is only sid=1234
        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
        
        # normalized story URL.
        self._setURL('http://' + self.getSiteDomain() + '/library/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -48,7 +48,7 @@ class TokraFandomnetComAdapter(BaseSiteAdapter):
        
        # get storyId from url--url validation guarantees query is only sid=1234
        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
        
        # normalized story URL.
        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -131,7 +131,7 @@ class TokraFandomnetComAdapter(BaseSiteAdapter):
        
        ## Title
        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -41,7 +41,7 @@ class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter):
        m = re.match(self.getSiteURLPattern(),url)
        if m:
            self.story.setMetadata('storyId',m.group('id'))
-            logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+            
            # normalized story URL.
            self._setURL("http://"+self.getSiteDomain()\
                         +"/Story-"+self.story.getMetadata('storyId'))
@@ -48,7 +48,7 @@ class TwilightArchivesComAdapter(BaseSiteAdapter):
        
        # get storyId from url--url validation guarantees query is only sid=1234
        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
        
        # normalized story URL. http://www.twilightarchives.com/read/9353
        self._setURL('http://' + self.getSiteDomain() + '/read/'+self.story.getMetadata('storyId'))
@@ -100,7 +100,7 @@ class TwilightArchivesComAdapter(BaseSiteAdapter):
        
        ## Title
        a = soup.find('h1')
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find the chapters:
        chapters=soup.find('ol', {'class' : 'chapters'})
@@ -43,7 +43,7 @@ class TwilightedNetSiteAdapter(BaseSiteAdapter):
        
        # get storyId from url--url validation guarantees query is only sid=1234
        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
        
        # normalized story URL.
        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -130,7 +130,7 @@ class TwilightedNetSiteAdapter(BaseSiteAdapter):

        ## Title
        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = soup.find('a', href=re.compile(r"viewuser.php"))
@@ -44,7 +44,7 @@ class TwiwriteNetSiteAdapter(BaseSiteAdapter):
        
        # get storyId from url--url validation guarantees query is only sid=1234
        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
        
        # normalized story URL.
        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -144,7 +144,7 @@ class TwiwriteNetSiteAdapter(BaseSiteAdapter):
        
        ## Title
        a = pagetitlediv.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = pagetitlediv.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -46,7 +46,7 @@ class WalkingThePlankOrgAdapter(BaseSiteAdapter):
        
        # get storyId from url--url validation guarantees query is only sid=1234
        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
        
        # normalized story URL.
        self._setURL('http://' + self.getSiteDomain() + '/archive/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -115,7 +115,7 @@ class WalkingThePlankOrgAdapter(BaseSiteAdapter):
        
        ## Title
        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -70,7 +70,7 @@ class WhoficComSiteAdapter(BaseSiteAdapter):
                raise e

        # pull title(title) and author from the HTML title.
-        title = soup.find('title').string
+        title = stripHTML(soup.find('title'))
        logger.debug('Title: %s' % title)
        title = title.split('::')[1].strip()
        self.story.setMetadata('title',title.split(' by ')[0].strip())
@@ -48,7 +48,7 @@ class WizardTalesNetAdapter(BaseSiteAdapter):
        
        # get storyId from url--url validation guarantees query is only sid=1234
        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
        
        # normalized story URL.
        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -171,7 +171,7 @@ class WizardTalesNetAdapter(BaseSiteAdapter):
        ## Title
        pt = soup.find('div', {'id' : 'pagetitle'})
        a = pt.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.text)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = pt.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -48,7 +48,7 @@ class WolverineAndRogueComAdapter(BaseSiteAdapter):
        
        # get storyId from url--url validation guarantees query is only sid=1234
        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
        
        # normalized story URL.
        self._setURL('http://' + self.getSiteDomain() + '/wrfa/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -101,7 +101,7 @@ class WolverineAndRogueComAdapter(BaseSiteAdapter):
        ## Title
        pt = soup.find('div', {'id' : 'pagetitle'})
        a = pt.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.text)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        a = pt.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -49,7 +49,7 @@ class WraithBaitComAdapter(BaseSiteAdapter):
        
        # get storyId from url--url validation guarantees query is only sid=1234
        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
        

        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -113,7 +113,7 @@ class WraithBaitComAdapter(BaseSiteAdapter):
        ## Title
        pt = soup.find('div', {'id' : 'pagetitle'})
        a = pt.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
        
        # Find authorid and URL from... author url.
        alist = pt.findAll('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -24,6 +24,7 @@ from gziphttp import GZipProcessor

 import adapters
 from configurable import Configuration
+from exceptions import UnknownSite

 def get_urls_from_page(url,configuration=None,normalize=False):

@@ -31,26 +32,31 @@ def get_urls_from_page(url,configuration=None,normalize=False):
        configuration = Configuration("test1.com","EPUB")

    data = None
+    adapter = None
+    try:
+        adapter = adapters.getAdapter(configuration,url,anyurl=True)
+        
+        # special stuff to log into archiveofourown.org, if possible.
+        # Unlike most that show the links to 'adult' stories, but protect
+        # them, AO3 doesn't even show them if not logged in.  Only works
+        # with saved user/pass--not going to prompt for list.
+        if 'archiveofourown.org' in url:
+            if adapter.getConfig("username"):
+                if adapter.getConfig("is_adult"):
+                    addurl = "?view_adult=true"
+                else:
+                    addurl=""
+                # just to get an authenticity_token.
+                data = adapter._fetchUrl(url+addurl)
+                # login the session.
+                adapter.performLogin(url,data)
+                # get the list page with logged in session.
    
-    # special stuff to log into archiveofourown.org, if possible.
-    # Unlike most that show the links to 'adult' stories, but protect
-    # them, AO3 doesn't even show them if not logged in.  Only works
-    # with saved user/pass--not going to prompt for list.
-    if 'archiveofourown.org' in url:
-        ao3adapter = adapters.getAdapter(configuration,"http://www.archiveofourown.org/works/0")
-        if ao3adapter.getConfig("username"):
-            if ao3adapter.getConfig("is_adult"):
-                addurl = "?view_adult=true"
-            else:
-                addurl=""
-            # just to get an authenticity_token.
-            data = ao3adapter._fetchUrl(url+addurl)
-            # login the session.
-            ao3adapter.performLogin(url,data)
-            # get the list page with logged in session.
-            data = ao3adapter._fetchUrl(url)
-
-    if not data:
+        # this way it uses User-Agent or other special settings.  Only AO3
+        # is doing login.
+        data = adapter._fetchUrl(url)
+    except UnknownSite:
+        # no adapter with anyurl=True, must be a random site.
        opener = u2.build_opener(u2.HTTPCookieProcessor(),GZipProcessor())
        data = opener.open(url).read()

@@ -72,11 +78,13 @@ def get_urls_from_html(data,url=None,configuration=None,normalize=False,restrict
    soup = BeautifulSoup(data)
    if restrictsearch:
        soup = soup.find(*restrictsearch)
-        print("restrict search:%s"%soup)
+        #print("restrict search:%s"%soup)
    
    for a in soup.findAll('a'):
        if a.has_key('href'):
+            #print("a['href']:%s"%a['href'])
            href = form_url(url,a['href'])
+            #print("1 urlhref:%s"%href)
            # this (should) catch normal story links, some javascript
            # 'are you old enough' links, and 'Report This' links.
            # The 'normalized' set prevents duplicates.
@@ -84,15 +92,19 @@ def get_urls_from_html(data,url=None,configuration=None,normalize=False,restrict
                #print("trying:%s"%a['href'])
                m = re.search(r"(?P<sid>(view)?story\.php\?(sid|psid|no|story|stid)=\d+)",a['href'])
                if m != None:
-                    href = form_url(url,m.group('sid'))
+                    href = form_url(a['href'] if '//' in a['href'] else url,
+                                    m.group('sid'))
                    
            try:
                href = href.replace('&index=1','')
+                #print("2 urlhref:%s"%href)
                adapter = adapters.getAdapter(configuration,href)
+                #print("found adapter")
                if adapter.story.getMetadata('storyUrl') not in normalized:
                    normalized.append(adapter.story.getMetadata('storyUrl'))
                    retlist.append(href)
-            except:
+            except Exception, e:
+                #print e
                pass

    if normalize:
@@ -411,6 +411,9 @@ class Story(Configurable):
        if not value in self.metadata[listname]:
            self.metadata[listname].append(value)

+        if listname == 'category' and self.getConfig('add_genre_when_multi_category') and len(self.metadata[listname]) > 1:
+            self.addToList('genre',self.getConfig('add_genre_when_multi_category'))
+
    def isList(self,listname):
        'Everything set with an include_in_* is considered a list.'
        return self.hasConfig("include_in_"+listname) or \
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Jim Miller	6674e19f94	Remove extra code for unchanged author link for calibre 1.0.0, bump version.	2013-08-31 19:50:18 -05:00
Jim Miller	29b1ca9a7e	Attempted debuging for another Mac issue.	2013-08-31 19:42:58 -05:00
Jim Miller	7cae65300a	Feature: add_genre_when_multi_category setting	2013-08-31 15:38:44 -05:00
Jim Miller	88fe13f603	Work around calibre's auto-split when author contains 'and' 'with' ',' or '&'.	2013-08-29 19:23:23 -05:00
Jim Miller	a9b6989341	Added tag calibre-plugin-1.7.38 for changeset d44d5f1006f6	2013-08-27 17:11:43 -05:00
Jim Miller	a54277058c	Bump version.	2013-08-27 17:11:32 -05:00
Jim Miller	4c27e19aa8	'Fix' author link updates bumping last_modified. User msg on metadata error.	2013-08-27 14:16:44 -05:00
Jim Miller	6f5ed2bcac	Added tag calibre-plugin-1.7.37 for changeset 73fa3ac021d3	2013-08-24 22:43:11 -05:00
Jim Miller	cc0bccd420	Added tag calibre-plugin-1.7.37 for changeset 73fa3ac021d3	2013-08-24 22:40:12 -05:00
Jim Miller	85bbcac92e	Fold only-case-different tags together to work around calibre bug 1216398 https://bugs.launchpad.net/calibre/+bug/1216398	2013-08-24 22:39:36 -05:00
Jim Miller	f8776e118a	Remove superfluous Fanfiction category set in adapter.	2013-08-24 22:38:30 -05:00
Jim Miller	31b7257496	Strip HTML from story titles in all the adapters that don't already.	2013-08-21 09:19:58 -05:00
Jim Miller	b5d00951de	Added tag calibre-plugin-1.7.36 for changeset e6f30eaf1bc7	2013-08-20 08:47:30 -05:00
Jim Miller	c73f7693a4	Bump PI version.	2013-08-20 08:47:18 -05:00
Jim Miller	e35927c9a2	Add options to force author & title values into author_sort and title_sort.	2013-08-18 17:19:26 -05:00
Jim Miller	e8e93b010a	Make check for changing story URL configurable.	2013-08-18 15:52:51 -05:00
Jim Miller	6414a09dbd	Added tag FanFictionDownLoader-4.4.68 for changeset db28f113a88e	2013-08-06 17:18:42 -05:00
Jim Miller	d5e7907068	Added tag calibre-plugin-1.7.35 for changeset db28f113a88e	2013-08-06 17:18:29 -05:00
Jim Miller	3175346214	Warn if no chapters on update in CLI, too. Bump versions.	2013-08-06 17:18:03 -05:00
Jim Miller	8b31ba59af	Warning if existing epub has no recognizable chapters.	2013-08-02 13:39:36 -05:00
Jim Miller	92c60d065e	Get story url lists from sites with adapter, get urls from non-fic sites in CLI.	2013-07-30 23:08:56 -05:00
Jim Miller	ba6852eec4	Remove logger.debug("storyId: ... from all adapters.	2013-07-30 21:57:22 -05:00
Jim Miller	72086e6ecf	Optimization for adapter lookup and anyurl (on site) adapter lookup.	2013-07-30 21:56:33 -05:00
Jim Miller	f8a2cb949f	Added tag calibre-plugin-1.7.34 for changeset c40bd6f33a0e	2013-07-28 16:50:37 -05:00
Jim Miller	07d1a1096f	Added tag FanFictionDownLoader-4.4.67 for changeset c40bd6f33a0e	2013-07-28 16:50:26 -05:00
Jim Miller	9f4fe192be	Bump versions.	2013-07-28 16:50:05 -05:00
Jim Miller	56cd2b2039	Fixes for dark-solace.org/elysian pagination of author/series story lists.	2013-07-28 16:46:00 -05:00
Jim Miller	b2a41a9b10	Use FFDL/1.7 User-Agent with hpfanficarchive.com because of python blocking.	2013-07-28 10:53:47 -05:00
Jim Miller	55e18f9446	Add is_adult to chapter downloads for storiesofarda.com.	2013-07-26 12:28:53 -05:00
Jim Miller	855c284389	Another fix for fanfiktion.de changes.	2013-07-25 12:02:49 -05:00
Jim Miller	a31de1d5ef	Added tag calibre-plugin-1.7.33 for changeset 9f7be11b5754	2013-07-24 13:02:25 -05:00
Jim Miller	7ac00b53ab	Fix for smarten_punctuation setting in pre-0.9.39 calibre.	2013-07-24 13:02:14 -05:00
Jim Miller	9c0519316d	Added tag FanFictionDownLoader-4.4.66 for changeset c32e4193a161	2013-07-23 15:19:01 -05:00
Jim Miller	d9fbccbef5	Added tag calibre-plugin-1.7.32 for changeset c32e4193a161	2013-07-23 15:18:48 -05:00
Jim Miller	0c622b379e	Bump versions.	2013-07-23 15:18:37 -05:00
Jim Miller	83d210566b	Fixes for fanfiktion.de & thehexfiles.net, add entries to teststory valid list.	2013-07-23 15:17:16 -05:00
Jim Miller	ab0b25317b	Added tag FanFictionDownLoader-4.4.65 for changeset 1def4c08104d	2013-07-18 11:05:16 -05:00
Jim Miller	e10ae5fbe4	Added tag calibre-plugin-1.7.31 for changeset 1def4c08104d	2013-07-18 11:04:59 -05:00