Changes for ffnet--longer sleep, config UA, bump versions.

https://bugs.launchpad.net/calibre/+bug/1216398

app.yaml

@@ -1,6 +1,6 @@
 # ffd-retief-hrd fanfictiondownloader
 application: fanfictiondownloader
-version: 4-4-67
+version: 4-4-71
 runtime: python27
 api_version: 1
 threadsafe: true

calibre-plugin/__init__.py

@@ -26,7 +26,7 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
     description         = 'UI plugin to download FanFiction stories from various sites.'
     supported_platforms = ['windows', 'osx', 'linux']
     author              = 'Jim Miller'
-    version             = (1, 7, 34)
+    version             = (1, 7, 42)
     minimum_calibre_version = (0, 8, 57)
 
     #: This field defines the GUI plugin class that contains all the code

calibre-plugin/config.py

@@ -169,6 +169,8 @@ class ConfigWidget(QWidget):
         prefs['updatecover'] = self.basic_tab.updatecover.isChecked()
         prefs['updateepubcover'] = self.basic_tab.updateepubcover.isChecked()
         prefs['keeptags'] = self.basic_tab.keeptags.isChecked()
+        prefs['suppressauthorsort'] = self.basic_tab.suppressauthorsort.isChecked()
+        prefs['suppresstitlesort'] = self.basic_tab.suppresstitlesort.isChecked()
         prefs['showmarked'] = self.basic_tab.showmarked.isChecked()
         prefs['urlsfromclip'] = self.basic_tab.urlsfromclip.isChecked()
         prefs['updatedefault'] = self.basic_tab.updatedefault.isChecked()
@@ -177,6 +179,7 @@ class ConfigWidget(QWidget):
         prefs['includeimages'] = self.basic_tab.includeimages.isChecked()
         prefs['lookforurlinhtml'] = self.basic_tab.lookforurlinhtml.isChecked()
         prefs['checkforseriesurlid'] = self.basic_tab.checkforseriesurlid.isChecked()
+        prefs['checkforurlchange'] = self.basic_tab.checkforurlchange.isChecked()
         prefs['injectseries'] = self.basic_tab.injectseries.isChecked()
         prefs['smarten_punctuation'] = self.basic_tab.smarten_punctuation.isChecked()
 
@@ -347,11 +350,26 @@ class BasicTab(QWidget):
         self.keeptags.setChecked(prefs['keeptags'])
         self.l.addWidget(self.keeptags)
 
+        self.suppressauthorsort = QCheckBox('Force Author into Author Sort?',self)
+        self.suppressauthorsort.setToolTip("If checked, the author(s) as given will be used for the Author Sort, too.\nIf not checked, calibre will apply it's built in algorithm which makes 'Bob Smith' sort as 'Smith, Bob', etc.")
+        self.suppressauthorsort.setChecked(prefs['suppressauthorsort'])
+        self.l.addWidget(self.suppressauthorsort)
+
+        self.suppresstitlesort = QCheckBox('Force Title into Title Sort?',self)
+        self.suppresstitlesort.setToolTip("If checked, the title as given will be used for the Title Sort, too.\nIf not checked, calibre will apply it's built in algorithm which makes 'The Title' sort as 'Title, The', etc.")
+        self.suppresstitlesort.setChecked(prefs['suppresstitlesort'])
+        self.l.addWidget(self.suppresstitlesort)
+
         self.checkforseriesurlid = QCheckBox("Check for existing Series Anthology books?",self)
         self.checkforseriesurlid.setToolTip("Check for existings Series Anthology books using each new story's series URL before downloading.\nOffer to skip downloading if a Series Anthology is found.")
         self.checkforseriesurlid.setChecked(prefs['checkforseriesurlid'])
         self.l.addWidget(self.checkforseriesurlid)
 
+        self.checkforurlchange = QCheckBox("Check for changed Story URL?",self)
+        self.checkforurlchange.setToolTip("Warn you if an update will change the URL of an existing book.")
+        self.checkforurlchange.setChecked(prefs['checkforurlchange'])
+        self.l.addWidget(self.checkforurlchange)
+
         self.lookforurlinhtml = QCheckBox("Search EPUB text for Story URL?",self)
         self.lookforurlinhtml.setToolTip("Look for first valid story URL inside EPUB text if not found in metadata.\nSomewhat risky, could find wrong URL depending on EPUB content.\nAlso finds and corrects bad ffnet URLs from ficsaver.com files.")
         self.lookforurlinhtml.setChecked(prefs['lookforurlinhtml'])
@@ -503,7 +521,7 @@ class PersonalIniTab(QWidget):
         self.ini = QTextEdit(self)
         try:
             self.ini.setFont(QFont("Courier",
-                                   self.plugin_action.gui.font().pointSize()+1));
+                                   self.plugin_action.gui.font().pointSize()+1))
         except Exception as e:
             print("Couldn't get font: %s"%e)
         self.ini.setLineWrapMode(QTextEdit.NoWrap)
@@ -539,7 +557,7 @@ class ShowDefaultsIniDialog(QDialog):
         self.ini.setToolTip("These are all of the plugin's configurable options\nand their default settings.")
         try:
             self.ini.setFont(QFont("Courier",
-                                   get_gui().font().pointSize()+1));
+                                   get_gui().font().pointSize()+1))
         except Exception as e:
             print("Couldn't get font: %s"%e)
         self.ini.setLineWrapMode(QTextEdit.NoWrap)

calibre-plugin/dialogs.py

@@ -164,7 +164,7 @@ class DroppableQTextEdit(QTextEdit):
         
     def canInsertFromMimeData(self, source):
         if source.hasUrls():
-            return True;
+            return True
         else:
             return QTextEdit.canInsertFromMimeData(self,source)
 
@@ -208,7 +208,7 @@ class AddNewDialog(SizePersistedDialog):
         self.groupbox.setChecked(False)
         self.groupbox.setFlat(True)
         print("style:%s"%self.groupbox.styleSheet())
-        self.groupbox.setStyleSheet(gpstyle);
+        self.groupbox.setStyleSheet(gpstyle)
 
         self.gbf = QFrame()
         self.gbl = QVBoxLayout()

calibre-plugin/ffdl_plugin.py

@@ -7,13 +7,14 @@ __license__   = 'GPL v3'
 __copyright__ = '2012, Jim Miller'
 __docformat__ = 'restructuredtext en'
 
-import time, os, copy, threading, re, platform
+import time, os, copy, threading, re, platform, sys
 from StringIO import StringIO
 from functools import partial
 from datetime import datetime
 from string import Template
 import urllib
 import email
+import traceback
 
 from PyQt4.Qt import (QApplication, QMenu, QToolButton, QTimer)
 
@@ -23,7 +24,7 @@ from PyQt4.QtCore import QBuffer
 from calibre.constants import numeric_version as calibre_version
 
 from calibre.ptempfile import PersistentTemporaryFile, PersistentTemporaryDirectory, remove_dir
-from calibre.ebooks.metadata import MetaInformation, authors_to_string
+from calibre.ebooks.metadata import MetaInformation
 from calibre.ebooks.metadata.meta import get_metadata
 from calibre.gui2 import error_dialog, warning_dialog, question_dialog, info_dialog
 from calibre.gui2.dialogs.message_box import ViewLog
@@ -628,7 +629,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
             return
 
         if not id_list:
-            id_list = self.gui.library_view.get_selected_ids();
+            id_list = self.gui.library_view.get_selected_ids()
         
         if len(id_list) == 0:
             self.gui.status_bar.show_message(_('No Selected Books to Update'), 3000)
@@ -918,7 +919,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
     
                 if book_id and mi: # book_id and mi only set if matched by title/author.
                     liburl = self.get_story_url(db,book_id)
-                    if book['url'] != liburl:
+                    if book['url'] != liburl and prefs['checkforurlchange']:
                         if collision in (OVERWRITE,OVERWRITEALWAYS):
                             updat="overwrit"
                         else:
@@ -976,6 +977,8 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                                 skip_date_update = True
                         elif chaptercount > urlchaptercount:
                             raise NotGoingToDownload("Existing epub contains %d chapters, web site only has %d. Use Overwrite to force update." % (chaptercount,urlchaptercount),'dialog_error.png')
+                        elif chaptercount == 0:
+                            raise NotGoingToDownload("FFDL doesn't recognize chapters in existing epub, epub is probably from a different source. Use Overwrite to force update.",'dialog_error.png')
         
                 if collision == OVERWRITE and \
                         db.has_format(book_id,formmapping[fileform],index_is_id=True):
@@ -1001,7 +1004,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                 mi = db.get_metadata(book_id,index_is_id=True)
                 if not book['series'] and mi.series != None:
                     book['calibre_series'] = (mi.series,mi.series_index)
-                    print("calibre_series:%s [%s]"%book['calibre_series'])
+                    #print("calibre_series:%s [%s]"%book['calibre_series'])
                 
         if book['good']: # there shouldn't be any !'good' books at this point.
             # if still 'good', make a temp file to write the output to.
@@ -1106,7 +1109,17 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
 
         if options['collision'] == CALIBREONLY or \
                 ( (options['updatemeta'] or book['added']) and book['good'] ):
-            self.update_metadata(db, book['calibre_id'], book, mi, options)
+            try:
+                self.update_metadata(db, book['calibre_id'], book, mi, options)
+            except:
+                det_msg = "".join(traceback.format_exception(*sys.exc_info()))+"\nStory Details:\n%s"%pretty_book(book)
+                print("Error Updating Metadata:\n%s"%det_msg)
+                error_dialog(self.gui,
+                             "Error Updating Metadata",
+                             "<p>An error has occurred while FFDL was updating calibre's metadata for <a href='%s'>%s</a>.</p>"%(book['url'],book['title'])+
+                             "The ebook has been updated, but the metadata has not.",
+                             det_msg=det_msg,
+                             show=True)
 
     def update_books_finish(self, book_list, options={}, showlist=True):
         '''Notify calibre about updated rows, update external plugins
@@ -1157,10 +1170,20 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                 self.gui.library_view.sort_by_named_field('marked', True)
                 
         self.gui.status_bar.show_message(_('Finished Adding/Updating %d books.'%(len(update_list) + len(add_list))), 3000)
-            
+
+        1+1
+        
         print("all done, remove temp dir.")
+
+        1+1
+        
         remove_dir(options['tdir'])
 
+        1+1
+        
+        print("removed temp dir.")
+
+        
         if 'Count Pages' in self.gui.iactions and len(prefs['countpagesstats']) and len(all_ids):
             cp_plugin = self.gui.iactions['Count Pages']
             cp_plugin.count_statistics(all_ids,prefs['countpagesstats'])
@@ -1368,14 +1391,23 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         oldmi = db.get_metadata(book_id,index_is_id=True)
         if prefs['keeptags']:
             old_tags = db.get_tags(book_id)
+            #print("old_tags:%s"%old_tags)
+            #print("mi.tags:%s"%mi.tags)
             # remove old Completed/In-Progress only if there's a new one.
             if 'Completed' in mi.tags or 'In-Progress' in mi.tags:
                 old_tags = filter( lambda x : x not in ('Completed', 'In-Progress'), old_tags)
                 # remove old Last Update tags if there are new ones.
             if len(filter( lambda x : not x.startswith("Last Update"), mi.tags)) > 0:
                 old_tags = filter( lambda x : not x.startswith("Last Update"), old_tags)
+                
             # mi.tags needs to be list, but set kills dups.
-            mi.tags = list(set(list(old_tags)+mi.tags))
+            # this way also removes case-mismatched dups, keeping old_tags version.
+            foldedcase_tags = dict()
+            for t in list(mi.tags) + list(old_tags):
+                foldedcase_tags[t.lower()] = t
+
+            mi.tags = foldedcase_tags.values()   
+            #print("mi.tags:%s"%mi.tags)
 
         if book['all_metadata']['langcode']:
             mi.languages=[book['all_metadata']['langcode']]
@@ -1408,6 +1440,8 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                             pass
 
         db.set_metadata(book_id,mi)
+        # mi.authors gets run through the string_to_authors and split on '&' ',' 'and' and 'with'
+        db.set_authors(book_id,book['author']) # author is a list.
 
         # do configured column updates here.
         #print("all_metadata: %s"%book['all_metadata'])
@@ -1432,7 +1466,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                 continue
             label = coldef['label']
             if coldef['datatype'] in ('enumeration','text','comments','datetime','series'):
-                db.set_custom(book_id, book['all_metadata'][meta], label=label, commit=False)
+                db.set_custom(book_id, book['all_metadata'][meta], label, commit=False)
             elif coldef['datatype'] in ('int','float'):
                 num = unicode(book['all_metadata'][meta]).replace(",","")
                 db.set_custom(book_id, num, label=label, commit=False)
@@ -1455,14 +1489,11 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                     if "," in custcol:
                         (custcol,flag) = map( lambda x: x.strip(), custcol.split(",") )
 
-                    print("meta:(%s) => custcol:(%s), flag(%s) "%(meta,custcol,flag))
-                    
                     if meta not in book['all_metadata']:
                         print("No value for %s, skipping custom column(%s) update."%(meta,custcol))
                         continue
                     
                     if custcol not in custom_columns:
-                        print("No custom column(%s), skipping."%(custcol))
                         continue
                     else:
                         coldef = custom_columns[custcol]
@@ -1499,12 +1530,26 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         # Moved down so author's already in the DB.
         if 'authorUrl' in book['all_metadata']:
             authurls = book['all_metadata']['authorUrl'].split(", ")
-            for i, auth in enumerate(book['author']):
-                #print("===Update author url for %s to %s"%(auth,authurls[i]))
-                autid=db.get_author_id(auth)
-                db.set_link_field_for_author(autid, unicode(authurls[i]),
-                                             commit=False, notify=False)
+            authorlist = [ a.replace('&',';') for a in book['author'] ]
+            if hasattr(db, 'new_api'): # new_api starts in calibre 1.0.0
+                authorids = db.new_api.get_item_ids('authors',authorlist)
+                authordata = db.new_api.author_data(authorids.values())
+                # print("\n\nauthorids:%s"%authorids)
+                # print("authordata:%s"%authordata)
 
+                author_id_to_link_map = dict()
+                for i, author in enumerate(authorlist):
+                    author_id_to_link_map[authorids[author]] = authurls[i]
+
+                # print("author_id_to_link_map:%s\n\n"%author_id_to_link_map)
+                db.new_api.set_link_for_authors(author_id_to_link_map)
+            else:
+                # keep for pre-calibre 1.0.0
+                for i, auth in enumerate(authorlist):
+                    #print("===Update author url for %s to %s"%(auth,authurls[i]))
+                    autid=db.get_author_id(auth)
+                    db.set_link_field_for_author(autid, unicode(authurls[i]),
+                                             commit=False, notify=False)
         db.commit()
 
         if 'Generate Cover' in self.gui.iactions and (book['added'] or not prefs['gcnewonly']):
@@ -1604,6 +1649,14 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
 
     def make_mi_from_book(self,book):
         mi = MetaInformation(book['title'],book['author']) # author is a list.
+        if prefs['suppressauthorsort']:
+            # otherwise author names will have calibre's sort algs
+            # applied automatically.
+            mi.author_sort = ' & '.join(book['author'])
+        if prefs['suppresstitlesort']:
+            # otherwise titles will have calibre's sort algs applied
+            # automatically.
+            mi.title_sort = book['title']
         mi.set_identifiers({'url':book['url']})
         mi.publisher = book['publisher']
         mi.tags = book['tags']
@@ -1850,7 +1903,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                 book['title'] = series
                 for sr in serieslist:
                     if series != sr:
-                        book['title'] = deftitle;
+                        book['title'] = deftitle
                         break
             
             configuration = get_ffdl_config(book['url'],fileform)
@@ -1889,3 +1942,17 @@ def split_text_to_urls(urls):
 def escapehtml(txt):
     return txt.replace("&","&amp;").replace(">","&gt;").replace("<","&lt;")
         
+def pretty_book(d, indent=0, spacer='     '):
+    kindent = spacer * indent
+
+    # if isinstance(d, list):
+    #     return '\n'.join([(pretty_book(v, indent, spacer)) for v in d])
+
+    if isinstance(d, dict):
+        for k in ('password','username'):
+            if k in d and d[k]:
+                d[k]='<was set, removed for security>'
+        return '\n'.join(['%s%s:\n%s' % (kindent, k, pretty_book(v, indent + 1, spacer)) 
+                          for k, v in d.items()])
+    return "%s%s"%(kindent, d)
+

calibre-plugin/prefs.py

@@ -30,6 +30,8 @@ default_prefs['updatemeta'] = True
 default_prefs['updatecover'] = False
 default_prefs['updateepubcover'] = False
 default_prefs['keeptags'] = False
+default_prefs['suppressauthorsort'] = False
+default_prefs['suppresstitlesort'] = False
 default_prefs['showmarked'] = False
 default_prefs['urlsfromclip'] = True
 default_prefs['updatedefault'] = True
@@ -40,6 +42,7 @@ default_prefs['adddialogstaysontop'] = False
 default_prefs['includeimages'] = False
 default_prefs['lookforurlinhtml'] = False
 default_prefs['checkforseriesurlid'] = True
+default_prefs['checkforurlchange'] = True
 default_prefs['injectseries'] = False
 default_prefs['smarten_punctuation'] = False
 

defaults.ini

@@ -16,7 +16,22 @@
 [defaults]
 
 ## [defaults] section applies to all formats and sites but may be
-## overridden at several levels
+## overridden at several levels.  Example:
+
+## [defaults]
+## titlepage_entries: category,genre, status
+## [www.whofic.com]
+## # overrides defaults.
+## titlepage_entries: category,genre, status,dateUpdated,rating
+## [epub]
+## # overrides defaults & site section
+## titlepage_entries: category,genre, status,datePublished,dateUpdated,dateCreated
+## [www.whofic.com:epub]
+## # overrides defaults, site section & format section
+## titlepage_entries: category,genre, status,datePublished
+## [overrides]
+## # overrides all other sections
+## titlepage_entries: category
 
 ## Some sites also require the user to confirm they are adult for
 ## adult content.  Uncomment by removing '#' in front of is_adult.
@@ -152,6 +167,9 @@ extratags: FanFiction
 #extraships:
 #extrawarnings:
 
+## Add this to genre if there's more than one category.
+#add_genre_when_multi_category: Crossover
+
 ## number of seconds to sleep between calls to the story site.  May by
 ## useful if pulling large numbers of stories or if the site is slow.
 #slow_down_sleep_time:0.5
@@ -259,6 +277,8 @@ sort_ships:false
 ## you added calibre_author: keep_in_order_calibre_author:true
 #keep_in_order_author:true
 
+## User-agent
+#user_agent:FFDL/1.7
 
 ## Each output format has a section that overrides [defaults]
 [html]
@@ -447,8 +467,8 @@ nook_img_fix:true
 ## mobi TOC cannot be turned off right now.
 #include_tocpage: true
 
-## Each site has a section that overrides [defaults] *and* the format
-## sections test1.com specifically is not a real story site.  Instead,
+## Each site has a section that overrides [defaults].
+## test1.com specifically is not a real story site.  Instead,
 ## it is a fake site for testing configuration and output.  It uses
 ## URLs like: http://test1.com?sid=12345
 [test1.com]
@@ -505,7 +525,8 @@ extratags: FanFiction,Testing,HTML
 #is_adult:true
 
 ## AO3 adapter defines a few extra metadata entries.
-extra_valid_entries:fandoms,freeformtags,freefromtags,ao3categories,comments,kudos,hits,bookmarks,collections
+## If there's ever more than 4 series, add series04,series04Url etc.
+extra_valid_entries:fandoms,freeformtags,freefromtags,ao3categories,comments,kudos,hits,bookmarks,collections,series00,series01,series02,series03,series00Url,series01Url,series02Url,series03Url
 fandoms_label:Fandoms
 freeformtags_label:Freeform Tags
 freefromtags_label:Freeform Tags
@@ -521,7 +542,7 @@ bookmarks_label:Bookmarks
 include_in_freefromtags:freeformtags
 
 ## adds to titlepage_entries instead of replacing it.
-#extra_titlepage_entries: fandoms,freeformtags,ao3categories,comments,kudos,hits,bookmarks
+#extra_titlepage_entries: fandoms,freeformtags,ao3categories,comments,kudos,hits,bookmarks,series00,series01,series02,series03,series00Url,series01Url,series02Url,series03Url
 
 ## adds to include_subject_tags instead of replacing it.
 #extra_subject_tags:fandoms,freeformtags,ao3categories

downloader.py

@@ -137,6 +137,9 @@ def main(argv,
    if options.update:
        try:
            (url,chaptercount) = get_dcsource_chaptercount(args[0])
+           if not url:
+               print "No story URL found in epub to update."
+               return
            print "Updating %s, URL: %s" % (args[0],url)
            output_filename = args[0]
        except:
@@ -145,8 +148,15 @@ def main(argv,
            url = args[0]
    else:
        url = args[0]
-           
-   configuration = Configuration(adapters.getConfigSectionFor(url),options.format)
+
+   try:
+       configuration = Configuration(adapters.getConfigSectionFor(url),options.format)
+   except exceptions.UnknownSite, e:
+       if options.list or options.normalize:
+           # list for page doesn't have to be a supported site.
+           configuration = Configuration("test1.com",options.format)
+       else:
+           raise e
 
    conflist = []
    homepath = join(expanduser("~"),".fanficdownloader")
@@ -199,8 +209,7 @@ def main(argv,
 
    if options.list or options.normalize:
        retlist = get_urls_from_page(args[0], configuration, normalize=options.normalize)
-       print "\n".join(retlist)
-               
+       print "\n".join(retlist)               
        return
 
    try:
@@ -258,6 +267,8 @@ def main(argv,
                print "%s already contains %d chapters." % (output_filename,chaptercount)
            elif chaptercount > urlchaptercount:
                print "%s contains %d chapters, more than source: %d." % (output_filename,chaptercount,urlchaptercount)
+           elif chaptercount == 0:
+               print "%s doesn't contain any recognizable chapters, probably from a different source.  Not updating." % (output_filename)
            else:
                print "Do update - epub(%d) vs url(%d)" % (chaptercount, urlchaptercount)
                if not options.metaonly:

fanficdownloader/adapters/__init__.py

@@ -126,6 +126,7 @@ import adapter_scarheadnet
     
 ## List of registered site adapters.
 __class_list = []
+__domain_map = {}
 
 def imports():
     for name, val in globals().items():
@@ -135,7 +136,10 @@ def imports():
 for x in imports():
     if "fanficdownloader.adapters.adapter_" in x:
         #print x
-        __class_list.append(sys.modules[x].getClass())
+        cls = sys.modules[x].getClass()
+        __class_list.append(cls)
+        for site in cls.getAcceptDomains():
+            __domain_map[site]=cls
 
 def getNormalStoryURL(url):
     if not getNormalStoryURL.__dummyconfig:
@@ -153,12 +157,14 @@ def getNormalStoryURL(url):
 # kludgey function static/singleton
 getNormalStoryURL.__dummyconfig = None
 
-def getAdapter(config,url):
+def getAdapter(config,url,anyurl=False):
 
     #logger.debug("trying url:"+url)
     (cls,fixedurl) = getClassFor(url)
     #logger.debug("fixedurl:"+fixedurl)
     if cls:
+        if anyurl:
+            fixedurl = cls.getSiteExampleURLs().split()[0]
         adapter = cls(config,fixedurl) # raises InvalidStoryURL
         return adapter
     # No adapter found.
@@ -204,12 +210,14 @@ def getClassFor(url):
         #logger.debug("trying site:www."+domain)
         cls = getClassFromList("www."+domain)
         fixedurl = fixedurl.replace("http://","http://www.")
-        
-    fixedurl = cls.stripURLParameters(fixedurl)
+
+    if cls:
+        fixedurl = cls.stripURLParameters(fixedurl)
     
     return (cls,fixedurl)
     
 def getClassFromList(domain):
-    for cls in __class_list:
-        if cls.matchesSite(domain):
-            return cls
+    try:
+        return __domain_map[domain]
+    except KeyError:
+        pass # return none.

fanficdownloader/adapters/adapter_adastrafanficcom.py

@@ -42,7 +42,7 @@ class AdAstraFanficComSiteAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -90,7 +90,7 @@ class AdAstraFanficComSiteAdapter(BaseSiteAdapter):
 
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php"))

fanficdownloader/adapters/adapter_archiveofourownorg.py

@@ -51,13 +51,13 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # get storyId from url--url validation guarantees query correct
         m = re.match(self.getSiteURLPattern(),url)
         if m:
             self.story.setMetadata('storyId',m.group('id'))
-            logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+            
             # normalized story URL.
             self._setURL('http://' + self.getSiteDomain() + '/works/'+self.story.getMetadata('storyId'))
         else:
@@ -163,7 +163,7 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('a', href=re.compile(r"^/works/\d+$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
 		
         # Find authorid and URL from... author url.
         alist = soup.findAll('a', href=re.compile(r"^/users/\w+/pseuds/\w+"))
@@ -210,10 +210,7 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
         if a != None:
             warnings = a.findAll('a',{'class':"tag"})
             for warning in warnings:
-                if warning.string == "Author Chose Not To Use Archive Warnings":
-                    warning.string = "No Archive Warnings Apply"
-                if warning.string != "No Archive Warnings Apply":
-                    self.story.addToList('warnings',warning.string)
+                self.story.addToList('warnings',warning.string)
 		
         a = metasoup.find('dd',{'class':"freeform tags"})
         if a != None:
@@ -288,19 +285,19 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
                 self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
 
 		
-        try:
-            # Find Series name from series URL.
-            a = metasoup.find('dd',{'class':"series"})
-            b = a.find('a', href=re.compile(r"/series/\d+"))
-            series_name = b.string
-            series_url = 'http://'+self.host+b['href']
-            series_index = int(a.text.split(' ')[1])
-            self.setSeries(series_name, series_index)
-            self.story.setMetadata('seriesUrl',series_url)
-            
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
+        # Find Series name from series URL.
+        ddseries = metasoup.find('dd',{'class':"series"})
+
+        if ddseries:
+            for i, a in enumerate(ddseries.findAll('a', href=re.compile(r"/series/\d+"))):
+                series_name = stripHTML(a)
+                series_url = 'http://'+self.host+a['href']
+                series_index = int(stripHTML(a.previousSibling).replace(', ','').split(' ')[1]) # "Part # of" or ", Part #"
+                self.story.setMetadata('series%02d'%i,"%s [%s]"%(series_name,series_index))
+                self.story.setMetadata('series%02dUrl'%i,series_url)
+                if i == 0:
+                    self.setSeries(series_name, series_index)
+                    self.story.setMetadata('seriesUrl',series_url)
 
     # grab the text for an individual chapter.
     def getChapterText(self, url):

fanficdownloader/adapters/adapter_archiveskyehawkecom.py

@@ -49,7 +49,7 @@ class ArchiveSkyeHawkeComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/story.php?no='+self.story.getMetadata('storyId'))
@@ -101,7 +101,7 @@ class ArchiveSkyeHawkeComAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('div', {'class':"story border"}).find('span',{'class':'left'})
-        title=a.text.split('"')[1]
+        title=stripHTML(a).split('"')[1]
         self.story.setMetadata('title',title)
         
         # Find authorid and URL from... author url.

fanficdownloader/adapters/adapter_ashwindersycophanthexcom.py

@@ -48,7 +48,7 @@ class AshwinderSycophantHexComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
 		
         
         # normalized story URL.
@@ -157,7 +157,7 @@ class AshwinderSycophantHexComAdapter(BaseSiteAdapter):
         for info in asoup.findAll('table', {'width' : '100%', 'bordercolor' : re.compile(r'#')}):
             a = info.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
             if a != None:
-                self.story.setMetadata('title',a.string)
+                self.story.setMetadata('title',stripHTML(a))
                 break
 		
 

fanficdownloader/adapters/adapter_asr3slashzoneorg.py

@@ -46,7 +46,7 @@ class Asr3SlashzoneOrgAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/archive/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -128,7 +128,7 @@ class Asr3SlashzoneOrgAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))

fanficdownloader/adapters/adapter_bloodtiesfancom.py

@@ -70,7 +70,7 @@ class BloodTiesFansComAdapter(BaseSiteAdapter): # XXX
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
@@ -207,7 +207,7 @@ class BloodTiesFansComAdapter(BaseSiteAdapter): # XXX
         
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))

fanficdownloader/adapters/adapter_buffynfaithnet.py

@@ -58,7 +58,7 @@ class BuffyNFaithNetAdapter(BaseSiteAdapter):
         m = re.match(self.getSiteURLPattern(),url)
         if m:
             self.story.setMetadata('storyId',m.group('id'))
-            logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+            
             # normalized story URL. gets rid of chapter if there, left with ch 1 URL on this site
             nurl = "http://"+self.getSiteDomain()+"/fanfictions/index.php?act=vie&id="+self.story.getMetadata('storyId')
             self._setURL(nurl)
@@ -69,7 +69,7 @@ class BuffyNFaithNetAdapter(BaseSiteAdapter):
             raise exceptions.InvalidStoryURL(url,
                                              self.getSiteDomain(),
                                              self.getSiteExampleURLs())
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','bnfnet')

fanficdownloader/adapters/adapter_castlefansorg.py

@@ -72,7 +72,7 @@ class CastleFansOrgAdapter(BaseSiteAdapter): # XXX
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
@@ -180,7 +180,7 @@ class CastleFansOrgAdapter(BaseSiteAdapter): # XXX
         pagetitle = soup.find('div',{'id':'pagetitle'})
         ## Title
         a = pagetitle.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = pagetitle.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))

fanficdownloader/adapters/adapter_chaossycophanthexcom.py

@@ -48,7 +48,7 @@ class ChaosSycophantHexComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
 		
         
         # normalized story URL.
@@ -116,7 +116,7 @@ class ChaosSycophantHexComAdapter(BaseSiteAdapter):
         ## Title
         pt = soup.find('div', {'id' : 'pagetitle'})
         a = pt.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.text)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = pt.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))

fanficdownloader/adapters/adapter_checkmatedcom.py

@@ -48,7 +48,7 @@ class CheckmatedComAdapter(BaseSiteAdapter):
 
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
 
         self._setURL('http://' + self.getSiteDomain() + '/story.php?story='+self.story.getMetadata('storyId'))
 
@@ -143,11 +143,11 @@ class CheckmatedComAdapter(BaseSiteAdapter):
         # Now go hunting for all the meta data and the chapter list.
 
         ## Title
-        a = soup.findAll('span', {'class' : 'storytitle'})
-        self.story.setMetadata('title',a[0].string)
+        a = soup.find('span', {'class' : 'storytitle'})
+        self.story.setMetadata('title',stripHTML(a))
 
         # Find authorid and URL from... author url.
-        a = a[1].find('a', href=re.compile(r"authors.php\?name\=\w+"))
+        a = a.parent.find('a', href=re.compile(r"authors.php\?name\=\w+"))
         self.story.setMetadata('authorId',a['href'].split('=')[1])
         self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
         self.story.setMetadata('author',a.string)

fanficdownloader/adapters/adapter_darksolaceorg.py

@@ -48,7 +48,7 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/elysian/viewstory.php?sid='+self.story.getMetadata('storyId'))

fanficdownloader/adapters/adapter_destinysgatewaycom.py

@@ -48,7 +48,7 @@ class DestinysGatewayComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -131,7 +131,7 @@ class DestinysGatewayComAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))

fanficdownloader/adapters/adapter_dokugacom.py

@@ -48,7 +48,7 @@ class DokugaComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[3])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
 
         # www.dokuga.com has two 'sections', shown in URL as
         # 'fanfiction' and 'spark' that change how things should be
@@ -164,7 +164,7 @@ class DokugaComAdapter(BaseSiteAdapter):
         aut.extract()
         
         a = a.string[:(len(a.string)-4)]
-        self.story.setMetadata('title',a)
+        self.story.setMetadata('title',stripHTML(a))
 
         # Find the chapters:
         chapters = soup.find('select').findAll('option')
@@ -205,7 +205,6 @@ class DokugaComAdapter(BaseSiteAdapter):
             
             a=div.text.split('Category: ')
             if len(a) == 2: self.story.addToList('category', a[1].split('&')[0])
-            self.story.addToList('category', 'Fanfiction')
         
             a=div.text.split('Created: ')
             if len(a) == 2: self.story.setMetadata('datePublished', makeDate(stripHTML(a[1].split('&')[0]), self.dateformat))

fanficdownloader/adapters/adapter_dotmoonnet.py

@@ -48,7 +48,7 @@ class DotMoonNetAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL. www.dotmoon.net/library_view.php?storyid=3
         self._setURL('http://' + self.getSiteDomain() + '/library_view.php?storyid='+self.story.getMetadata('storyId'))
@@ -141,7 +141,7 @@ class DotMoonNetAdapter(BaseSiteAdapter):
         
         ## Title
         a = body.find('b')
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url. http://www.dotmoon.net/board/index.php?action=profile;u=1'
         a = body.find('a', href=re.compile(r"index.php\?action=profile;u=\d+"))

fanficdownloader/adapters/adapter_dracoandginnycom.py

@@ -48,7 +48,7 @@ class DracoAndGinnyComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -170,7 +170,7 @@ class DracoAndGinnyComAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))

fanficdownloader/adapters/adapter_dramioneorg.py

@@ -48,7 +48,7 @@ class DramioneOrgAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -154,7 +154,7 @@ class DramioneOrgAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))

fanficdownloader/adapters/adapter_efictionestelielde.py

@@ -48,7 +48,7 @@ class EfictionEstelielDeAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -104,7 +104,7 @@ class EfictionEstelielDeAdapter(BaseSiteAdapter):
         pagetitle = soup.find('div',{'id':'pagetitle'})
         ## Title
         a = pagetitle.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = pagetitle.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))

fanficdownloader/adapters/adapter_efpfanficnet.py

@@ -48,7 +48,7 @@ class EFPFanFicNet(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -166,7 +166,7 @@ class EFPFanFicNet(BaseSiteAdapter):
         logger.debug("Normalizing to URL: "+url)
         self._setURL(url)
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
 
         # eFiction sites don't help us out a lot with their meta data
         # formating, so it's a little ugly.

fanficdownloader/adapters/adapter_erosnsapphosycophanthexcom.py

@@ -48,7 +48,7 @@ class ErosnSapphoSycophantHexComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
 		
         
         # normalized story URL.
@@ -132,7 +132,7 @@ class ErosnSapphoSycophantHexComAdapter(BaseSiteAdapter):
         ## Title
         pt = soup.find('div', {'id' : 'pagetitle'})
         a = pt.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.text)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = pt.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))

fanficdownloader/adapters/adapter_fanfictionnet.py

@@ -199,7 +199,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         for g in genrelist:
             #logger.debug("g:(%s)"%g)
             if g.strip() not in ffnetgenres:
-                logger.info("g not in ffnetgenres")
+                #logger.info("g not in ffnetgenres")
                 goodgenres=False
         if goodgenres:
             self.story.extendList('genre',genrelist)
@@ -266,11 +266,11 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         return
 
     def getChapterText(self, url):
-        logger.debug('Getting chapter text from: %s' % url)
-        time.sleep(0.5) ## ffnet(and, I assume, fpcom) tends to fail
+        time.sleep(5.0) ## ffnet(and, I assume, fpcom) tends to fail
                         ## more if hit too fast.  This is in
                         ## additional to what ever the
                         ## slow_down_sleep_time setting is.
+        logger.debug('Getting chapter text from: %s' % url)
         data = self._fetchUrl(url)
 
         if "Please email this error message in full to <a href='mailto:support@fanfiction.com'>support@fanfiction.com</a>" in data:

fanficdownloader/adapters/adapter_fanfiktionde.py

@@ -50,7 +50,7 @@ class FanFiktionDeAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/s/'+self.story.getMetadata('storyId') + '/1')

fanficdownloader/adapters/adapter_ficbooknet.py

@@ -52,7 +52,7 @@ class FicBookNetAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/readfic/'+self.story.getMetadata('storyId'))

fanficdownloader/adapters/adapter_fictionalleyorg.py

@@ -45,7 +45,7 @@ class FictionAlleyOrgSiteAdapter(BaseSiteAdapter):
         if m:
             self.story.setMetadata('authorId',m.group('auth'))
             self.story.setMetadata('storyId',m.group('id'))
-            logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+            
             # normalized story URL.
             self._setURL(url)
         else:
@@ -111,7 +111,7 @@ class FictionAlleyOrgSiteAdapter(BaseSiteAdapter):
             url=self.url
             logger.debug("Normalizing to URL: "+url)
             ## title's right there...
-            self.story.setMetadata('title',storya.string)
+            self.story.setMetadata('title',stripHTML(storya))
             data = self._fetchUrl(url)
             soup = bs.BeautifulSoup(data)
             chapterlinklist = soup.findAll('a',{'class':'chapterlink'})
@@ -121,7 +121,7 @@ class FictionAlleyOrgSiteAdapter(BaseSiteAdapter):
             ## same way.
             chapsoup = bs.BeautifulSoup(chapterdata)
             storya = chapsoup.find('div',{'class':'breadcrumbs'}).findAll('a')[1]
-            self.story.setMetadata('title',storya.string)
+            self.story.setMetadata('title',stripHTML(storya))
             del chapsoup
 
         del chapterdata

fanficdownloader/adapters/adapter_ficwadcom.py

@@ -116,7 +116,7 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
 
         # title - first h4 tag will be title.
         titleh4 = soup.find('h4')
-        self.story.setMetadata('title', titleh4.a.string)
+        self.story.setMetadata('title', stripHTML(titleh4.a))
 
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"^/author/\d+"))

fanficdownloader/adapters/adapter_fimfictionnet.py

@@ -117,7 +117,7 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
             
         ## Title
         a = soup.find('a', href=re.compile(r'^/story/'+self.story.getMetadata('storyId')))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # self.story.setMetadata("title", storyMetadata["title"])
         # if not storyMetadata["title"]:

fanficdownloader/adapters/adapter_finestoriescom.py

@@ -50,7 +50,7 @@ class FineStoriesComAdapter(BaseSiteAdapter):
         self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2].split(':')[0])
         if 'storyInfo' in self.story.getMetadata('storyId'):
             self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/s/storyInfo.php?id='+self.story.getMetadata('storyId'))
@@ -142,7 +142,7 @@ class FineStoriesComAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('a', href=re.compile(r'/s/'+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.text)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"/a/\w+"))

fanficdownloader/adapters/adapter_grangerenchantedcom.py

@@ -48,7 +48,7 @@ class GrangerEnchantedCom(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         self.section=self.parsedUrl.path.split('/',)[1]
              
         # normalized story URL.
@@ -182,7 +182,7 @@ class GrangerEnchantedCom(BaseSiteAdapter):
         
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))

fanficdownloader/adapters/adapter_harrypotterfanfictioncom.py

@@ -42,7 +42,7 @@ class HarryPotterFanFictionComSiteAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only psid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?psid='+self.story.getMetadata('storyId'))
@@ -92,7 +92,7 @@ class HarryPotterFanFictionComSiteAdapter(BaseSiteAdapter):
 
         ## Title
         a = soup.find('a', href=re.compile(r'\?psid='+self.story.getMetadata('storyId')))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         ## javascript:if (confirm('Please note. This story may contain adult themes. By clicking here you are stating that you are over 17. Click cancel if you do not meet this requirement.')) location = '?psid=290995'
         if "This story may contain adult themes." in a['href'] and not (self.is_adult or self.getConfig("is_adult")):
             raise exceptions.AdultCheckRequired(self.url)
@@ -131,55 +131,55 @@ class HarryPotterFanFictionComSiteAdapter(BaseSiteAdapter):
 
         ## Finding the metadata is a bit of a pain.  Most of the meta
         ## data is in a center.table without a bgcolor.
-        for center in soup.findAll('center'):
-            table = center.find('table',{'bgcolor':None})
-            if table:
-                metastr = stripHTML(str(table)).replace('\n',' ').replace('\t',' ')
-                # Rating: 12+ Story Reviews: 3
-                # Chapters: 3
-                # Characters: Andromeda, Ted, Bellatrix, R. Lestrange, Lucius, Narcissa, OC
-                # Genre(s): Fluff, Romance, Young Adult Era: OtherPairings: Other Pairing, Lucius/Narcissa
-                # Status: Completed
-                # First Published: 2010.09.02
-                # Last Published Chapter: 2010.09.28
-                # Last Updated: 2010.09.28
-                # Favorite Story Of: 1 users
-                # Warnings: Scenes of a Mild Sexual Nature
+        #for center in soup.findAll('center'):
+        table = soup.find('table',{'class':'storymaininfo'})
+        if table:
+            metastr = stripHTML(str(table)).replace('\n',' ').replace('\t',' ')
+            # Rating: 12+ Story Reviews: 3
+            # Chapters: 3
+            # Characters: Andromeda, Ted, Bellatrix, R. Lestrange, Lucius, Narcissa, OC
+            # Genre(s): Fluff, Romance, Young Adult Era: OtherPairings: Other Pairing, Lucius/Narcissa
+            # Status: Completed
+            # First Published: 2010.09.02
+            # Last Published Chapter: 2010.09.28
+            # Last Updated: 2010.09.28
+            # Favorite Story Of: 1 users
+            # Warnings: Scenes of a Mild Sexual Nature
 
-                m = re.match(r".*?Status: Completed.*?",metastr)
-                if m:
-                    self.story.setMetadata('status','Completed')
-                else:
-                    self.story.setMetadata('status','In-Progress')
+            m = re.match(r".*?Status: Completed.*?",metastr)
+            if m:
+                self.story.setMetadata('status','Completed')
+            else:
+                self.story.setMetadata('status','In-Progress')
 
-                m = re.match(r".*?Rating: (.+?) Story Reviews.*?",metastr)
-                if m:
-                    self.story.setMetadata('rating', m.group(1))
+            m = re.match(r".*?Rating: (.+?) Story Reviews.*?",metastr)
+            if m:
+                self.story.setMetadata('rating', m.group(1))
 
-                m = re.match(r".*?Genre\(s\): (.+?) Era.*?",metastr)
-                if m:
-                    for g in m.group(1).split(','):
-                        self.story.addToList('genre',g)
-        
-                m = re.match(r".*?Characters: (.+?) Genre.*?",metastr)
-                if m:
-                    for g in m.group(1).split(','):
-                        self.story.addToList('characters',g)
-        
-                m = re.match(r".*?Warnings: (.+).*?",metastr)
-                if m:
-                    for w in m.group(1).split(','):
-                        if w != 'Now Warnings':
-                            self.story.addToList('warnings',w)
-        
-                m = re.match(r".*?First Published: ([0-9\.]+).*?",metastr)
-                if m:
-                    self.story.setMetadata('datePublished',makeDate(m.group(1), "%Y.%m.%d"))
+            m = re.match(r".*?Genre\(s\): (.+?) Era.*?",metastr)
+            if m:
+                for g in m.group(1).split(','):
+                    self.story.addToList('genre',g)
+    
+            m = re.match(r".*?Characters: (.+?) Genre.*?",metastr)
+            if m:
+                for g in m.group(1).split(','):
+                    self.story.addToList('characters',g)
+    
+            m = re.match(r".*?Warnings: (.+).*?",metastr)
+            if m:
+                for w in m.group(1).split(','):
+                    if w != 'Now Warnings':
+                        self.story.addToList('warnings',w)
+    
+            m = re.match(r".*?First Published: ([0-9\.]+).*?",metastr)
+            if m:
+                self.story.setMetadata('datePublished',makeDate(m.group(1), "%Y.%m.%d"))
 
-                # Updated can have more than one space after it. <shrug>
-                m = re.match(r".*?Last Updated: ([0-9\.]+).*?",metastr)
-                if m:
-                    self.story.setMetadata('dateUpdated',makeDate(m.group(1), "%Y.%m.%d"))
+            # Updated can have more than one space after it. <shrug>
+            m = re.match(r".*?Last Updated: ([0-9\.]+).*?",metastr)
+            if m:
+                self.story.setMetadata('dateUpdated',makeDate(m.group(1), "%Y.%m.%d"))
 
     def getChapterText(self, url):
 

fanficdownloader/adapters/adapter_hennethannunnet.py

@@ -48,7 +48,7 @@ class HennethAnnunNetAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/stories/chapter.cfm?stid='+self.story.getMetadata('storyId'))
@@ -101,7 +101,7 @@ class HennethAnnunNetAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('h2', {'id':'page_heading'})
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
 
         # Find the chapters: chapter_view.cfm?stid=6663&spordinal=1" 
         for chapter in soup.findAll('a', href=re.compile(r'chapter_view.cfm\?stid='+self.story.getMetadata('storyId')+"&spordinal=\d+$")):

fanficdownloader/adapters/adapter_hlfictionnet.py

@@ -48,7 +48,7 @@ class HLFictionNetAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -106,7 +106,7 @@ class HLFictionNetAdapter(BaseSiteAdapter):
         self.story.setMetadata('author',aut.string)
         aut.extract()
         
-        self.story.setMetadata('title',a.string[:(len(a.string)-3)])
+        self.story.setMetadata('title',stripHTML(a)[:(len(a.string)-3)])
 
         # Find the chapters:
         chapters=soup.find('select')

fanficdownloader/adapters/adapter_hpfandomnet.py

@@ -52,7 +52,7 @@ class HPFandomNetAdapterAdapter(BaseSiteAdapter): # XXX
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         # XXX Most sites don't have the /eff part.  Replace all to remove it usually.

fanficdownloader/adapters/adapter_hpfanficarchivecom.py

@@ -37,11 +37,6 @@ class HPFanficArchiveComAdapter(BaseSiteAdapter):
     def __init__(self, config, url):
         BaseSiteAdapter.__init__(self, config, url)
 
-        # hpfanficarchive.com blocks the default user-agent.  However,
-        # when asked, they said it was just general anti-spam, not
-        # targeted at us.  That lets me do this in good conscience:
-        self.opener.addheaders = [('User-agent', 'FFDL/1.7')]
-        
         self.decode = ["Windows-1252",
                        "utf8"] # 1252 is a superset of iso-8859-1.
                                # Most sites that claim to be
@@ -53,7 +48,7 @@ class HPFanficArchiveComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/stories/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -105,7 +100,7 @@ class HPFanficArchiveComAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))

fanficdownloader/adapters/adapter_iketernalnet.py

@@ -48,7 +48,7 @@ class IkEternalNetAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -183,7 +183,7 @@ class IkEternalNetAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))

fanficdownloader/adapters/adapter_imagineeficcom.py

@@ -48,7 +48,7 @@ class ImagineEFicComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -170,7 +170,7 @@ class ImagineEFicComAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))

fanficdownloader/adapters/adapter_indeathnet.py

@@ -51,7 +51,7 @@ class InDeathNetAdapter(BaseSiteAdapter):
         m = re.match(self.getSiteURLPattern(),url)
         if m:
             self.story.setMetadata('storyId',m.group('id'))
-            logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+            
             # normalized story URL.
             self._setURL('http://www.' + self.getSiteDomain() + '/blog/archive/'+self.story.getMetadata('storyId')+'-'+m.group('name')+'/')
         else:
@@ -109,7 +109,7 @@ class InDeathNetAdapter(BaseSiteAdapter):
         ## Title
         h = soup.find('a', id="blog_title")
         t = h.find('span')
-        self.story.setMetadata('title',t.contents[0].string.strip())
+        self.story.setMetadata('title',stripHTML(t.contents[0]).strip())
 		
         s = t.find('div')        
         if s != None:
@@ -122,9 +122,6 @@ class InDeathNetAdapter(BaseSiteAdapter):
         self.story.setMetadata('authorUrl',a['href'])
         self.story.setMetadata('author',m.group('name'))
 
-
-
-
         # Find the chapters:
         chapters=soup.findAll('a', title="View entry", href=re.compile(r'http://www.indeath.net/blog/'+self.story.getMetadata('storyId')+"/entry\-(\d+)\-([^/]*)/$"))
 

fanficdownloader/adapters/adapter_ksarchivecom.py

@@ -52,7 +52,7 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.

fanficdownloader/adapters/adapter_libraryofmoriacom.py

@@ -49,7 +49,7 @@ class LibraryOfMoriaComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/a/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -129,7 +129,7 @@ class LibraryOfMoriaComAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))

fanficdownloader/adapters/adapter_lumossycophanthexcom.py

@@ -48,7 +48,7 @@ class LumosSycophantHexComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
 		
         
         # normalized story URL.
@@ -116,7 +116,7 @@ class LumosSycophantHexComAdapter(BaseSiteAdapter):
         ## Title
         pt = soup.find('div', {'id' : 'pagetitle'})
         a = pt.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.text)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = pt.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))

fanficdownloader/adapters/adapter_mediaminerorg.py

@@ -43,7 +43,7 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
         m = re.match(self.getSiteURLPattern(),url)
         if m:
             self.story.setMetadata('storyId',m.group('id'))
-            logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+            
             # normalized story URL.
             self._setURL('http://' + self.getSiteDomain() + '/fanfic/view_st.php/'+self.story.getMetadata('storyId'))
         else:
@@ -107,10 +107,10 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
         for font in title.findAll('font'):
             font.extract() # removes 'font' tags from inside the td.        
         if title.has_key('colspan'):
-            titlet = title.text
+            titlet = stripHTML(title)
         else:
             ## No colspan, it's part chapter title--even if it's a one-shot.
-            titlet = ':'.join(title.text.split(':')[:-1]) # strip trailing 'Chapter X' or chapter title
+            titlet = ':'.join(stripHTML(title).split(':')[:-1]) # strip trailing 'Chapter X' or chapter title
         self.story.setMetadata('title',titlet)
         ## The story title is difficult to reliably parse from the
         ## story pages.  Getting it from the author page is, but costs

fanficdownloader/adapters/adapter_merlinficdtwinscouk.py

@@ -48,7 +48,7 @@ class MerlinFicDtwinsCoUk(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -170,7 +170,7 @@ class MerlinFicDtwinsCoUk(BaseSiteAdapter):
         
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))

fanficdownloader/adapters/adapter_midnightwhispersca.py

@@ -52,7 +52,7 @@ class MidnightwhispersCaAdapter(BaseSiteAdapter): # XXX
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.

fanficdownloader/adapters/adapter_mugglenetcom.py

@@ -52,7 +52,7 @@ class MuggleNetComAdapter(BaseSiteAdapter): # XXX
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -193,7 +193,7 @@ class MuggleNetComAdapter(BaseSiteAdapter): # XXX
         
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))

fanficdownloader/adapters/adapter_nationallibrarynet.py

@@ -48,7 +48,7 @@ class NationalLibraryNetAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only storyid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?storyid='+self.story.getMetadata('storyId'))
@@ -104,7 +104,7 @@ class NationalLibraryNetAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('h1')
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"authorresults.php\?author=\d+"))

fanficdownloader/adapters/adapter_ncisficcom.py

@@ -48,7 +48,7 @@ class NCISFicComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only storyid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?storyid='+self.story.getMetadata('storyId'))
@@ -103,7 +103,7 @@ class NCISFicComAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('h1')
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"authorresults.php\?author=\d+"))

fanficdownloader/adapters/adapter_ncisfictionnet.py

@@ -37,11 +37,6 @@ class NCISFictionNetAdapter(BaseSiteAdapter):
     def __init__(self, config, url):
         BaseSiteAdapter.__init__(self, config, url)
 
-        # ncisfiction.net blocks the default user-agent.  However,
-        # when asked, they said it was just general anti-spam, not
-        # targeted at us.  That lets me do this in good conscience:
-        self.opener.addheaders = [('User-agent', 'FFDL/1.7')]
-        
         self.decode = ["iso-8859-1",
                        "Windows-1252"] # 1252 is a superset of iso-8859-1.
                                # Most sites that claim to be
@@ -53,7 +48,7 @@ class NCISFictionNetAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL("http://"+self.getSiteDomain()\

fanficdownloader/adapters/adapter_netraptororg.py

@@ -46,7 +46,7 @@ class NetRaptorOrgAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/fanfiction/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -97,7 +97,7 @@ class NetRaptorOrgAdapter(BaseSiteAdapter):
         ## Title
         pagetitle = soup.find('div',{'id':'pagetitle'})
         a = pagetitle.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = pagetitle.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))

fanficdownloader/adapters/adapter_nfacommunitycom.py

@@ -52,7 +52,7 @@ class NfaCommunityComAdapter(BaseSiteAdapter): # XXX
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
@@ -156,7 +156,7 @@ class NfaCommunityComAdapter(BaseSiteAdapter): # XXX
         
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))

fanficdownloader/adapters/adapter_nhamagicalworldsus.py

@@ -48,7 +48,7 @@ class NHAMagicalWorldsUsAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
 		
         
         # normalized story URL.
@@ -115,7 +115,7 @@ class NHAMagicalWorldsUsAdapter(BaseSiteAdapter):
             a = info.find('a')
             if 'viewstory.php?sid='+self.story.getMetadata('storyId') == a['href'] or \
                     ('viewstory.php?sid='+self.story.getMetadata('storyId')+'&') in a['href']:
-                self.story.setMetadata('title',a.string)
+                self.story.setMetadata('title',stripHTML(a))
                 break
 		
 

fanficdownloader/adapters/adapter_nickandgregnet.py

@@ -48,7 +48,7 @@ class NickAndGregNetAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
@@ -101,7 +101,7 @@ class NickAndGregNetAdapter(BaseSiteAdapter):
 
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))

fanficdownloader/adapters/adapter_occlumencysycophanthexcom.py

@@ -48,7 +48,7 @@ class OcclumencySycophantHexComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
 		
         
         # normalized story URL.
@@ -158,7 +158,7 @@ class OcclumencySycophantHexComAdapter(BaseSiteAdapter):
         for info in asoup.findAll('table', {'class' : 'border'}):
             a = info.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
             if a != None:
-                self.story.setMetadata('title',a.text)
+                self.story.setMetadata('title',stripHTML(a))
                 break
 		
 

fanficdownloader/adapters/adapter_onedirectionfanfictioncom.py

@@ -48,7 +48,7 @@ class OneDirectionFanfictionComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -158,7 +158,7 @@ class OneDirectionFanfictionComAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))

fanficdownloader/adapters/adapter_phoenixsongnet.py

@@ -48,7 +48,7 @@ class PhoenixSongNetAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[3])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/fanfiction/story/' +self.story.getMetadata('storyId')+'/')
@@ -136,7 +136,7 @@ class PhoenixSongNetAdapter(BaseSiteAdapter):
         ## Title
         b = soup.find('div', {'id' : 'nav25'})
         a = b.find('a', href=re.compile(r'fanfiction/story/'+self.story.getMetadata('storyId')+"/$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.  /fanfiction/stories.php?psid=125
         a = b.find('a', href=re.compile(r"/fanfiction/stories.php\?psid=\d+"))

fanficdownloader/adapters/adapter_pommedesangcom.py

@@ -48,7 +48,7 @@ class PommeDeSangComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # pommedesang.com has two 'sections', shown in URL as
         # 'efiction' and 'sds' that change how things should be
@@ -180,7 +180,7 @@ class PommeDeSangComAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('a', href=re.compile('viewstory.php\?sid=\d+'))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))

fanficdownloader/adapters/adapter_ponyfictionarchivenet.py

@@ -48,8 +48,7 @@ class PonyFictionArchiveNetAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-        logger.info(self.parsedUrl.netloc)
+        
         # normalized story URL.
         if "explicit" in self.parsedUrl.netloc:
             self._setURL('http://explicit.' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -137,7 +136,7 @@ class PonyFictionArchiveNetAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))

fanficdownloader/adapters/adapter_portkeyorg.py

@@ -55,7 +55,7 @@ class PortkeyOrgAdapter(BaseSiteAdapter): # XXX
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/story/'+self.story.getMetadata('storyId'))

fanficdownloader/adapters/adapter_potionsandsnitchesnet.py

@@ -41,7 +41,7 @@ class PotionsAndSnitchesNetSiteAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/fanfiction/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -83,7 +83,7 @@ class PotionsAndSnitchesNetSiteAdapter(BaseSiteAdapter):
 
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))

fanficdownloader/adapters/adapter_potterficscom.py

@@ -54,7 +54,7 @@ class PotterFicsComAdapter(BaseSiteAdapter):
         m = re.match(self.getSiteURLPattern(),url)
         if m:
             self.story.setMetadata('storyId',m.group('id'))
-            logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+            
             # normalized story URL. gets rid of chapter if there, left with chapter index URL
             nurl = "http://"+self.getSiteDomain()+"/historias/"+self.story.getMetadata('storyId')
             self._setURL(nurl)
@@ -62,7 +62,7 @@ class PotterFicsComAdapter(BaseSiteAdapter):
             raise exceptions.InvalidStoryURL(url,
                                              self.getSiteDomain(),
                                              self.getSiteExampleURLs())
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','potficscom')
@@ -147,7 +147,7 @@ class PotterFicsComAdapter(BaseSiteAdapter):
         #find first metadata block
         mb = cell.div.findNextSibling('div')
         #Get meta...
-        self.story.setMetadata('title', mb.b.string)
+        self.story.setMetadata('title', stripHTML(mb.b))
         #strip out brackets on rating
         self.story.setMetadata('rating', mb.span.string[1:-1])
         #Completion status is denoted by the presence of this image:

fanficdownloader/adapters/adapter_potterheadsanonymouscom.py

@@ -48,7 +48,7 @@ class PotterHeadsAnonymousComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -181,7 +181,7 @@ class PotterHeadsAnonymousComAdapter(BaseSiteAdapter):
         
         ## Title
         a = pagetitle.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = pagetitle.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))

fanficdownloader/adapters/adapter_pretendercentrecom.py

@@ -48,7 +48,7 @@ class PretenderCenterComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/missingpieces/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -134,7 +134,7 @@ class PretenderCenterComAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.text)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))

fanficdownloader/adapters/adapter_psychficcom.py

@@ -48,7 +48,7 @@ class PsychFicComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -130,7 +130,7 @@ class PsychFicComAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))

fanficdownloader/adapters/adapter_qafficcom.py

@@ -48,7 +48,7 @@ class QafFicComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/atp/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -138,7 +138,7 @@ class QafFicComAdapter(BaseSiteAdapter):
         self.story.setMetadata('author',aut.string)
         aut.extract()
         
-        self.story.setMetadata('title',a.string[:(len(a.string)-3)])
+        self.story.setMetadata('title',stripHTML(a)[:(len(a.string)-3)])
 
         # Find the chapters:
         chapters=soup.find('select')

fanficdownloader/adapters/adapter_restrictedsectionorg.py

@@ -52,10 +52,9 @@ class RestrictedSectionOrgSiteAdapter(BaseSiteAdapter):
         m = re.match(self.getSiteURLPattern(),url)
         if m:
             self.story.setMetadata('storyId',m.group('id'))
-            logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+            
             # normalized story URL.
             self._setURL('http://' + self.getSiteDomain() + '/' + m.group('filestory') + '.php?' + m.group('filestory') + '=' + self.story.getMetadata('storyId'))
-            logger.debug("storyUrl: (%s)"%self.story.getMetadata('storyUrl'))
         else:
             raise exceptions.InvalidStoryURL(url,
                                              self.getSiteDomain(),
@@ -127,10 +126,12 @@ class RestrictedSectionOrgSiteAdapter(BaseSiteAdapter):
         
         self.story.setMetadata('authorId',ahref.split('=')[1])
         self.story.setMetadata('authorUrl','http://'+self.host+'/'+ahref)
-        self.story.setMetadata('author',a.string)
+        self.story.setMetadata('author',stripHTML(a))
 
         # title, remove byauthorname.
-        self.story.setMetadata('title',h2.text[:h2.text.index("by"+a.string)])
+        auth=stripHTML(a)
+        title=stripHTML(h2)
+        self.story.setMetadata('title',title[:title.index(" by "+auth)])
 
         dates = soup.findAll('span', {'class':'date'})
         if dates: # only for multi-chapter

fanficdownloader/adapters/adapter_samdeanarchivenu.py

@@ -48,7 +48,7 @@ class SamDeanArchiveNuAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -110,7 +110,7 @@ class SamDeanArchiveNuAdapter(BaseSiteAdapter):
         self.story.setMetadata('author',aut.string)
         aut.extract()
         
-        self.story.setMetadata('title',a.string[:(len(a.string)-3)])
+        self.story.setMetadata('title',stripHTML(a)[:(len(stripHTML(a))-3)])
 
         # Find the chapters:
         chapters=soup.find('select')

fanficdownloader/adapters/adapter_scarheadnet.py

@@ -48,7 +48,7 @@ class ScarHeadNetAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -181,7 +181,7 @@ class ScarHeadNetAdapter(BaseSiteAdapter):
         
         ## Title
         a = pagetitle.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = pagetitle.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))

fanficdownloader/adapters/adapter_scarvesandcoffeenet.py

@@ -48,7 +48,7 @@ class ScarvesAndCoffeeNetAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -139,7 +139,7 @@ class ScarvesAndCoffeeNetAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('div',{"id":"pagetitle"}).find('a', href=re.compile(r"viewuser.php\?uid=\d+"))

fanficdownloader/adapters/adapter_sg1heliopoliscom.py

@@ -48,7 +48,7 @@ class SG1HeliopolisComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         self.section=self.parsedUrl.path.split('/',)[1]
         
         # normalized story URL.
@@ -140,7 +140,7 @@ class SG1HeliopolisComAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))

fanficdownloader/adapters/adapter_simplyundeniablecom.py

@@ -48,7 +48,7 @@ class SimplyUndeniableComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -143,7 +143,7 @@ class SimplyUndeniableComAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('h1')
-        self.story.setMetadata('title',a.text)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -155,7 +155,7 @@ class SimplyUndeniableComAdapter(BaseSiteAdapter):
         for info in asoup.findAll('table', {'cellpadding' : '5'}):
             a = info.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
             if a != None:
-                self.story.setMetadata('title',a.string)
+                self.story.setMetadata('title',stripHTML(a))
                 break
                 
         # Find the chapters:

fanficdownloader/adapters/adapter_sinfuldesireorg.py

@@ -48,7 +48,7 @@ class SinfulDesireOrgAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/archive/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -130,7 +130,7 @@ class SinfulDesireOrgAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))

fanficdownloader/adapters/adapter_siyecouk.py

@@ -49,7 +49,7 @@ class SiyeCoUkAdapter(BaseSiteAdapter): # XXX
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/siye/viewstory.php?sid='+self.story.getMetadata('storyId'))

fanficdownloader/adapters/adapter_squidgeorgpeja.py

@@ -59,7 +59,7 @@ class SquidgeOrgPejaAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/peja/cgi-bin/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -122,7 +122,7 @@ class SquidgeOrgPejaAdapter(BaseSiteAdapter):
         # formating, so it's a little ugly.
 
         title = authorSoup.find('a',{'href':'viewstory.php?sid='+self.story.getMetadata('storyId')})
-        self.story.setMetadata('title',title.string)
+        self.story.setMetadata('title',stripHTML(title))
         titleblock=title.parent.parent
         
         chapterselect=soup.find('select',{'name':'chapter'})

fanficdownloader/adapters/adapter_stargateatlantisorg.py

@@ -48,7 +48,7 @@ class StargateAtlantisOrgAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/fanfics/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -106,7 +106,7 @@ class StargateAtlantisOrgAdapter(BaseSiteAdapter):
         self.story.setMetadata('author',aut.string)
         aut.extract()
         
-        self.story.setMetadata('title',a.string[:(len(a.string)-3)])
+        self.story.setMetadata('title',stripHTML(a)[:(len(stripHTML(a))-3)])
 
         # Find the chapters:
         chapters=soup.findAll('div', {'class' : 'content'})

fanficdownloader/adapters/adapter_storiesofardacom.py

@@ -48,7 +48,7 @@ class StoriesOfArdaComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/chapterlistview.asp?SID='+self.story.getMetadata('storyId'))
@@ -104,7 +104,7 @@ class StoriesOfArdaComAdapter(BaseSiteAdapter):
         asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
         
         a.find('em').extract()
-        self.story.setMetadata('title',a.text)
+        self.story.setMetadata('title',stripHTML(a))
 
         # Find the chapters: chapterview.asp?sid=7000&cid=30919
         chapters=soup.findAll('a', href=re.compile(r'chapterview.asp\?sid='+self.story.getMetadata('storyId')+"&cid=\d+$"))

fanficdownloader/adapters/adapter_tenhawkpresentscom.py

@@ -44,7 +44,7 @@ class TenhawkPresentsComSiteAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -133,7 +133,7 @@ class TenhawkPresentsComSiteAdapter(BaseSiteAdapter):
 
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))

fanficdownloader/adapters/adapter_thealphagatecom.py

@@ -48,7 +48,7 @@ class TheAlphaGateComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -99,7 +99,7 @@ class TheAlphaGateComAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))

fanficdownloader/adapters/adapter_thehexfilesnet.py

@@ -48,7 +48,7 @@ class TheHexFilesNetAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))

fanficdownloader/adapters/adapter_thehookupzonenet.py

@@ -72,7 +72,7 @@ class TheHookupZoneNetAdapter(BaseSiteAdapter): # XXX
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
@@ -179,7 +179,7 @@ class TheHookupZoneNetAdapter(BaseSiteAdapter): # XXX
         
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))

fanficdownloader/adapters/adapter_themasquenet.py

@@ -48,7 +48,7 @@ class TheMasqueNetAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         if self.parsedUrl.path.split('/',)[1] == 'wiktt':
             self.story.addToList("category","Harry Potter")
@@ -177,7 +177,7 @@ class TheMasqueNetAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))

fanficdownloader/adapters/adapter_thepetulantpoetesscom.py

@@ -48,7 +48,7 @@ class ThePetulantPoetessComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId') +'&i=1')
@@ -165,7 +165,7 @@ class ThePetulantPoetessComAdapter(BaseSiteAdapter):
             for info in asoup.findAll('td', {'class' : 'highlightcolor1'}):
                 a = info.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
                 if a != None:
-                    self.story.setMetadata('title',a.text)                
+                    self.story.setMetadata('title',stripHTML(a))
                     found = 1
                     break
             index=index+1

fanficdownloader/adapters/adapter_thequidditchpitchorg.py

@@ -52,7 +52,7 @@ class TheQuidditchPitchOrgAdapter(BaseSiteAdapter): # XXX
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         # XXX Most sites don't have the  part.  Replace all to remove it usually.
@@ -165,7 +165,7 @@ class TheQuidditchPitchOrgAdapter(BaseSiteAdapter): # XXX
         
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))

fanficdownloader/adapters/adapter_thewriterscoffeeshopcom.py

@@ -44,7 +44,7 @@ class TheWritersCoffeeShopComSiteAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/library/viewstory.php?sid='+self.story.getMetadata('storyId'))

fanficdownloader/adapters/adapter_tokrafandomnetcom.py

@@ -48,7 +48,7 @@ class TokraFandomnetComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -131,7 +131,7 @@ class TokraFandomnetComAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))

fanficdownloader/adapters/adapter_tthfanficorg.py

@@ -41,7 +41,7 @@ class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter):
         m = re.match(self.getSiteURLPattern(),url)
         if m:
             self.story.setMetadata('storyId',m.group('id'))
-            logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+            
             # normalized story URL.
             self._setURL("http://"+self.getSiteDomain()\
                          +"/Story-"+self.story.getMetadata('storyId'))

fanficdownloader/adapters/adapter_twilightarchivescom.py

@@ -48,7 +48,7 @@ class TwilightArchivesComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL. http://www.twilightarchives.com/read/9353
         self._setURL('http://' + self.getSiteDomain() + '/read/'+self.story.getMetadata('storyId'))
@@ -100,7 +100,7 @@ class TwilightArchivesComAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('h1')
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find the chapters:
         chapters=soup.find('ol', {'class' : 'chapters'})

fanficdownloader/adapters/adapter_twilightednet.py

@@ -43,7 +43,7 @@ class TwilightedNetSiteAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -130,7 +130,7 @@ class TwilightedNetSiteAdapter(BaseSiteAdapter):
 
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php"))

fanficdownloader/adapters/adapter_twiwritenet.py

@@ -44,7 +44,7 @@ class TwiwriteNetSiteAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -144,7 +144,7 @@ class TwiwriteNetSiteAdapter(BaseSiteAdapter):
         
         ## Title
         a = pagetitlediv.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = pagetitlediv.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))

fanficdownloader/adapters/adapter_walkingtheplankorg.py

@@ -46,7 +46,7 @@ class WalkingThePlankOrgAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/archive/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -115,7 +115,7 @@ class WalkingThePlankOrgAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))

fanficdownloader/adapters/adapter_whoficcom.py

@@ -70,7 +70,7 @@ class WhoficComSiteAdapter(BaseSiteAdapter):
                 raise e
 
         # pull title(title) and author from the HTML title.
-        title = soup.find('title').string
+        title = stripHTML(soup.find('title'))
         logger.debug('Title: %s' % title)
         title = title.split('::')[1].strip()
         self.story.setMetadata('title',title.split(' by ')[0].strip())

fanficdownloader/adapters/adapter_wizardtalesnet.py

@@ -48,7 +48,7 @@ class WizardTalesNetAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -171,7 +171,7 @@ class WizardTalesNetAdapter(BaseSiteAdapter):
         ## Title
         pt = soup.find('div', {'id' : 'pagetitle'})
         a = pt.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.text)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = pt.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))

fanficdownloader/adapters/adapter_wolverineandroguecom.py

@@ -48,7 +48,7 @@ class WolverineAndRogueComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/wrfa/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -101,7 +101,7 @@ class WolverineAndRogueComAdapter(BaseSiteAdapter):
         ## Title
         pt = soup.find('div', {'id' : 'pagetitle'})
         a = pt.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.text)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = pt.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))

fanficdownloader/adapters/adapter_wraithbaitcom.py

@@ -49,7 +49,7 @@ class WraithBaitComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
 
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -113,7 +113,7 @@ class WraithBaitComAdapter(BaseSiteAdapter):
         ## Title
         pt = soup.find('div', {'id' : 'pagetitle'})
         a = pt.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         alist = pt.findAll('a', href=re.compile(r"viewuser.php\?uid=\d+"))

fanficdownloader/adapters/base_adapter.py

@@ -77,6 +77,8 @@ class BaseSiteAdapter(Configurable):
         self.is_adult=False
         
         self.opener = u2.build_opener(u2.HTTPCookieProcessor(),GZipProcessor())
+        ## Specific UA because too many sites are blocking the default python UA.
+        self.opener.addheaders = [('User-agent', self.getConfig('user_agent'))]
         self.storyDone = False
         self.metadataDone = False
         self.story = Story(configuration)