Bump versions.

meta section, fix highbyte chars in description.

app.yaml

@@ -1,6 +1,6 @@
 # ffd-retief-hrd fanfictiondownloader
 application: fanfictiondownloader
-version: 4-4-57
+version: 4-4-62
 runtime: python27
 api_version: 1
 threadsafe: true

calibre-plugin/__init__.py

@@ -4,7 +4,7 @@ from __future__ import (unicode_literals, division, absolute_import,
                         print_function)
 
 __license__   = 'GPL v3'
-__copyright__ = '2011, Jim Miller'
+__copyright__ = '2013, Jim Miller'
 __docformat__ = 'restructuredtext en'
 
 # The class that all Interface Action plugin wrappers must inherit from
@@ -26,7 +26,7 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
     description         = 'UI plugin to download FanFiction stories from various sites.'
     supported_platforms = ['windows', 'osx', 'linux']
     author              = 'Jim Miller'
-    version             = (1, 7, 24)
+    version             = (1, 7, 29)
     minimum_calibre_version = (0, 8, 57)
 
     #: This field defines the GUI plugin class that contains all the code

calibre-plugin/config.py

@@ -15,8 +15,8 @@ from PyQt4.Qt import (QDialog, QWidget, QVBoxLayout, QHBoxLayout, QLabel,
                       QCheckBox, QPushButton, QTabWidget, QVariant, QScrollArea,
                       QDialogButtonBox )
 
-from calibre.utils.config import JSONConfig
 from calibre.gui2.ui import get_gui
+from calibre.gui2 import dynamic, info_dialog
 
 from calibre_plugins.fanfictiondownloader_plugin.prefs import prefs, PREFS_NAMESPACE
 from calibre_plugins.fanfictiondownloader_plugin.dialogs \

calibre-plugin/ffdl_plugin.py

@@ -862,11 +862,25 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                                                "<p>Click '<b>Yes</b>' to %se book with new URL.</p>"%updat+
                                                "<p>Click '<b>No</b>' to skip %sing this book.</p>"%updat,
                                                show_copy_button=False):
-                            book['comment'] = "Update declined by user due to differing story URL(%s)"%liburl
-                            book['good']=False
-                            book['icon']='rotate-right.png'
-                            book['status'] = 'Different URL'
-                            return
+                            if question_dialog(self.gui, 'Download as New Book?',
+                                               '<h3>Download as New Book?</h3>'+
+                                               '<p><b>%s</b> by <b>%s</b> is already in your library with a different source URL.</p>'%
+                                               (mi.title,', '.join(mi.author))+
+                                               '<p>You chose not to update the existing book.  Do you want to add a new book for this URL?</p>'+
+                                               '<p>New URL: <a href="%(newurl)s">%(newurl)s</a></p>'%
+                                               {'newurl':book['url']}+
+                                               "<p>Click '<b>Yes</b>' to a new book with new URL.</p>"+
+                                               "<p>Click '<b>No</b>' to skip URL.</p>",
+                                               show_copy_button=False):
+                                book_id = None
+                                mi = None
+                                book['calibre_id'] = None
+                            else:
+                                book['comment'] = "Update declined by user due to differing story URL(%s)"%liburl
+                                book['good']=False
+                                book['icon']='rotate-right.png'
+                                book['status'] = 'Different URL'
+                                return
         
             if book_id != None and collision != ADDNEW:
                 if collision in (CALIBREONLY):
@@ -1309,15 +1323,6 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                 except:
                     print("Failed to set_cover, skipping")
 
-        # set author link if found.  All current adapters have authorUrl, except anonymous on AO3.
-        if 'authorUrl' in book['all_metadata']:
-            authurls = book['all_metadata']['authorUrl'].split(", ")
-            for i, auth in enumerate(book['author']):
-                #print("===Update author url for %s to %s"%(auth,authurls[i]))
-                autid=db.get_author_id(auth)
-                db.set_link_field_for_author(autid, unicode(authurls[i]),
-                                             commit=False, notify=False)
-
         # implement 'newonly' flags here by setting to the current
         # value again.
         if not book['added']:
@@ -1395,7 +1400,11 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
 
                     if flag == 'r' or book['added']: # flag 'n' isn't actually needed--*always* set if configured and new book.
                         if coldef['datatype'] in ('int','float'): # for favs, etc--site specific metadata.
-                            val = unicode(book['all_metadata'][meta]).replace(",","")
+                            if 'anthology_meta_list' in book and meta in book['anthology_meta_list']:
+                                # re-split list, strip commas, convert to floats, sum up.
+                                val = sum([ float(x.replace(",","")) for x in book['all_metadata'][meta].split(", ") ])
+                            else:
+                                val = unicode(book['all_metadata'][meta]).replace(",","")
                         else:
                             val = book['all_metadata'][meta]
                         db.set_custom(book_id, val, label=label, commit=False)
@@ -1416,6 +1425,16 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                             
                         db.set_custom(book_id, ", ".join(vallist), label=label, commit=False)
 
+        # set author link if found.  All current adapters have authorUrl, except anonymous on AO3.
+        # Moved down so author's already in the DB.
+        if 'authorUrl' in book['all_metadata']:
+            authurls = book['all_metadata']['authorUrl'].split(", ")
+            for i, auth in enumerate(book['author']):
+                #print("===Update author url for %s to %s"%(auth,authurls[i]))
+                autid=db.get_author_id(auth)
+                db.set_link_field_for_author(autid, unicode(authurls[i]),
+                                             commit=False, notify=False)
+
         db.commit()
 
         if 'Generate Cover' in self.gui.iactions and (book['added'] or not prefs['gcnewonly']):
@@ -1649,7 +1668,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                     # identifiers have :->| in uri.
                     # print("uri from ident uri:%s"%identifiers['uri'].replace('|',':'))
                     return identifiers['uri'].replace('|',':')
-            elif path.lower().endswith('.epub'):
+            elif path and path.lower().endswith('.epub'):
                 existingepub = path
                 
             ## only epub has URL in it--at least where I can easily find it.
@@ -1674,6 +1693,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         book['tags'] = []
         book['url'] = ''
         book['all_metadata'] = {}
+        book['anthology_meta_list'] = {}
         book['comment'] = ''
         book['added'] = True
         book['good'] = True
@@ -1740,6 +1760,10 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                         book['all_metadata'][k]=book['all_metadata'][k]+"\n\n"+v
                     else:
                         book['all_metadata'][k]=book['all_metadata'][k]+", "+v
+                        # flag psuedo list element.  Used so numeric
+                        # cust cols can convert back to numbers and
+                        # add.
+                        book['anthology_meta_list'][k]=True
     
         if existingbook:
             book['title'] = deftitle = existingbook['title']

defaults.ini

@@ -1,4 +1,4 @@
-# Copyright 2012 Fanficdownloader team
+# Copyright 2013 Fanficdownloader team
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -191,6 +191,10 @@ extratags: FanFiction
 ## summary.
 keep_summary_html:true
 
+## If set true, any style attributes on tags in the story HTML will be
+## kept.  Useful for keeping extra colors & formatting from original.
+#keep_style_attr: false
+
 ## Don't like the numbers at the start of chapter titles on some
 ## sites?  You can use strip_chapter_numbers to strip them off.  Just
 ## want to make them all look the same?  Strip them off, then add them
@@ -618,7 +622,7 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 ## composite metadata entries.  dramione.org, for example, adds
 ## 'cliches' and then defines as the composite of hermiones,dracos in
 ## include_in_cliches.
-extra_valid_entries:themes,hermiones,dracos,timeline,cliches
+extra_valid_entries:themes,hermiones,dracos,timeline,cliches,read,reviews
 include_in_cliches:hermiones,dracos
 
 ## For another example, you could, by uncommenting this line, include
@@ -663,6 +667,11 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter
 
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -670,11 +679,22 @@ extracategories:Harry Potter
 #username:YourName
 #password:yourpassword
 
-## Some sites also require the user to confirm they are adult for
-## adult content.  In commandline version, this should go in your
-## personal.ini, not defaults.ini.
+[fanfic.potterheadsanonymous.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
 [fanfiction.portkey.org]
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter
@@ -718,6 +738,10 @@ extracharacters:Hermione Granger
 ## personal.ini, not defaults.ini.
 #is_adult:true
 
+## Extra metadata that this adapter knows about.  See [dramione.org]
+## for examples of how to use them.
+extra_valid_entries:read,reviews
+
 [hlfiction.net]
 ## Site dedicated to these categories/characters/ships
 extracategories:Highlander
@@ -1175,6 +1199,11 @@ cover_exclusion_regexp:/stories/999/images/.*?_trophy.png
 ## Site dedicated to these categories/characters/ships
 extracategories:NCIS
 
+[www.nickandgreg.net]
+## Site dedicated to these categories/characters/ships
+extracategories:CSI
+extraships:Nick Stokes/Greg Sanders
+
 [www.phoenixsong.net]
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter
@@ -1237,6 +1266,17 @@ extracharacters:Kurt Hummel,Blaine Anderson
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+[www.simplyundeniable.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
 [www.sinful-desire.org]
 ## Site dedicated to these categories/characters/ships
 extracategories:Supernatural
@@ -1383,3 +1423,47 @@ extracategories:Stargate: Atlantis
 ## extratags params in all other sections.  Only commandline options
 ## beat overrides.
 #extratags:fanficdownloader
+
+
+[teststory:defaults]
+valid_entries:title,author_list,authorId_list,authorUrl_list,storyUrl,
+ datePublished,dateUpdated,numWords,status,language,series,seriesUrl,
+ rating,category_list,genre_list,warnings_list,characters_list
+
+# {{storyId}} is a special case--it's the only one that works.
+title:Test Story Title {{storyId}}
+author_list:Test Author aa
+authorId_list:1
+authorUrl_list:http://test1.com?authid=1
+storyUrl:http://test1.com?sid={{storyId}}
+datePublished:1975-03-15
+dateUpdated:1975-04-15
+numWords:123,456
+status:In-Progress
+language:English
+
+chaptertitles:Prologue
+
+## Add additional sections with different numbers to get different
+## parameters for different story urls.
+## test1.com?sid=1000
+[teststory:1000]
+# note the leading commas when doing add_to_ with valid_entries and *_list
+add_to_valid_entries:,favs
+title:Testing New Feature {{storyId}}
+author_list:Bob Smith
+authorId_list:45
+authorUrl_list:http://test1.com?authid=45
+datePublished:2013-03-15
+dateUpdated:2013-04-15
+numWords:1456
+favs:56
+series:The Great Test [4]
+seriesUrl:http://test1.com?seriesid=1
+rating:Tweenie
+category_list:Harry Potter,Furbie,Crossover,Puella Magi Madoka Magica/魔法少女まどか★マギカ,Magical Girl Lyrical Nanoha
+genre_list:Fantasy,Comedy,Sci-Fi,Noir
+warnings_list:Swearing,Violence
+characters_list:Bob Smith,George Johnson,Fred Smythe
+
+chaptertitles:Prologue,Chapter 1\, Xenos on Cinnabar,Chapter 2\, Sinmay on Kintikin,3. Chapter 3

fanficdownloader/adapters/__init__.py

@@ -115,6 +115,9 @@ import adapter_hennethannunnet
 import adapter_tokrafandomnetcom
 import adapter_netraptororg
 import adapter_asr3slashzoneorg
+import adapter_nickandgregnet
+import adapter_potterheadsanonymouscom
+import adapter_simplyundeniablecom
 
 ## This bit of complexity allows adapters to be added by just adding
 ## importing.  It eliminates the long if/else clauses we used to need

fanficdownloader/adapters/adapter_dramioneorg.py

@@ -237,6 +237,9 @@ class DramioneOrgAdapter(BaseSiteAdapter):
             if 'Word count' in label:
                 self.story.setMetadata('numWords', value)
 
+            if 'Read' in label:
+                self.story.setMetadata('read', value)
+
             if 'Categories' in label:
                 cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
                 for cat in cats:
@@ -282,6 +285,14 @@ class DramioneOrgAdapter(BaseSiteAdapter):
             # I find it hard to care if the series parsing fails
             pass
             
+        try:
+            self.story.setMetadata('reviews',
+                                   stripHTML(soup.find('h2',{'id':'pagetitle'}).
+                                             findAll('a', href=re.compile(r'^reviews.php'))[1]))
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 

fanficdownloader/adapters/adapter_fanfictionnet.py

@@ -135,12 +135,25 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         categories = soup.find('div',{'id':'pre_story_links'}).findAll('a',{'class':'xcontrast_txt'})
         #print("xcontrast_txt a:%s"%categories)
         if len(categories) > 1:
+            # Strangely, the ones with *two* links are the
+            # non-crossover categories.  Each is in a category itself
+            # of Book, Movie, etc.
             self.story.addToList('category',stripHTML(categories[1]))
         elif 'Crossover' in categories[0]['href']:
             caturl = "http://%s%s"%(self.getSiteDomain(),categories[0]['href'])
             catsoup = bs.BeautifulSoup(self._fetchUrl(caturl))
             for a in catsoup.findAll('a',href=re.compile(r"^/crossovers/")):
                 self.story.addToList('category',stripHTML(a))
+            else:
+                # Fall back.  I ran across a story with a Crossver
+                # category link to a broken page once.
+                # http://www.fanfiction.net/s/2622060/1/
+                # Naruto + Harry Potter Crossover
+                logger.info("Fall back category collection")
+                for c in stripHTML(categories[0]).replace(" Crossover","").split(' + '):
+                    self.story.addToList('category',c)
+                
+                
             
         a = soup.find('a', href='http://www.fictionratings.com/')
         rating = a.string

fanficdownloader/adapters/adapter_grangerenchantedcom.py

@@ -227,6 +227,9 @@ class GrangerEnchantedCom(BaseSiteAdapter):
             if 'Word count' in label:
                 self.story.setMetadata('numWords', value)
 
+            if 'Read' in label:
+                self.story.setMetadata('read', value)
+
             if 'Categories' in label:
                 cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
                 for cat in cats:
@@ -278,7 +281,14 @@ class GrangerEnchantedCom(BaseSiteAdapter):
                         self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
-            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+
+        try:
+            self.story.setMetadata('reviews',
+                                   stripHTML(soup.find('div',{'id':'sort'}).
+                                             findAll('a', href=re.compile(r'^reviews.php'))[1]))
         except:
             # I find it hard to care if the series parsing fails
             pass

fanficdownloader/adapters/adapter_mugglenetcom.py

@@ -66,14 +66,18 @@ class MuggleNetComAdapter(BaseSiteAdapter): # XXX
             
     @staticmethod # must be @staticmethod, don't remove it.
     def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'fanfiction.mugglenet.com' # XXX
+        # The site domain.
+        return 'fanfiction.mugglenet.com'
+
+    @classmethod
+    def getAcceptDomains(cls):
+        return ['fanfiction.mugglenet.com','fanfic.mugglenet.com']
 
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+        return re.escape("http://")+r"fanfic(tion)?\.mugglenet\.com"+re.escape("/viewstory.php?sid=")+r"\d+$"
 
     ## Login seems to be reasonably standard across eFiction sites.
     def needToLoginCheck(self, data):

fanficdownloader/adapters/adapter_nhamagicalworldsus.py

@@ -112,7 +112,8 @@ class NHAMagicalWorldsUsAdapter(BaseSiteAdapter):
 		
         for info in asoup.findAll('table', {'width' : '100%', 'bordercolor' : re.compile(r'#')}):
             a = info.find('a')
-            if ('viewstory.php?sid='+self.story.getMetadata('storyId')) in a['href']:
+            if 'viewstory.php?sid='+self.story.getMetadata('storyId') == a['href'] or \
+                    ('viewstory.php?sid='+self.story.getMetadata('storyId')+'&') in a['href']:
                 self.story.setMetadata('title',a.string)
                 break
 		
@@ -142,14 +143,14 @@ class NHAMagicalWorldsUsAdapter(BaseSiteAdapter):
         for cat in cats:
             self.story.addToList('category',cat.string)
 					
-        a = info.find('a', href=re.compile(r'reviews.php\?sid='+self.story.getMetadata('storyId')))
+        a = info.find('a', href=re.compile(r'viewuser.php'))
         val = a.nextSibling
         svalue = ""
         while not defaultGetattr(val) == 'br':
             val = val.nextSibling
         val = val.nextSibling
         while not defaultGetattr(val) == 'br':
-            svalue += str(val)
+            svalue += unicode(val)
             val = val.nextSibling
         self.setDescription(url,svalue)
         

fanficdownloader/adapters/adapter_nickandgregnet.py

@@ -0,0 +1,174 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return NickAndGregNetAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class NickAndGregNetAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
+        self._setURL('http://' + self.getSiteDomain() + '/desert_archive/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','nag')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%Y/%m/%d"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.nickandgreg.net'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/desert_archive/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/desert_archive/viewstory.php?sid=")+r"\d+$"
+    
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&i=1'
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/desert_archive/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        chapters = soup.find('select')
+        for chapter in chapters.findAll('option'):
+            if chapter.text != 'Story Index' and chapter.text != 'Chapters':
+                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/desert_archive/'+chapter['value']))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+        
+        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
+        
+        for div in asoup.findAll('td', {'class' : 'tblborder6'}):
+            a = div.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+            if a != None:
+                break
+        
+        self.setDescription(url,div.find('br').nextSibling)
+        
+        a=div.text.split('Rating:')
+        if len(a) == 2: self.story.setMetadata('rating', a[1].split(' -')[0])
+        
+        a=div.text.split('Characters:')
+        if len(a) == 2: 
+        	for char in a[1].split(' -')[0].split(', '):
+        	    self.story.addToList('characters',char)
+        	    
+        a=div.text.split('Genres:')
+        if len(a) == 2: 
+        	for genre in a[1].split(' -')[0].split(', '):
+        	    self.story.addToList('genre',genre)
+        	    
+        a=div.text.split('Warnings:')
+        if len(a) == 2: 
+        	for warn in a[1].split(' -')[0].split(', '):
+        	    if 'none' not in warn:
+        	        self.story.addToList('warnings',warn)
+        	    
+        a=div.text.split('Completed:')
+        if len(a) ==2:
+            if 'Yes' in a[1]:
+                self.story.setMetadata('status', 'Completed')
+            else:
+                self.story.setMetadata('status', 'In-Progress')
+                
+        a=div.text.split('Published:')
+        if len(a) == 2: self.story.setMetadata('datePublished', makeDate(stripHTML(a[1].split(' -')[0]), self.dateformat))
+        
+        a=div.text.split('Updated:')
+        if len(a) == 2: self.story.setMetadata('dateUpdated', makeDate(stripHTML(a[1].split(' -')[0]), self.dateformat))
+
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('table', {'class' : 'tblborder6'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_potterheadsanonymouscom.py

@@ -0,0 +1,297 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return PotterHeadsAnonymousComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class PotterHeadsAnonymousComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','pha')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%d %b %Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'fanfic.potterheadsanonymous.com'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Registered Users Only' in data \
+                or 'There is no such account on our website' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['cookiecheck'] = '1'
+        params['submit'] = 'Submit'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "Member Account" not in d : #Member Account
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=4"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+            
+        # Since the warning text can change by warning level, let's
+        # look for the warning pass url.  ksarchive uses
+        # &warning= -- actually, so do other sites.  Must be an
+        # eFiction book.
+            
+        # viewstory.php?sid=1882&warning=4
+        # viewstory.php?sid=1654&ageconsent=ok&warning=5
+        #print data
+        #m = re.search(r"'viewstory.php\?sid=1882(&warning=4)'",data)
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid & error in url.
+                addurl = addurl.replace("&","&")
+                url = self.url+'&index=1'+addurl
+                logger.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+
+        pagetitle = soup.find('div',{'id':'pagetitle'})
+        
+        ## Title
+        a = pagetitle.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = pagetitle.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=3'))
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_scarvesandcoffeenet.py

@@ -141,7 +141,7 @@ class ScarvesAndCoffeeNetAdapter(BaseSiteAdapter):
         self.story.setMetadata('title',a.string)
         
         # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        a = soup.find('div',{"id":"pagetitle"}).find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
         self.story.setMetadata('authorId',a['href'].split('=')[1])
         self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
         self.story.setMetadata('author',a.string)

fanficdownloader/adapters/adapter_simplyundeniablecom.py

@@ -0,0 +1,217 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return SimplyUndeniableComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class SimplyUndeniableComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','sud')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%m/%d/%Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.simplyundeniable.com'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Please log in now' in data \
+                or 'There is no such account on our website' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['cookiecheck'] = '1'
+        params['submit'] = 'Submit'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "My Account" not in d : #Member Account
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        if "Please log in now" in data:
+                raise exceptions.FailedToDownload(self.getSiteDomain() +" says: You need to have access to the restricted section.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('h1')
+        self.story.setMetadata('title',a.text)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
+        
+        for info in asoup.findAll('table', {'cellpadding' : '5'}):
+            a = info.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+            if a != None:
+                self.story.setMetadata('title',a.string)
+                break
+                
+        # Find the chapters:
+        if "Disclaimer" in data:
+            self.chapterUrls.append((self.story.getMetadata('title'),url))
+        else:
+            for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+&i=1$')):
+            # just in case there's tags, like <i> in chapter titles.
+                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+        
+        a = info.find('td', {'valign' : 'top'}).find('p')
+        self.setDescription(url,a)
+        
+        a = info.find('td', {'class' : 'greysm'}).findAll('b')
+        self.story.setMetadata('datePublished', makeDate(stripHTML(a[0].nextSibling), self.dateformat))
+        self.story.setMetadata('dateUpdated', makeDate(stripHTML(a[1].nextSibling), self.dateformat))
+        if 'Yes' in a[2].nextSibling:
+            self.story.setMetadata('status', 'Completed')
+        else:
+            self.story.setMetadata('status', 'In-Progress')
+        self.story.setMetadata('rating', a[3].nextSibling)
+        self.story.setMetadata('numWords', a[4].nextSibling)
+        
+        
+        warnings = info.find('td', {'width' : '45'}).nextSibling.nextSibling.text.split(', ')
+        for warning in warnings:
+            if 'none' not in warning:
+                self.story.addToList('warnings',warning)
+        chars = info.find('td', {'width' : '51'}).nextSibling.nextSibling.text.split(', ')
+        for char in chars:
+            if '&' in char:
+                self.story.addToList('ships',char)
+            else:
+                self.story.addToList('characters',char)
+        genres = info.find('td', {'width' : '36'}).nextSibling.nextSibling.text.split(', ')
+        for genre in genres:
+            self.story.addToList('genre',genre)
+            
+        cat = info.find('a', href=re.compile(r'categories.php'))
+        self.story.addToList('category',cat.string)
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('span', {'class' : 'style'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_siyecouk.py

@@ -110,7 +110,7 @@ class SiyeCoUkAdapter(BaseSiteAdapter): # XXX
 
         ## Title
         titlea = authsoup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',titlea.string)
+        self.story.setMetadata('title',stripHTML(titlea))
         
         # Find the chapters (from soup, not authsoup):
         for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):

fanficdownloader/adapters/adapter_test1.py

@@ -47,31 +47,63 @@ class TestSiteAdapter(BaseSiteAdapter):
         return BaseSiteAdapter.getSiteURLPattern(self)+r'/?\?sid=\d+$'
 
     def extractChapterUrlsAndMetadata(self):
+        idstr = self.story.getMetadata('storyId')
+        idnum = int(idstr)
 
-        if self.story.getMetadata('storyId') == '665' and not (self.is_adult or self.getConfig("is_adult")):
+        if idnum >= 1000:
+            logger.warn("storyId:%s - Custom INI data will be used."%idstr)
+
+            sections = ['teststory:%s'%idstr,'teststory:defaults']
+            #print("self.get_config_list(sections,'valid_entries'):%s"%self.get_config_list(sections,'valid_entries'))
+            for key in self.get_config_list(sections,'valid_entries'):
+                if key.endswith("_list"):
+                    nkey = key[:-len("_list")]
+                    #print("addList:%s"%(nkey))
+                    for val in self.get_config_list(sections,key):
+                        #print("addList:%s->%s"%(nkey,val))
+                        self.story.addToList(nkey,val.decode('utf-8').replace('{{storyId}}',idstr))
+                else:
+                    # Special cases:
+                    if key in ['datePublished','dateUpdated']:
+                        self.story.setMetadata(key,makeDate(self.get_config(sections,key),"%Y-%m-%d"))
+                    else:
+                        self.story.setMetadata(key,self.get_config(sections,key).decode('utf-8').replace('{{storyId}}',idstr))
+                    #print("set:%s->%s"%(key,self.story.getMetadata(key)))
+
+            self.chapterUrls = []
+            for (j,chap) in enumerate(self.get_config_list(sections,'chaptertitles'),start=1):
+                self.chapterUrls.append( (chap,self.url+"&chapter=%d"%j) )
+            # self.chapterUrls = [(u'Prologue '+self.crazystring,self.url+"&chapter=1"),
+            #                 ('Chapter 1, Xenos on Cinnabar',self.url+"&chapter=2"),
+            #                 ]
+            self.story.setMetadata('numChapters',len(self.chapterUrls))
+            
+            return
+
+        if idstr == '665' and not (self.is_adult or self.getConfig("is_adult")):
             logger.warn("self.is_adult:%s"%self.is_adult)
             raise exceptions.AdultCheckRequired(self.url)
 
-        if self.story.getMetadata('storyId') == '666':
+        if idstr == '666':
             raise exceptions.StoryDoesNotExist(self.url)
 
-        if self.story.getMetadata('storyId').startswith('670'):
+        if idstr.startswith('670'):
             time.sleep(1.0)
             
-        if self.story.getMetadata('storyId').startswith('671'):
+        if idstr.startswith('671'):
             time.sleep(1.0)
             
         if self.getConfig("username"):
             self.username = self.getConfig("username")
         
-        if self.story.getMetadata('storyId') == '668' and self.username != "Me" :
+        if idstr == '668' and self.username != "Me" :
             raise exceptions.FailedToLogin(self.url,self.username)
 
-        if self.story.getMetadata('storyId') == '664':
-            self.story.setMetadata(u'title',"Test Story Title "+self.story.getMetadata('storyId')+self.crazystring)
+        if idstr == '664':
+            self.story.setMetadata(u'title',"Test Story Title "+idstr+self.crazystring)
             self.story.setMetadata('author','Test Author aa bare amp(&) quote(') amp(&)')
         else:
-            self.story.setMetadata(u'title',"Test Story Title "+self.story.getMetadata('storyId'))
+            self.story.setMetadata(u'title',"Test Story Title "+idstr)
             self.story.setMetadata('author','Test Author aa')
         self.story.setMetadata('storyUrl',self.url)
         self.setDescription(self.url,u'Description '+self.crazystring+u''' Done
@@ -79,13 +111,12 @@ class TestSiteAdapter(BaseSiteAdapter):
 Some more longer description.  "I suck at summaries!"  "Better than it sounds!"  "My first fic"
 ''')
         self.story.setMetadata('datePublished',makeDate("1975-03-15","%Y-%m-%d"))
-        if self.story.getMetadata('storyId') == '669':
+        if idstr == '669':
             self.story.setMetadata('dateUpdated',datetime.datetime.now())
         else:
             self.story.setMetadata('dateUpdated',makeDate("1975-04-15","%Y-%m-%d"))
         self.story.setMetadata('numWords','123456')
 
-        idnum = int(self.story.getMetadata('storyId'))
         if idnum % 2 == 1:
             self.story.setMetadata('status','In-Progress')
         else:
@@ -108,7 +139,7 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
             
         self.story.setMetadata('rating','Tweenie')
 
-        if self.story.getMetadata('storyId') == '673':
+        if idstr == '673':
             self.story.addToList('author','Author From List')
             self.story.addToList('author','Author From List 2')
         
@@ -128,19 +159,19 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
         self.story.addToList('warnings','Swearing')
         self.story.addToList('warnings','Violence')
 
-        if self.story.getMetadata('storyId') == '80':
+        if idstr == '80':
             self.story.addToList('category',u'Rizzoli & Isles')
             self.story.addToList('characters','J. Rizzoli')
-        elif self.story.getMetadata('storyId') == '81':
+        elif idstr == '81':
             self.story.addToList('category',u'Pitch Perfect')
             self.story.addToList('characters','Chloe B.')
-        elif self.story.getMetadata('storyId') == '83':
+        elif idstr == '83':
             self.story.addToList('category',u'Rizzoli & Isles')
             self.story.addToList('characters','J. Rizzoli')
             self.story.addToList('category',u'Pitch Perfect')
             self.story.addToList('characters','Chloe B.')
             self.story.addToList('ships','Chloe B. & J. Rizzoli')
-        elif self.story.getMetadata('storyId') == '82':
+        elif idstr == '82':
             self.story.addToList('characters','Henry (Once Upon a Time)')        
             self.story.addToList('category',u'Once Upon a Time (TV)')
         else:
@@ -218,6 +249,9 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
 <div>
 <h3>Prologue</h3>
 <p>This is a fake adapter for testing purposes.  Different sid's will give different errors:</p>
+<h4>Config(personal.ini)</h4>
+<p>sid&gt;=1000 will use custom test story data from your configuration(personal.ini)</p>
+<p>Hard coded ids:</p>
 <p>http://test1.com?sid=664 - Crazy string title</p>
 <p>http://test1.com?sid=665 - raises AdultCheckRequired</p>
 <p>http://test1.com?sid=666 - raises StoryDoesNotExist</p>
@@ -225,10 +259,6 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
 <p>http://test1.com?sid=668 - raises FailedToLogin unless username='Me'</p>
 <p>http://test1.com?sid=669 - Succeeds with Updated Date=now</p>
 <p>http://test1.com?sid=670 - Succeeds, but sleeps 2sec on each chapter</p>
-
-
-
-
 <p>http://test1.com?sid=671 - Succeeds, but sleeps 2sec metadata only</p>
 <p>http://test1.com?sid=672 - Succeeds, quick meta, sleeps 2sec chapters only</p>
 <p>http://test1.com?sid=673 - Succeeds, multiple authors, extra categories, genres</p>

fanficdownloader/adapters/base_adapter.py

@@ -315,6 +315,8 @@ class BaseSiteAdapter(Configurable):
             fetch=self._fetchUrlRaw
 
         acceptable_attributes = ['href','name','class','id']
+        if self.getConfig("keep_style_attr"):
+            acceptable_attributes.append('style')
         #print("include_images:"+self.getConfig('include_images'))
         if self.getConfig('include_images'):
             acceptable_attributes.extend(('src','alt','longdesc'))

fanficdownloader/configurable.py

@@ -15,7 +15,7 @@
 # limitations under the License.
 #
 
-import ConfigParser
+import ConfigParser, re
 
 # All of the writers(epub,html,txt) and adapters(ffnet,twlt,etc)
 # inherit from Configurable.  The config file(s) uses ini format:
@@ -99,8 +99,12 @@ class Configuration(ConfigParser.SafeConfigParser):
     def getValidMetaList(self):
         return self.validEntries + self.getConfigList("extra_valid_entries")
 
+    # used by adapters & writers, non-convention naming style
     def hasConfig(self, key):
-        for section in self.sectionslist:
+        return self.has_config(self.sectionslist, key)
+
+    def has_config(self, sections, key):
+        for section in sections:
             try:
                 self.get(section,key)
                 #print("found %s in section [%s]"%(key,section))
@@ -114,10 +118,14 @@ class Configuration(ConfigParser.SafeConfigParser):
                     pass
 
         return False
-        
+
+    # used by adapters & writers, non-convention naming style
     def getConfig(self, key, default=""):
+        return self.get_config(self.sectionslist,key,default)
+    
+    def get_config(self, sections, key, default=""):        
         val = default
-        for section in self.sectionslist:
+        for section in sections:
             try:
                 val = self.get(section,key)
                 if val and val.lower() == "false":
@@ -127,7 +135,7 @@ class Configuration(ConfigParser.SafeConfigParser):
             except (ConfigParser.NoOptionError, ConfigParser.NoSectionError), e:
                 pass
 
-        for section in self.sectionslist[::-1]:
+        for section in sections[::-1]:
             # 'martian smiley' [::-1] reverses list by slicing whole list with -1 step.
             try:
                 val = val + self.get(section,"add_to_"+key)
@@ -138,11 +146,15 @@ class Configuration(ConfigParser.SafeConfigParser):
         return val
 
     # split and strip each.
-    def getConfigList(self, key):
-        vlist = self.getConfig(key).split(',')
-        vlist = filter( lambda x : x !='', [ v.strip() for v in vlist ])
+    def get_config_list(self, sections, key):
+        vlist = re.split(r'(?<!\\),',self.get_config(sections,key)) # don't split on \,
+        vlist = filter( lambda x : x !='', [ v.strip().replace('\,',',') for v in vlist ])
         #print "vlist("+key+"):"+str(vlist)
         return vlist        
+    
+    # used by adapters & writers, non-convention naming style
+    def getConfigList(self, key):
+        return self.get_config_list(self.sectionslist, key)
 
 # extended by adapter, writer and story for ease of calling configuration.
 class Configurable(object):
@@ -157,11 +169,19 @@ class Configurable(object):
         return self.configuration.getValidMetaList()
     
     def hasConfig(self, key):
-        return self.configuration.hasConfig(key)
+        return self.configuration.hasConfig(key)        
         
+    def has_config(self, sections, key):
+        return self.configuration.has_config(sections, key)
+
     def getConfig(self, key, default=""):
         return self.configuration.getConfig(key,default)
 
+    def get_config(self, sections, key, default=""):
+        return self.configuration.get_config(sections,key,default)
+
     def getConfigList(self, key):
         return self.configuration.getConfigList(key)
-        
+
+    def get_config_list(self, sections, key):
+        return self.configuration.get_config_list(sections,key)

fanficdownloader/geturls.py

@@ -54,9 +54,14 @@ def get_urls_from_page(url,configuration=None,normalize=False):
         opener = u2.build_opener(u2.HTTPCookieProcessor(),GZipProcessor())
         data = opener.open(url).read()
 
-    return get_urls_from_html(data,url,configuration,normalize)
+    # kludge because I don't see it on enough sites to be worth generalizing yet.
+    restrictsearch=None
+    if 'scarvesandcoffee.net' in url:
+        restrictsearch=('div',{'id':'mainpage'})
 
-def get_urls_from_html(data,url=None,configuration=None,normalize=False):
+    return get_urls_from_html(data,url,configuration,normalize,restrictsearch)
+
+def get_urls_from_html(data,url=None,configuration=None,normalize=False,restrictsearch=None):
 
     normalized = [] # normalized url
     retlist = [] # orig urls.
@@ -65,6 +70,9 @@ def get_urls_from_html(data,url=None,configuration=None,normalize=False):
         configuration = Configuration("test1.com","EPUB")
 
     soup = BeautifulSoup(data)
+    if restrictsearch:
+        soup = soup.find(*restrictsearch)
+        print("restrict search:%s"%soup)
     
     for a in soup.findAll('a'):
         if a.has_key('href'):

fanficdownloader/story.py

@@ -300,7 +300,7 @@ class Story(Configurable):
                 replacement=replacement.replace('\s',' ') 
                 self.replacements.append([metakeys,regexp,replacement,condkey,condregexp])
     
-    def doReplacments(self,value,key):
+    def doReplacements(self,value,key):
         for (metakeys,regexp,replacement,condkey,condregexp) in self.replacements:
             if (metakeys == None or key in metakeys) \
                     and isinstance(value,basestring) \
@@ -340,7 +340,7 @@ class Story(Configurable):
                     value = value.strftime(self.getConfig(key+"_format","%Y-%m-%d"))
 
             if doreplacements:
-                value=self.doReplacments(value,key)
+                value=self.doReplacements(value,key)
             if removeallentities and value != None:
                 return removeAllEntities(value)
             else:
@@ -364,8 +364,8 @@ class Story(Configurable):
                 auth = v
                 # make sure doreplacements & removeallentities are honored.
                 if doreplacements:
-                    aurl=self.doReplacments(aurl,'authorUrl')
-                    auth=self.doReplacments(auth,'author')
+                    aurl=self.doReplacements(aurl,'authorUrl')
+                    auth=self.doReplacements(auth,'author')
                 if removeallentities:
                     aurl=removeAllEntities(aurl)
                     auth=removeAllEntities(auth)
@@ -438,7 +438,7 @@ class Story(Configurable):
         if retlist:
             if doreplacements:
                 retlist = filter( lambda x : x!=None and x!='' ,
-                                  map(partial(self.doReplacments,key=listname),retlist) )
+                                  map(partial(self.doReplacements,key=listname),retlist) )
             if removeallentities:
                 retlist = filter( lambda x : x!=None and x!='' ,
                                   map(removeAllEntities,retlist) )

index.html

@@ -57,9 +57,12 @@
 	  <h3>Changes:</h3>
 	  <p>
 	    <ul>
-	      <li>Don't strip lead/trail whitespace from replace_metadata, add feature \s->' ' in replace_metadata replacements.</li>
+	      <li>New Site: fanfic.potterheadsanonymous.com - Thanks Ida!</li>
+	      <li>New Site: www.simplyundeniable.com - Thanks Ida!</li>
+	      <li>Allow domain fanfic.mugglenet.com for mugglenet.com</li>
 	    </ul>
 	  </p>
+	  
 	  <p>
 	    Questions?  Check out our 
 	    <a href="http://code.google.com/p/fanficdownloader/wiki/FanFictionDownloaderFAQs">FAQs</a>.
@@ -68,7 +71,7 @@
 	    If you have any problems with this application, please
 	    report them in
 	    the <a href="http://groups.google.com/group/fanfic-downloader">FanFictionDownLoader Google Group</a>.  The
-	    <a href="http://4-4-56.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
+	    <a href="http://4-4-59.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
 	  </p>
 	  <div id='error'>
 	    {{ error_message }}
@@ -612,6 +615,21 @@
 	    Use the URL of the story's chapter list, such as
 	    <br /><a href="http://netraptor.org/archive/viewstory.php?sid=1234">http://netraptor.org/archive/viewstory.php?sid=1234</a>
 	  </dd>
+	  <dt>nickandgreg.net</dt>
+	  <dd>
+	    Use the URL of the story's chapter list, such as
+	    <br /><a href="http://www.nickandgreg.net/desert_archive/viewstory.php?sid=1234">http://www.nickandgreg.net/desert_archive/viewstory.php?sid=1234</a>
+	  </dd>
+	  <dt>fanfic.potterheadsanonymous.com</dt>
+	  <dd>
+	    Use the URL of the story's chapter list, such as
+	    <br /><a href="http://fanfic.potterheadsanonymous.com/viewstory.php?sid=1234">http://fanfic.potterheadsanonymous.com/viewstory.php?sid=1234</a>
+	  </dd>
+	  <dt>www.simplyundeniable.com</dt>
+	  <dd>
+	    Use the URL of the story's chapter list, such as
+	    <br /><a href="http://www.simplyundeniable.com/viewstory.php?sid=1234">http://www.simplyundeniable.com/viewstory.php?sid=1234</a>
+	  </dd>
 	</dl>
 
 	<p>