Bump versions.

http://archive.skyehawke.com/story.php?no=1234
http://www.skyehawke.com/archive/story.php?no=1234
http://skyehawke.com/archive/story.php?no=1234

app.yaml

@@ -1,6 +1,6 @@
 # ffd-retief-hrd fanfictiondownloader
 application: fanfictiondownloader
-version: 4-4-37
+version: 4-4-43
 runtime: python27
 api_version: 1
 threadsafe: true

calibre-plugin/__init__.py

@@ -27,7 +27,7 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
     description         = 'UI plugin to download FanFiction stories from various sites.'
     supported_platforms = ['windows', 'osx', 'linux']
     author              = 'Jim Miller'
-    version             = (1, 7, 3)
+    version             = (1, 7, 9)
     minimum_calibre_version = (0, 8, 57)
 
     #: This field defines the GUI plugin class that contains all the code

calibre-plugin/ffdl_plugin.py

@@ -290,12 +290,10 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         url_list = get_urls_from_page(url,configuration)
 
         if url_list:
-            d = ViewLog(_("List of URLs"),"\n".join(url_list),parent=self.gui)
-            d.setWindowIcon(get_icon('bookmarks.png'))
-            d.exec_()
+            self.add_dialog("\n".join(url_list))
         else:
-            info_dialog(self.gui, _('List of URLs'),
-                        _('No Valid URLs found on given page.'),
+            info_dialog(self.gui, _('List of Story URLs'),
+                        _('No Valid Story URLs found on given page.'),
                         show=True,
                         show_copy_button=False)
         
@@ -338,12 +336,12 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
     def _finish_get_list_urls(self, book_list):
         url_list = [ x['url'] for x in book_list if x['good'] ]
         if url_list:
-            d = ViewLog(_("List of URLs"),"\n".join(url_list),parent=self.gui)
+            d = ViewLog(_("List of Story URLs"),"\n".join(url_list),parent=self.gui)
             d.setWindowIcon(get_icon('bookmarks.png'))
             d.exec_()
         else:
             info_dialog(self.gui, _('List of URLs'),
-                        _('No URLs found in selected books.'),
+                        _('No Story URLs found in selected books.'),
                         show=True,
                         show_copy_button=False)
                 
@@ -424,12 +422,13 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
             if confirm(message,'fanfictiondownloader_reject_non_fanfiction', self.gui):
                 self.gui.iactions['Remove Books'].delete_books()
                 
-    def add_dialog(self):
+    def add_dialog(self,url_list_text=None):
 
         #print("add_dialog()")
 
-        url_list = self.get_urls_clip()
-        url_list_text = "\n".join(url_list)
+        if not url_list_text:
+            url_list = self.get_urls_clip()
+            url_list_text = "\n".join(url_list)
         
         # self.gui is the main calibre GUI. It acts as the gateway to access
         # all the elements of the calibre user interface, it should also be the
@@ -524,13 +523,14 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         tdir = PersistentTemporaryDirectory(prefix='fanfictiondownloader_')
         options['tdir']=tdir
 
-        self.gui.status_bar.show_message(_('Started fetching metadata for %s stories.'%len(books)), 3000)
-        
         if 0 < len(filter(lambda x : x['good'], books)):
+            self.gui.status_bar.show_message(_('Started fetching metadata for %s stories.'%len(books)), 3000)
             LoopProgressDialog(self.gui,
                                books,
                                partial(self.get_metadata_for_book, options = options),
                                partial(self.start_download_list, options = options))
+        else:
+            self.gui.status_bar.show_message(_('No valid story URLs entered.'), 3000)
         # LoopProgressDialog calls get_metadata_for_book for each 'good' story,
         # get_metadata_for_book updates book for each,
         # LoopProgressDialog calls start_download_list at the end which goes
@@ -875,9 +875,14 @@ make_firstimage_cover:true
         add_ids = [ x['calibre_id'] for x in add_list ]
         update_list = filter(lambda x : x['good'] and not x['added'], book_list)
         update_ids = [ x['calibre_id'] for x in update_list ]
+        all_ids = add_ids
+        all_ids.extend(update_ids)
+
+        if options['collision'] != CALIBREONLY and \
+                (prefs['addtolists'] or prefs['addtoreadlists']):
+            self._update_reading_lists(all_ids,add=True)
         
         if len(add_list):
-            ## even shows up added to searchs.  Nice.
             self.gui.library_view.model().books_added(len(add_list))
             self.gui.library_view.model().refresh_ids(add_ids)
 
@@ -906,8 +911,6 @@ make_firstimage_cover:true
         print("all done, remove temp dir.")
         remove_dir(options['tdir'])
 
-        all_ids = add_ids
-        all_ids.extend(update_ids)
         if 'Count Pages' in self.gui.iactions and len(prefs['countpagesstats']) and len(all_ids):
             cp_plugin = self.gui.iactions['Count Pages']
             cp_plugin.count_statistics(all_ids,prefs['countpagesstats'])
@@ -922,7 +925,8 @@ make_firstimage_cover:true
 
         book_list = job.result
         good_list = filter(lambda x : x['good'], book_list)
-        bad_list = filter(lambda x : x['calibre_id'] and not x['good'], book_list)
+        bad_list = filter(lambda x : not x['good'], book_list)
+        print("book_list:%s"%book_list)
         payload = (good_list, bad_list, options)
         
         msg = '''
@@ -937,14 +941,14 @@ make_firstimage_cover:true
                 status = book['status']
             else:
                 status = 'Good'
-            htmllog = htmllog + '<tr><td>' + '</td><td>'.join([status,book['title'],", ".join(book['author']),book['comment'],book['url']]) + '</td></tr>'
+            htmllog = htmllog + '<tr><td>' + '</td><td>'.join([escapehtml(status),escapehtml(book['title']),escapehtml(", ".join(book['author'])),escapehtml(book['comment']),book['url']]) + '</td></tr>'
         
         for book in bad_list:
             if 'status' in book:
                 status = book['status']
             else:
                 status = 'Bad'
-            htmllog = htmllog + '<tr><td>' + '</td><td>'.join([status,book['title'],", ".join(book['author']),book['comment'],book['url']]) + '</td></tr>'
+            htmllog = htmllog + '<tr><td>' + '</td><td>'.join([escapehtml(status),escapehtml(book['title']),escapehtml(", ".join(book['author'])),escapehtml(book['comment']),book['url']]) + '</td></tr>'
         
         htmllog = htmllog + '</table></body></html>'
         
@@ -953,7 +957,6 @@ make_firstimage_cover:true
                 'FFDL log', 'FFDL download complete', msg,
                 show_copy_button=False)
         
-        
     def _do_download_list_update(self, payload):
         
         (good_list,bad_list,options) = payload
@@ -1017,8 +1020,9 @@ make_firstimage_cover:true
                     print("remove f:"+fmt)
                     db.remove_format(book['calibre_id'], fmt, index_is_id=True)#, notify=False
 
-        if prefs['addtolists'] or prefs['addtoreadlists']:
-            self._update_reading_lists([book_id],add=True)
+        # moved up so whole lists are done at once for efficiency.
+        # if prefs['addtolists'] or prefs['addtoreadlists']:
+        #     self._update_reading_lists([book_id],add=True)
                 
         return book_id
 
@@ -1052,6 +1056,7 @@ make_firstimage_cover:true
         if 'authorUrl' in book['all_metadata']:
             authurls = book['all_metadata']['authorUrl'].split(", ")
             for i, auth in enumerate(book['author']):
+                #print("===Update author url for %s to %s"%(auth,authurls[i]))
                 autid=db.get_author_id(auth)
                 db.set_link_field_for_author(autid, unicode(authurls[i]),
                                              commit=False, notify=False)
@@ -1423,3 +1428,6 @@ def get_url_list(urls):
     # set removes dups.
     return set(filter(f,urls.strip().splitlines()))
 
+def escapehtml(txt):
+    return txt.replace("&","&amp;").replace(">","&gt;").replace("<","&lt;")
+        

calibre-plugin/jobs.py

@@ -39,11 +39,12 @@ def do_download_worker(book_list, options,
 
     print(options['version'])
     total = 0
+    alreadybad = []
     # Queue all the jobs
     print("Adding jobs for URLs:")
     for book in book_list:
+        print("%s"%book['url'])
         if book['good']:
-            print("%s"%book['url'])
             total += 1
             args = ['calibre_plugins.fanfictiondownloader_plugin.jobs',
                     'do_download_for_worker',
@@ -58,6 +59,9 @@ def do_download_worker(book_list, options,
             # job._modified_date = modified_date
             # job._existing_isbn = existing_isbn
             server.add_job(job)
+        else:
+            # was already bad before the subprocess ever started.
+            alreadybad.append(book)
     
     # This server is an arbitrary_n job, so there is a notifier available.
     # Set the % complete to a small number to avoid the 'unavailable' indicator
@@ -90,11 +94,11 @@ def do_download_worker(book_list, options,
             print("Successfully downloaded:")
             for book in book_list:
                 if book['good']:
-                    print(book['title'])
+                    print("%s %s"%(book['title'],book['url']))
             print("\nUnsuccessful:")
             for book in book_list:
                 if not book['good']:
-                    print(book['title'])
+                    print("%s %s"%(book['title'],book['url']))
             break
 
     server.close()

defaults.ini

@@ -320,12 +320,13 @@ background_color: ffffff
 ## Allow customization of CSS.  Make sure to keep at least one space
 ## at the start of each line and to escape % to %%.  Also need
 ## background_color to be in the same section, if included in CSS.
-## 'adobe-text-layout: optimizeSpeed;' prevents hyphenation on newer Nooks
+## 'adobe-hyphenate: none;' prevents hyphenation on newer Nooks
+## STR(wG) (1.2.1+ for sure)
 output_css: 
  body { background-color: #%(background_color)s;
         text-align: justify;
         margin: 2%%;
-	adobe-text-layout: optimizeSpeed; }
+	adobe-hyphenate: none; }
  pre { font-size: x-small; }
  sml { font-size: small; }
  h1 { text-align: center; }
@@ -519,6 +520,15 @@ extracategories:Blood Ties
 ## personal.ini, not defaults.ini.
 #is_adult:true
 
+[buffynfaith.net]
+## Site dedicated to these categories/characters/ships
+extracategories:Buffy: The Vampire Slayer
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
 [castlefans.org]
 ## Site dedicated to these categories/characters/ships
 extracategories:Castle
@@ -693,6 +703,19 @@ extracharacters:Hermione Granger
 ## Site dedicated to these categories/characters/ships
 extracategories:Highlander
 
+[imagine.e-fic.com]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
 [indeath.net]
 ## Site dedicated to these categories/characters/ships
 extracategories:In Death
@@ -1048,6 +1071,10 @@ extracategories:Harry Potter
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+[www.henneth-annun.net]
+## Site dedicated to these categories/characters/ships
+extracategories:The Hobbit
+
 [www.hpfandom.net]
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter
@@ -1135,6 +1162,15 @@ extracategories:Harry Potter
 ## Site dedicated to these categories/characters/ships
 extracategories:Prison Break
 
+[www.psychfic.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Psych
+
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
 [www.qaf-fic.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:Queer as Folk

fanficdownloader/adapters/__init__.py

@@ -111,6 +111,10 @@ import adapter_efictionestelielde
 import adapter_dotmoonnet
 import adapter_pommedesangcom
 import adapter_restrictedsectionorg
+import adapter_imagineeficcom
+import adapter_buffynfaithnet
+import adapter_psychficcom
+import adapter_hennethannunnet
 
 ## This bit of complexity allows adapters to be added by just adding
 ## importing.  It eliminates the long if/else clauses we used to need

fanficdownloader/adapters/adapter_archiveofourownorg.py

@@ -82,7 +82,8 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
 
     def getSiteURLPattern(self):
         # http://archiveofourown.org/collections/Smallville_Slash_Archive/works/159770
-        return re.escape("http://")+re.escape(self.getSiteDomain())+r"(/collections/[^/]+)?/works/(?P<id>\d+)"
+        # Discard leading zeros from story ID numbers--AO3 doesn't use them in it's own chapter URLs.
+        return re.escape("http://")+re.escape(self.getSiteDomain())+r"(/collections/[^/]+)?/works/0*(?P<id>\d+)"
         
     ## Login
     def needToLoginCheck(self, data):

fanficdownloader/adapters/adapter_archiveskyehawkecom.py

@@ -66,14 +66,16 @@ class ArchiveSkyeHawkeComAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'archive.skyehawke.com'
 
+    @classmethod
+    def getAcceptDomains(cls):
+        # mobile.fimifction.com isn't actually a valid domain, but we can still get the story id from URLs anyway
+        return ['archive.skyehawke.com','www.skyehawke.com']
+
     def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/story.php?no=1234"
+        return "http://archive.skyehawke.com/story.php?no=1234 http://www.skyehawke.com/archive/story.php?no=1234  http://skyehawke.com/archive/story.php?no=1234"
 
     def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/story.php?no=")+r"\d+$"
-
-
-        
+        return re.escape("http://")+r"(archive|www)\.skyehawke\.com/(archive/)?story\.php\?no=\d+$"
 
     ## Getting the chapter list and the meta data, plus 'is adult' checking.
     def extractChapterUrlsAndMetadata(self):

fanficdownloader/adapters/adapter_buffynfaithnet.py

@@ -0,0 +1,290 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+import cookielib as cl
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+
+# This function is called by the downloader in all adapter_*.py files
+# in this dir to register the adapter class.  So it needs to be
+# updated to reflect the class below it.  That, plus getSiteDomain()
+# take care of 'Registering'.
+def getClass():
+    return BuffyNFaithNetAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class BuffyNFaithNetAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+        
+        self.setHeader()
+        
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query correct
+        m = re.match(self.getSiteURLPattern(),url)
+        if m:
+            self.story.setMetadata('storyId',m.group('id'))
+            logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+            # normalized story URL. gets rid of chapter if there, left with ch 1 URL on this site
+            nurl = "http://"+self.getSiteDomain()+"/fanfictions/index.php?act=vie&id="+self.story.getMetadata('storyId')
+            self._setURL(nurl)
+            #argh, this mangles the ampersands I need on metadata['storyUrl']
+            #will set it this way
+            self.story.setMetadata('storyUrl',nurl,condremoveentities=False)
+        else:
+            raise exceptions.InvalidStoryURL(url,
+                                             self.getSiteDomain(),
+                                             self.getSiteExampleURLs())
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','bnfnet')
+
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'buffynfaith.net'
+
+    @classmethod
+    def stripURLParameters(cls,url):
+        "Only needs to be overriden if URL contains more than one parameter"
+        ## This adapter needs at least two parameters left on the URL, act and id
+        return re.sub(r"(\?act=(vie|ovr)&id=\d+)&.*$",r"\1",url)
+    
+    def setHeader(self):
+        "buffynfaith.net wants a Referer for images.  Used both above and below(after cookieproc added)"
+        self.opener.addheaders = [('Referer', 'http://'+self.getSiteDomain()+'/')]
+        
+    def getSiteExampleURLs(self):
+        return "http://buffynfaith.net/fanfictions/index.php?act=vie&id=963 http://buffynfaith.net/fanfictions/index.php?act=vie&id=949 http://buffynfaith.net/fanfictions/index.php?act=vie&id=949&ch=2"
+
+    def getSiteURLPattern(self):
+        #http://buffynfaith.net/fanfictions/index.php?act=vie&id=963
+        #http://buffynfaith.net/fanfictions/index.php?act=vie&id=949
+        #http://buffynfaith.net/fanfictions/index.php?act=vie&id=949&ch=2
+        p = re.escape("http://"+self.getSiteDomain()+"/fanfictions/index.php?act=")+\
+            r"(vie|ovr)&id=(?P<id>\d+)(&ch=(?P<ch>\d+))?$"
+        return p
+
+    def extractChapterUrlsAndMetadata(self):
+
+        dateformat = "%d %B %Y"
+        url = self.url
+        logger.debug("URL: "+url)
+        
+        #set a cookie to get past adult check
+        if self.is_adult or self.getConfig("is_adult"):
+            cookieproc = urllib2.HTTPCookieProcessor()
+            cookie = cl.Cookie(version=0, name='my_age', value='yes',
+                               port=None, port_specified=False,
+                               domain=self.getSiteDomain(), domain_specified=False, domain_initial_dot=False,
+                               path='/', path_specified=True,
+                               secure=False,
+                               expires=time.time()+10000,
+                               discard=False,
+                               comment=None,
+                               comment_url=None,
+                               rest={'HttpOnly': None},
+                               rfc2109=False)
+            cookieproc.cookiejar.set_cookie(cookie)
+            self.opener = urllib2.build_opener(cookieproc)        
+            self.setHeader()
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        #print data
+
+        if "ADULT CONTENT WARNING" in data: 
+            raise exceptions.AdultCheckRequired(self.url)
+
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        #stuff in <head>: description
+        svalue = soup.head.find('meta',attrs={'name':'description'})['content']
+        #self.story.setMetadata('description',svalue)
+        self.setDescription(url,svalue)
+        
+        #useful stuff in rest of doc, all contained in this:
+        doc = soup.body.find('div', id='my_wrapper')
+        
+        #first the site category (more of a genre to me, meh) and title, in this element:
+        mt = doc.find('div',attrs={'class':'maintitle'})
+        self.story.addToList('genre',mt.findAll('a')[1].string)
+        self.story.setMetadata('title',mt.findAll('a')[1].nextSibling[len('&nbsp;&raquo;&nbsp;'):])
+        del mt
+
+        #the actual category, for me, is 'Buffy: The Vampire Slayer'
+        #self.story.addToList('category','Buffy: The Vampire Slayer')
+        #No need to do it here, it is better to set it in in plugin-defaults.ini and defaults.ini
+        
+        #then a block that sits in a table cell like so:
+        #(contains a lot of metadata)
+        mblock = doc.find('td', align='left', width = '70%').contents
+        while len(mblock) > 0:
+            i = mblock.pop(0)
+            if 'Author:' in i.string:
+                #drop empty space
+                mblock.pop(0)
+                #get author link
+                a = mblock.pop(0)
+                authre = re.escape('./index.php?act=bio&id=')+'(?P<authid>\d+)'
+                m = re.match(authre,a['href'])
+                self.story.setMetadata('author',a.string)
+                self.story.setMetadata('authorId',m.group('authid'))
+                authurl = u'http://%s/fanfictions/index.php?act=bio&id=%s' % ( self.getSiteDomain(),
+                                            self.story.getMetadata('authorId'))
+                self.story.setMetadata('authorUrl',authurl,condremoveentities=False)
+                #drop empty space
+                mblock.pop(0)
+            if 'Rating:' in i.string:
+                self.story.setMetadata('rating',mblock.pop(0).strip())
+            if 'Published:' in i.string:
+                date = mblock.pop(0).strip()
+                #get rid of 'st', 'nd', 'rd', 'th' after day number
+                date = date[0:2]+date[4:]
+                self.story.setMetadata('datePublished',makeDate(date, dateformat))
+            if 'Last Updated:' in i.string:
+                date = mblock.pop(0).strip()
+                #get rid of 'st', 'nd', 'rd', 'th' after day number
+                date = date[0:2]+date[4:]
+                self.story.setMetadata('dateUpdated',makeDate(date, dateformat))
+            if 'Genre:' in i.string:
+                genres = mblock.pop(0).strip()
+                genres = genres.split('/')
+                for genre in genres: self.story.addToList('genre',genre)
+            #end ifs
+        #end while
+
+        # Find the chapter selector 
+        select = soup.find('select', { 'name' : 'ch' } )
+         
+        if select is None:
+           # no selector found, so it's a one-chapter story.
+           #self.chapterUrls.append((self.story.getMetadata('title'),url))
+           self.chapterUrls.append((self.story.getMetadata('title'),url))
+        else:
+            allOptions = select.findAll('option')
+            for o in allOptions:
+                url = u'http://%s/fanfictions/index.php?act=vie&id=%s&ch=%s' % ( self.getSiteDomain(),
+                                            self.story.getMetadata('storyId'),
+                                            o['value'])
+                title = u"%s" % o
+                title = stripHTML(title)
+                ts = title.split(' ',1)
+                title = ts[0]+'. '+ts[1]
+                self.chapterUrls.append((title,url))
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+        
+        ## Go scrape the rest of the metadata from the author's page.
+        data = self._fetchUrl(self.story.getMetadata('authorUrl'))
+        soup = bs.BeautifulSoup(data)
+        #find the story link and its parent div
+        storya = soup.find('a',{'href':self.story.getMetadata('storyUrl')})
+        storydiv = storya.parent
+        #warnings come under a <spawn> tag. Never seen that before...
+        #appears to just be a line of freeform text, not necessarily a list
+        #optional
+        spawn = storydiv.find('spawn',{'id':'warnings'})
+        if spawn is not None:
+            warns = spawn.nextSibling.strip()
+            self.story.addToList('warnings',warns)
+        #some meta in spans - this should get all, even the ones jammed in a table
+        spans = storydiv.findAll('span')
+        for s in spans:
+            if s.string == 'Ship:':
+                list = s.nextSibling.strip().split()
+                self.story.extendList('ships',list)
+            if s.string == 'Characters:':
+                list = s.nextSibling.strip().split(',')
+                self.story.extendList('characters',list)
+            if s.string == 'Status:':
+                st = s.nextSibling.strip()
+                self.story.setMetadata('status',st)
+            if s.string == 'Words:':
+                st = s.nextSibling.strip()
+                self.story.setMetadata('numWords',st)
+
+        #reviews - is this worth having?
+        #ffnet adapter gathers it, don't know if anything else does
+        #or if it's ever going to be used!
+        a = storydiv.find('a',{'id':'bold-blue'})
+        if a:
+            revs = a.nextSibling.strip()[1:-1]
+            self.story.setMetadata('reviews',st)
+        else:
+            revs = '0'
+            self.story.setMetadata('reviews',st)
+
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'fanfiction'})
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+        
+        #remove all the unnecessary bookmark tags
+        [s.extract() for s in div('div',{'class':"tiny_box2"})]
+        
+        #is there a review link?
+        r = div.find('a',href=re.compile(re.escape("./index.php?act=irv")+".*$"))
+        if r is not None:
+        #remove the review link and its parent div
+            r.parent.extract()
+        
+        #There might also be a link to the sequel on the last chapter
+        #I'm inclined to keep it in, but the URL needs to be changed from relative to absolute
+        #Shame there isn't proper series metadata available
+        #(I couldn't find it anyway)
+        s = div.find('a',href=re.compile(re.escape("./index.php?act=ovr")+".*$"))
+        if s is not None:
+            s['href'] = 'http://'+self.getSiteDomain()+'/fanfictions'+s['href'][1:]
+
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_fimfictionnet.py

@@ -97,9 +97,6 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
         if "/images/missing_story.png" in data:
             raise exceptions.StoryDoesNotExist(self.url)
         
-        if "Invalid story id" in apiData.values():
-            raise exceptions.StoryDoesNotExist(self.url)
-        
         if "This story has been marked as having adult content." in data:
             raise exceptions.AdultCheckRequired(self.url)
         
@@ -114,6 +111,9 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
             else:
                 raise exceptions.FailedToLogin(self.url,"Story requires individual password",passwdonly=True)
          
+        if "Invalid story id" in apiData.values():
+            raise exceptions.StoryDoesNotExist(self.url)
+        
         storyMetadata = apiData["story"]    
             
         ## Title

fanficdownloader/adapters/adapter_hennethannunnet.py

@@ -0,0 +1,171 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return HennethAnnunNetAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class HennethAnnunNetAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/stories/chapter.cfm?stid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','htan')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%m/%d/%y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.henneth-annun.net'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/stories/chapter.cfm?stid=1234"
+
+    def getSiteURLPattern(self):
+        return "http://"+self.getSiteDomain()+"/stories/chapter(_view)?.cfm\?stid="+r"\d+$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+            
+        if "We're sorry. This story is not available." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: This story is not available.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('h2', {'id':'page_heading'})
+        self.story.setMetadata('title',a.string)
+
+        # Find the chapters: chapter_view.cfm?stid=6663&spordinal=1" 
+        for chapter in soup.findAll('a', href=re.compile(r'chapter_view.cfm\?stid='+self.story.getMetadata('storyId')+"&spordinal=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/stories/'+chapter['href']))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+        
+        self.story.setMetadata('numWords', soup.find('tr', {'class':'foot'}).findAll('td')[1].text)
+
+        self.setDescription(url,soup.find('div', {'id':'summary'}))
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        info = soup.find('div', {'id':'storyinformation'})
+        labels=info.findAll('b')
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+                
+            if 'Completion' in label:
+                if 'Complete' in value.string:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Rating' in label:
+                self.story.setMetadata('rating', value.string)
+
+            if 'Era:' in label:
+                self.story.addToList('category',value.string)
+
+            if 'Genre' in label:
+                self.story.addToList('genre',value.string)
+                
+        labels=info.findAll('strong')
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+            
+            if 'Author' in label:
+                value=value.nextSibling
+                self.story.setMetadata('authorId',value['href'].split('=')[1])
+                self.story.setMetadata('authorUrl','http://'+self.host+'/'+value['href'])
+                self.story.setMetadata('author',value.string)
+                
+            if 'Post' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated:' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+                
+        for char in soup.findAll('a', href=re.compile(r"/resources/bios_view.cfm\?scid=\d+")):
+            self.story.addToList('characters',stripHTML(char))
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'class' : 'block chapter'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_imagineeficcom.py

@@ -0,0 +1,287 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return ImagineEFicComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class ImagineEFicComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','ime')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%Y.%m.%d"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'imagine.e-fic.com'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Registered Users Only' in data \
+                or 'There is no such account on our website' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['cookiecheck'] = '1'
+        params['submit'] = 'Submit'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "Member Account" not in d : #Member Account
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=4"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid & error in url.
+                addurl = addurl.replace("&","&")
+                url = self.url+'&index=1'+addurl
+                logger.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+                
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            i=1
+            for a in storyas:
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        break
+                    i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_psychficcom.py

@@ -0,0 +1,246 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return PsychFicComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class PsychFicComAdapter(BaseSiteAdapter): 
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','psyf')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%B %d, %Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.psychfic.com'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=4"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid & error in url.
+                addurl = addurl.replace("&","&")
+                url = self.url+'&index=1'+addurl
+                logger.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.text
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            i=1
+            for a in storyas:
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        break
+                    i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_restrictedsectionorg.py

@@ -205,10 +205,26 @@ class RestrictedSectionOrgSiteAdapter(BaseSiteAdapter):
         params['accept.x'] = 1
         params['accept.y'] = 1
 
-        data = self._postUrl(url, params)
-        if "I certify that I am over the age of 18 and that accessing the following story will not violate the laws of my country or local ordinances." in data:
-            raise exceptions.FailedToLogin(url,params['username'])
-        return data
+        excpt=None
+        for sleeptime in [0.5, 1.5, 4, 9]:
+            time.sleep(sleeptime)	
+            try:
+                data = self._postUrl(url, params)
+                if data == "Unable to connect to the database":
+                    raise exceptions.FailedToDownload("Site reported 'Unable to connect to the database'")
+                if "I certify that I am over the age of 18 and that accessing the following story will not violate the laws of my country or local ordinances." in data:
+                    raise exceptions.FailedToLogin(url,params['username'])
+                return data
+            except exceptions.FailedToLogin, ftl:
+                # no need to retry these.
+                raise(ftl)
+            except Exception, e:
+                excpt=e
+                logger.warn("Caught an exception reading URL: %s  Exception %s."%(unicode(url),unicode(e)))
+
+        logger.error("Giving up on %s" %url)
+        logger.exception(excpt)
+        raise(excpt)
         
     # grab the text for an individual chapter.
     def getChapterText(self, url):
@@ -216,6 +232,13 @@ class RestrictedSectionOrgSiteAdapter(BaseSiteAdapter):
         logger.debug('Getting chapter text from: %s' % url)
 
         data = self._postUrlUP(url)
+        #print("data:%s"%data)
+        
+        # some stories have html that confuses the parser.  For story
+        # text we don't care about anything before '<table id="page"'
+        # and seems to clear the issue.
+        data = data[data.index('<table id="page"'):]
+        
         soup = bs.BeautifulSoup(data)
         
         div = soup.find('td',{'id':'page_content'})

fanficdownloader/adapters/adapter_test1.py

@@ -101,9 +101,10 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
                 }
             self.story.setMetadata('language',langs[idnum%len(langs)])
             self.setSeries('The Great Test',idnum)
-        
+        if idnum == 0:
+            self.setSeries("A Nook Hyphen Test "+self.story.getMetadata('dateCreated'),idnum)
+            
         self.story.setMetadata('rating','Tweenie')
-        
 
         if self.story.getMetadata('storyId') == '673':
             self.story.addToList('author','Author From List')
@@ -205,11 +206,24 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
 <p>http://test1.com?sid=668 - raises FailedToLogin unless username='Me'</p>
 <p>http://test1.com?sid=669 - Succeeds with Updated Date=now</p>
 <p>http://test1.com?sid=670 - Succeeds, but sleeps 2sec on each chapter</p>
+
+
+
+
 <p>http://test1.com?sid=671 - Succeeds, but sleeps 2sec metadata only</p>
-<p>http://test1.com?sid=672 - Succeeds, quick meta, sleeps 2sec chapters only</p>
-<p>http://test1.com?sid=673 - Succeeds, multiple authors</p>
-<p>Odd sid's will be In-Progress, evens complete.  sid&lt;10 will be assigned one of four languages and included in a series.</p>
+<p>http://test1.com?sid=672 - Succeeds, quick meta, sleeps 2sec chapters only</p><p>http://test1.com?sid=0 - Succeeds, generates some text specifically for testing hyphenation problems with Nook STR/STRwG</p><p>Odd sid's will be In-Progress, evens complete.  sid&lt;10 will be assigned one of four languages and included in a series.</p>
 </div>
+'''
+        elif self.story.getMetadata('storyId') == '0':
+            text=u'''
+<h3>45. Pronglet Returns to Hogwarts: Chapter 7</h3>
+<br />
+    eyes… but I’m not convinced we should automatically<br />
+<br /><br />
+<b>Thanks to the latest to recommend me: Alastor</b><br />
+<br /><br />
+    “Sure, invite her along. Does she have children?”<br />
+<br />
 '''
         else:
             text=u'''

fanficdownloader/adapters/adapter_thewriterscoffeeshopcom.py

@@ -48,7 +48,7 @@ class TheWritersCoffeeShopComSiteAdapter(BaseSiteAdapter):
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/library/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        self.dateformat = "%B %d, %Y"
+        self.dateformat = "%d %b %Y"
 
             
     @staticmethod

fanficdownloader/configurable.py

@@ -74,7 +74,8 @@ class Configuration(ConfigParser.SafeConfigParser):
             # internal stuff.
             'langcode',
             'output_css',
-            'authorHTML'
+            'authorHTML',
+            'lastupdate'
             ]
 
     def addConfigSection(self,section):

fanficdownloader/story.py

@@ -432,8 +432,13 @@ class Story(Configurable):
                                   map(removeAllEntities,retlist) )
 
         if retlist:
-            # remove dups and sort.
-            return sorted(list(set(retlist)))
+            if listname in ('author','authorUrl'):
+                # need to retain order for author & authorUrl so the
+                # two match up.
+                return retlist
+            else:
+                # remove dups and sort.
+                return sorted(list(set(retlist)))
         else:
             return []
 

fanficdownloader/writers/writer_epub.py

@@ -21,6 +21,7 @@ import StringIO
 import zipfile
 from zipfile import ZipFile, ZIP_STORED, ZIP_DEFLATED
 import urllib
+import re
 
 ## XML isn't as forgiving as HTML, so rather than generate as strings,
 ## use DOM to generate the XML files.
@@ -657,7 +658,12 @@ div { margin: 0pt; padding: 0pt; }
                 # as one line.  This causes problems for nook(at
                 # least) when the chapter size starts getting big
                 # (200k+)
-                fullhtml = fullhtml.replace('</p>','</p>\n').replace('<br />','<br />\n')
+                #fullhtml = fullhtml.replace('</p>','</p>\n').replace('<br />','<br />\n')
+                # The replaces above added tons of extra newlines
+                # during *each* epub update.  The regexp version adds
+                # only one and removes any extra.
+                fullhtml = re.sub(r'(</p>|<br />)\n*',r'\1\n',fullhtml)
+
                 outputepub.writestr("OEBPS/file%04d.xhtml"%(index+1),fullhtml.encode('utf-8'))
                 del fullhtml
 

index.html

@@ -57,14 +57,10 @@
 	  <h3>Changes:</h3>
 	  <p>
 	    <ul>
-	      <li>New site: www.dotmoon.net (Thanks Ida!)</li>
-	      <li>New site: efiction.esteliel.de (Thanks Ida!)</li>
-	      <li>New site: pommedesang.com (Thanks Ida!)</li>
-	      <li>New Spanish language site supported: www.potterfics.com (Thanks Dan!)</li>
-	      <li>New site: www.restrictedsection.org (Yes, this site has been frozen since March 2009.)</li>
-	      <li>More workarounds for fimfiction.net's API issues.</li>
+	      <li>New site: www.henneth-annun.net -- Thanks Ida!</li>
+	      <li>New site: www.psychfic.com -- Thanks Ida!</li>
+	      <li>Now accepting www.skyehawke.com/archive URLs for  archive.skyehawke.com stories.</li>
 	    </ul>
-	    FFDL now supports over 80 different sites.
 	  </p>
 	  <p>
 	    Questions?  Check out our 
@@ -74,7 +70,7 @@
 	    If you have any problems with this application, please
 	    report them in
 	    the <a href="http://groups.google.com/group/fanfic-downloader">FanFictionDownLoader Google Group</a>.  The
-	    <a href="http://4-4-36.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
+	    <a href="http://4-4-42.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
 	  </p>
 	  <div id='error'>
 	    {{ error_message }}
@@ -298,6 +294,8 @@
 	  <dd>
 	    Use the URL of the story's summary, such as
 	    <br /><a href="http://archive.skyehawke.com/story.php?no=17466">http://archive.skyehawke.com/story.php?no=17466</a>.
+	    <br /><a href="http://www.skyehawke.com/archive/story.php?no=17466">http://www.skyehawke.com/archive/story.php?no=17466</a>.
+	    <br /><a href="http://skyehawke.com/archive/story.php?no=17466">http://skyehawke.com/archive/story.php?no=17466</a>.
 	  </dd>
 	  <dt>www.libraryofmoria.com</dt>
 	  <dd>
@@ -594,6 +592,28 @@
 	    <br />Or the story URL for one-shots, such as
 	    <br /><a href="http://www.restrictedsection.org/file.php?file=1234">http://www.restrictedsection.org/file.php?file=1234</a>
 	  </dd>
+	  <dt>imagine.e-fic.com</dt>
+	  <dd>
+	    Use the URL of the story's chapter list, such as
+	    <br /><a href="http://imagine.e-fic.com/viewstory.php?sid=1234">http://imagine.e-fic.com/viewstory.php?sid=1234</a>
+	  </dd>
+	  <dt>buffynfaith.net</dt>
+	  <dd>
+	    Use the URL of the story's chapter list, such as
+	    <br /><a href="http://buffynfaith.net/fanfictions/index.php?act=vie&id=1234">http://buffynfaith.net/fanfictions/index.php?act=vie&amp;id=1234</a>
+	    <br />Or, use the URL of any story chapter, such as
+	    <br /><a href="http://buffynfaith.net/fanfictions/index.php?act=vie&id=1234&ch=2">http://buffynfaith.net/fanfictions/index.php?act=vie&amp;id=1234&amp;ch=2</a>
+	  </dd>
+	  <dt>www.henneth-annun.net</dt>
+	  <dd>
+	    Use the URL of the story's chapter list, such as
+	    <br /><a href="http://www.henneth-annun.net/stories/chapter.cfm?stid=1234">http://www.henneth-annun.net/stories/chapter.cfm?stid=1234</a>
+	  </dd>
+	  <dt>http://www.psychfic.com</dt>
+	  <dd>
+	    Use the URL of the story's chapter list, such as
+	    <br /><a href="http://www.psychfic.com/viewstory.php?sid=1234">http://www.psychfic.com/viewstory.php?sid=1234</a>
+	  </dd>
 	</dl>
 
 	<p>

plugin-defaults.ini

@@ -296,12 +296,13 @@ background_color: ffffff
 ## Allow customization of CSS.  Make sure to keep at least one space
 ## at the start of each line and to escape % to %%.  Also need
 ## background_color to be in the same section, if included in CSS.
-## 'adobe-text-layout: optimizeSpeed;' prevents hyphenation on newer Nooks
+## 'adobe-hyphenate: none;' prevents hyphenation on newer Nooks
+## STR(wG) (1.2.1+ for sure)
 output_css: 
  body { background-color: #%(background_color)s;
         text-align: justify;
         margin: 2%%;
-	adobe-text-layout: optimizeSpeed; }
+	adobe-hyphenate: none; }
  pre { font-size: x-small; }
  sml { font-size: small; }
  h1 { text-align: center; }
@@ -483,6 +484,15 @@ extracategories:Blood Ties
 ## personal.ini, not defaults.ini.
 #is_adult:true
 
+[buffynfaith.net]
+## Site dedicated to these categories/characters/ships
+extracategories:Buffy: The Vampire Slayer
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
 [castlefans.org]
 ## Site dedicated to these categories/characters/ships
 extracategories:Castle
@@ -673,6 +683,19 @@ extracharacters:Hermione Granger
 ## Site dedicated to these categories/characters/ships
 extracategories:Highlander
 
+[imagine.e-fic.com]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
 [indeath.net]
 ## Site dedicated to these categories/characters/ships
 extracategories:In Death
@@ -1025,6 +1048,10 @@ extracategories:Harry Potter
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+[www.henneth-annun.net]
+## Site dedicated to these categories/characters/ships
+extracategories:The Hobbit
+
 [www.hpfandom.net]
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter
@@ -1112,6 +1139,15 @@ extracategories:Harry Potter
 ## Site dedicated to these categories/characters/ships
 extracategories:Prison Break
 
+[www.psychfic.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Psych
+
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
 [www.qaf-fic.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:Queer as Folk