Fix for yet another ffnet change, bump versions.

the soup processing.

app.yaml

@@ -1,6 +1,6 @@
 # ffd-retief-hrd fanfictiondownloader
 application: fanfictiondownloader
-version: 4-4-79
+version: 4-4-82
 runtime: python27
 api_version: 1
 threadsafe: true

calibre-plugin/__init__.py

@@ -36,7 +36,7 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
     description         = 'UI plugin to download FanFiction stories from various sites.'
     supported_platforms = ['windows', 'osx', 'linux']
     author              = 'Jim Miller'
-    version             = (1, 7, 50)
+    version             = (1, 7, 53)
     minimum_calibre_version = (0, 8, 57)
 
     #: This field defines the GUI plugin class that contains all the code

calibre-plugin/ffdl_plugin.py

@@ -78,7 +78,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
     # (text, icon_path, tooltip, keyboard shortcut)
     # icon_path isn't in the zip--icon loaded below.
     action_spec = (name, None,
-                   'Download FanFiction stories from various web sites', ())
+                   _('Download FanFiction stories from various web sites'), ())
     # None for keyboard shortcut doesn't allow shortcut.  () does, there just isn't one yet
 
     action_type = 'global'
@@ -108,7 +108,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         # are not found in the zip file will result in null QIcons.
         icon = get_icon('images/icon.png')
 
-        self.qaction.setText('FanFictionDL')
+        self.qaction.setText(_('FanFictionDL'))
         
         # The qaction is automatically created from the action_spec defined
         # above
@@ -709,7 +709,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
             options['tdir']=tdir
 
         if 0 < len(filter(lambda x : x['good'], books)):
-            self.gui.status_bar.show_message(_('Started fetching metadata for %s stories.'%len(books)), 3000)
+            self.gui.status_bar.show_message(_('Started fetching metadata for %s stories.')%len(books), 3000)
             LoopProgressDialog(self.gui,
                                books,
                                partial(self.prep_download_loop, options = options, merge=merge),
@@ -1172,7 +1172,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                 # Sort by our marked column to display the books in order
                 self.gui.library_view.sort_by_named_field('marked', True)
                 
-        self.gui.status_bar.show_message(_('Finished Adding/Updating %d books.'%(len(update_list) + len(add_list))), 3000)
+        self.gui.status_bar.show_message(_('Finished Adding/Updating %d books.')%(len(update_list) + len(add_list)), 3000)
         remove_dir(options['tdir'])
         
         if 'Count Pages' in self.gui.iactions and len(prefs['countpagesstats']) and len(all_ids):
@@ -1273,7 +1273,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         good_list = sorted(good_list,key=lambda x : x['listorder'])
         bad_list = sorted(bad_list,key=lambda x : x['listorder'])
         
-        self.gui.status_bar.show_message(_('Merging %s books.'%total_good))
+        self.gui.status_bar.show_message(_('Merging %s books.')%total_good)
 
         
         existingbook = None
@@ -1331,7 +1331,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         custom_columns = self.gui.library_view.model().custom_columns
         if prefs['mark'] or (prefs['errorcol'] != '' and prefs['errorcol'] in custom_columns):
             self.previous = self.gui.library_view.currentIndex() # used by update_books_finish.
-            self.gui.status_bar.show_message(_('Adding/Updating %s BAD books.'%len(book_list)))
+            self.gui.status_bar.show_message(_('Adding/Updating %s BAD books.')%len(book_list))
             label = custom_columns[prefs['errorcol']]['label']
             LoopProgressDialog(self.gui,
                                book_list,

defaults.ini

@@ -220,6 +220,11 @@ extratags: FanFiction
 ## doesn't work on some devices either.)
 #replace_hr: false
 
+## Some sites/authors/stories use br tags instead of p tags for
+## paragraphs.  This feature uses some heuristics to find and replace
+## br paragraphs with p tags while preserving scene breaks.
+#replace_br_with_p: false
+
 ## If set false, the summary will have all html stripped.
 ## Both this and include_images must be true to get images in the
 ## summary.
@@ -793,6 +798,24 @@ dislikes_label:Dislikes
 #username:YourName
 #password:yourpassword
 
+[storiesonline.net]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Clear FanFiction from defaults, site is original fiction.
+extratags:
+
+extra_valid_entries:size,universe,codes
+#extra_titlepage_entries:size,universe,codes
+
+size_label:Size
+universe_label:Universe
+codes_label:Codes
+
 [grangerenchanted.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter
@@ -1155,6 +1178,10 @@ never_make_cover: true
 ## for examples of how to use them.
 extra_valid_entries:reviews,favs,follows
 
+## ffnet uses 'Pairings', not 'Relationship', stating they don't have
+## to be romantic pairings.
+ships_label:Pairings
+
 [www.fanfiktion.de]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In

fanficdownloader/adapters/__init__.py

@@ -120,6 +120,7 @@ import adapter_potterheadsanonymouscom
 import adapter_simplyundeniablecom
 import adapter_scarheadnet
 import adapter_fictionpadcom
+import adapter_storiesonlinenet
 
 ## This bit of complexity allows adapters to be added by just adding
 ## importing.  It eliminates the long if/else clauses we used to need

fanficdownloader/adapters/adapter_archiveofourownorg.py

@@ -303,7 +303,7 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
     def getChapterText(self, url):
         logger.debug('Getting chapter text from: %s' % url)
 		
-        chapter=bs.BeautifulSoup('<div class="story"></div>')
+        chapter=bs.BeautifulSoup('<div class="story"></div>').find('div')
         data = self._fetchUrl(url)
         soup = bs.BeautifulSoup(data,selfClosingTags=('br','hr'))
 		

fanficdownloader/adapters/adapter_fanfictionnet.py

@@ -165,7 +165,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
 
         # after Rating, the same bit of text containing id:123456 contains
         # Complete--if completed.
-        gui_table1i = soup.find('table',{'cellpadding':'5'})
+        gui_table1i = soup.find('div',{'id':'content_wrapper_inner'})
 
         self.story.setMetadata('title', stripHTML(gui_table1i.find('b'))) # title appears to be only(or at least first) bold tag in gui_table1i
 
@@ -222,14 +222,18 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
             elif  metalist[0].startswith('Words'):
                 self.story.setMetadata('numWords',metalist[0].split(':')[1].strip())
             elif not donechars:
-                self.story.extendList('characters',metalist[0].split('&'))
+                # with 'pairing' support, pairings are bracketed w/o comma after
+                # [Caspian X, Lucy Pevensie] Edmund Pevensie, Peter Pevensie
+                self.story.extendList('characters',metalist[0].replace('[','').replace(']',',').split(','))
+
+                l = metalist[0]
+                while '[' in l:
+                    self.story.addToList('ships',l[l.index('[')+1:l.index(']')].replace(', ','/'))
+                    l = l[l.index(']')+1:]
+
                 donechars = True
             metalist=metalist[1:]
 
-        # next might be characters, otherwise Reviews, Updated, Published, Words
-        # if not ( metalist[0].startswith('Reviews') or metalist[0].startswith('Updated') or metalist[0].startswith('Published') or metalist[0].startswith('Words') or metalist[0].startswith('Chapters') ):
-        #     self.story.extendList('characters',metalist[0].split('&'))
-        
         if 'Status: Complete' in metatext:
             self.story.setMetadata('status', 'Completed')
         else:

fanficdownloader/adapters/adapter_fictionpadcom.py

@@ -180,7 +180,11 @@ class FictionPadSiteAdapter(BaseSiteAdapter):
             
     def getChapterText(self, url):
         logger.debug('Getting chapter text from: %s' % url)
-        soup = bs.BeautifulSoup(self._fetchUrl(url))
+        if not url:
+            data = u"<em>This chapter has no text.</em>"
+        else:
+            data = self._fetchUrl(url)
+        soup = bs.BeautifulSoup(u"<div id='story'>"+data+u"</div>")
         return self.utf8FromSoup(url,soup)
     
 def getClass():

fanficdownloader/adapters/adapter_fimfictionnet.py

@@ -252,7 +252,7 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
         rawGroupList = soup.find('ul', {'id':'story_group_list'})
         if rawGroupList is not None:
             for groupName in rawGroupList.findAll('a', {'href':re.compile('^/group/')}):
-                self.story.addToList("groups",stripHTML(groupName))
+                self.story.addToList("groups",stripHTML(groupName).replace(',', ';'))
             
     def hookForUpdates(self,chaptercount):
         if self.oldchapters and len(self.oldchapters) > self.newestChapterNum:

fanficdownloader/adapters/adapter_finestoriescom.py

@@ -93,7 +93,7 @@ class FineStoriesComAdapter(BaseSiteAdapter):
             params['theusername'] = self.getConfig("username")
             params['thepassword'] = self.getConfig("password")
         params['rememberMe'] = '1'
-        params['page'] = 'http://finestories.com/'
+        params['page'] = 'http://'+self.getSiteDomain()+'/'
         params['submit'] = 'Login'
     
         loginUrl = 'http://' + self.getSiteDomain() + '/login.php'
@@ -262,8 +262,6 @@ class FineStoriesComAdapter(BaseSiteAdapter):
                 last[len(last)-1]=last[len(last)-1].append(next)
                 div.append(div1)
             
-            
-            
         # removing all the left-over stuff    
         for a in div.findAll('span'):
             a.extract()

fanficdownloader/adapters/adapter_nickandgregnet.py

@@ -166,10 +166,11 @@ class NickAndGregNetAdapter(BaseSiteAdapter):
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.find('table', {'class' : 'tblborder6'})
 
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
+        # wrap a div around it.
+        divsoup = bs.BeautifulStoneSoup('<div class="story"></div>',
+                                        selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        div = divsoup.find('div')
+        div.append(soup.find('table', {'class' : 'tblborder6'}))
+
         return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_simplyundeniablecom.py

@@ -214,5 +214,6 @@ class SimplyUndeniableComAdapter(BaseSiteAdapter):
 
         if None == div:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+        div.name='div'
     
         return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_siyecouk.py

@@ -241,5 +241,6 @@ class SiyeCoUkAdapter(BaseSiteAdapter): # XXX
         
         if None == story:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+        story.name='div'
     
         return self.utf8FromSoup(url,story)

fanficdownloader/adapters/adapter_storiesonlinenet.py

@@ -0,0 +1,303 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return StoriesOnlineNetAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class StoriesOnlineNetAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url
+        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2].split(':')[0])
+        if 'storyInfo' in self.story.getMetadata('storyId'):
+            self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/s/'+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','strol')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%Y-%m-%d"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'storiesonline.net'
+
+    @classmethod
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/s/1234 http://"+self.getSiteDomain()+"/s/1234:4010"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain())+r"/s/\d+((:\d+)?(;\d+)?$|(:i)?$)"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Free Registration' in data \
+                or "Invalid Password!" in data \
+                or "Invalid User Name!" in data \
+                or "Log In" in data \
+                or "Access to unlinked chapters requires" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['theusername'] = self.username
+            params['thepassword'] = self.password
+        else:
+            params['theusername'] = self.getConfig("username")
+            params['thepassword'] = self.getConfig("password")
+        params['rememberMe'] = '1'
+        params['page'] = 'http://'+self.getSiteDomain()+'/'
+        params['submit'] = 'Login'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/login.php'
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['theusername']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "My Account" not in d : #Member Account
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['theusername']))
+            raise exceptions.FailedToLogin(url,params['theusername'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url+":i")
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url+":i")
+        
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+        elif "Error! The story you're trying to access is being filtered by your choice of contents filtering." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Error! The story you're trying to access is being filtered by your choice of contents filtering.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        #print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('h1')
+        self.story.setMetadata('title',stripHTML(a))
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"/a/\w+"))
+        self.story.setMetadata('authorId',a['href'].split('/')[2])
+        self.story.setMetadata('authorUrl','http://'+self.host+a['href'])
+        self.story.setMetadata('author',stripHTML(a).replace("'s Page",""))
+
+        # Find the chapters:
+        chapters = soup.findAll('a', href=re.compile(r'^/s/'+self.story.getMetadata('storyId')+":\d+$"))
+        if len(chapters) != 0:
+            for chapter in chapters:
+                # just in case there's tags, like <i> in chapter titles.
+                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+chapter['href']))
+        else:
+            self.chapterUrls.append((self.story.getMetadata('title'),'http://'+self.host+'/s/'+self.story.getMetadata('storyId')))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # surprisingly, the detailed page does not give enough details, so go to author's page
+        skip=0
+        i=0
+        while i == 0:
+            asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')+"&skip="+str(skip)))
+
+            a = asoup.findAll('td', {'class' : 'lc2'})
+            for lc2 in a:
+                if lc2.find('a')['href'] == '/s/'+self.story.getMetadata('storyId'):
+                    i=1
+                    break
+                if a[len(a)-1] == lc2:
+                    skip=skip+10
+
+        for cat in lc2.findAll('div', {'class' : 'typediv'}):
+            self.story.addToList('genre',cat.text)
+
+        # in lieu of word count.
+        self.story.setMetadata('size', lc2.findNext('td', {'class' : 'num'}).text)
+        
+        lc4 = lc2.findNext('td', {'class' : 'lc4'})        
+        
+        try:
+            a = lc4.find('a', href=re.compile(r"/library/show_series.php\?id=\d+"))
+            i = a.parent.text.split('(')[1].split(')')[0]
+            self.setSeries(stripHTML(a), i)
+            self.story.setMetadata('seriesUrl','http://'+self.host+a['href'])
+        except:
+            pass
+        try:
+            a = lc4.find('a', href=re.compile(r"/library/universe.php\?id=\d+"))
+            if a:
+                self.story.setMetadata("universe",stripHTML(a))
+        except:
+            pass
+            
+
+        desc = lc4.contents[0]
+        self.setDescription('http://'+self.host+'/s/'+self.story.getMetadata('storyId'),desc)
+            
+        for b in lc4.findAll('b'):
+            #logger.debug('Getting metadata: "%s"' % b)
+            label = b.text
+            if label in ['Posted:', 'Concluded:', 'Updated:']:
+                value = b.findNext('noscript').text
+                #logger.debug('Have a date field label: "%s", value: "%s"' % (label, value))
+            else:
+                value = b.nextSibling
+            #logger.debug('label: "%s", value: "%s"' % (label, value))
+            
+            if 'Sex' in label:
+                self.story.setMetadata('rating', value)
+                
+            if 'Codes' in label:
+                for code in value.split(' '):
+                    self.story.addToList('codes',code)
+                    
+            if 'Posted' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+                
+            if 'Concluded' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+                
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+#                
+        status = lc4.find('span', {'class' : 'ab'})
+        if  status != None:
+            self.story.setMetadata('status', 'In-Progress')
+            if "Last Activity" in status.text:
+                # date is passed as a timestamp and converted in JS.
+                value = status.findNext('noscript').text
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+        else:
+            self.story.setMetadata('status', 'Completed')
+
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+        
+        # some big chapters are split over several pages
+        pager = div.find('span', {'class' : 'pager'})
+        if pager != None:
+            urls=pager.findAll('a')
+            urls=urls[:len(urls)-1]
+                        
+            for ur in urls:
+                soup = bs.BeautifulSoup(self._fetchUrl("http://"+self.getSiteDomain()+ur['href']),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+                div1 = soup.find('div', {'id' : 'story'})
+                
+                # appending next section
+                last=div.findAll('p')
+                next=div1.find('span', {'class' : 'conTag'}).nextSibling
+            
+                last[len(last)-1]=last[len(last)-1].append(next)
+                div.append(div1)
+            
+        # removing all the left-over stuff    
+        for a in div.findAll('span'):
+            a.extract() 
+            
+        for a in div.findAll('h1'):
+            a.extract()
+        for a in div.findAll('h2'):
+            a.extract()
+        for a in div.findAll('h3'):
+            a.extract()
+        for a in div.findAll('h4'):
+            a.extract()
+        for a in div.findAll('br'):
+            a.extract()
+        for a in div.findAll('div', {'class' : 'date'}):
+            a.extract()
+            
+        a = div.find('form')
+        if a != None:
+            b = a.nextSibling
+            while b != None:
+                a.extract()
+                a=b
+                b=b.nextSibling
+        
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_test1.py

@@ -323,7 +323,7 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
 </div>
 '''
         elif self.story.getMetadata('storyId') == '0':
-            text=u'''
+            text=u'''<div>
 <h3>45. Pronglet Returns to Hogwarts: Chapter 7</h3>
 <br />
     eyes… but I’m not convinced we should automatically<br />
@@ -332,6 +332,7 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
 <br /><br />
     “Sure, invite her along. Does she have children?”<br />
 <br />
+</div>
 '''
         else:
             if self.story.getMetadata('storyId') == '667':

fanficdownloader/adapters/adapter_whoficcom.py

@@ -228,7 +228,8 @@ class WhoficComSiteAdapter(BaseSiteAdapter):
 
         if None == span:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
+
+        span.name='div'
         return self.utf8FromSoup(url,span)
 
 def getClass():

fanficdownloader/adapters/base_adapter.py

@@ -26,6 +26,7 @@ from functools import partial
 
 from .. import BeautifulSoup as bs
 from ..htmlcleanup import stripHTML
+from ..htmlheuristics import replace_br_with_p
 
 logger = logging.getLogger(__name__)
 
@@ -351,16 +352,11 @@ class BaseSiteAdapter(Configurable):
             if t.name in ('center'):
                 t['class']=t.name
                 t.name='div'
-	    # removes paired, but empty tags.
-            if t.string != None and len(t.string.strip()) == 0 :
+	    # removes paired, but empty non paragraph tags.
+            if t.name not in ('p') and t.string != None and len(t.string.strip()) == 0 :
                 t.extract()
-                
-        retval = soup.__str__('utf8').decode('utf-8')
 
-        if self.getConfig('replace_hr'):
-            # replacing a self-closing tag with a container tag in the
-            # soup is more difficult than it first appears.  So cheat.
-            retval = retval.replace("<hr />","<div class='center'>* * *</div>")
+        retval = soup.__str__('utf8').decode('utf-8')
 
         if self.getConfig('nook_img_fix'):
             # if the <img> tag doesn't have a div or a p around it,
@@ -371,7 +367,19 @@ class BaseSiteAdapter(Configurable):
             
         # Don't want body tags in chapter html--writers add them.
         # This is primarily for epub updates.
-        return re.sub(r"</?body>\r?\n?","",retval)
+        retval = re.sub(r"</?body>\r?\n?","",retval)
+        
+        if self.getConfig("replace_br_with_p"):
+            # Apply heuristic processing to replace <br> paragraph
+            # breaks with <p> tags.
+            retval = replace_br_with_p(retval)
+            
+        if self.getConfig('replace_hr'):
+            # replacing a self-closing tag with a container tag in the
+            # soup is more difficult than it first appears.  So cheat.
+            retval = retval.replace("<hr />","<div class='center'>* * *</div>")
+
+        return retval
 
 def cachedfetch(realfetch,cache,url):
     if url in cache:

fanficdownloader/htmlheuristics.py

@@ -0,0 +1,283 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import logging
+logger = logging.getLogger(__name__)
+import re
+import BeautifulSoup as bs
+
+from . import exceptions as exceptions
+
+def replace_br_with_p(body):
+
+    # Ascii character (and Unicode as well) xA0 is a non-breaking space, ascii code 160.
+    # However, Python Regex does not recognize it as a whitespace, so we'll be changing it to a reagular space.
+    body = body.replace(u'\xa0', u' ')
+
+    if body.find('>') == -1 or body.rfind("<") == -1:
+        return body
+
+    # logger.debug(u'BODY start.: ' + body[:250])
+    # logger.debug(u'BODY end...: ' + body[-250:])
+    # logger.debug(u'BODY.......: ' + body)
+
+    # clean breaks (<br />), removing whitespaces between them.
+    body = re.sub(r'\s*<br[^>]*>\s*', r'<br />', body)
+
+    # change surrounding div to a p and remove attrs Top surrounding
+    # tag in all cases now should be div, to just strip the first and
+    # last tags.
+    if is_valid_block(body) and body.find('<div') == 0:
+        body = body[body.index('>')+1:body.rindex("<")]
+
+    body = soup_up_div(u'<div>' + body + u'</div>')
+
+    body = body[body.index('>')+1:body.rindex("<")]
+
+    # Find all bexisting blocks with p, pre and blockquote tags, we need to shields break tags inside those.
+    # This is for "lenient" mode, however it is also used to clear break tags before and after the block elements.
+    blocksRegex = re.compile(r'(\s*<br\ />\s*)*\s*<(pre|p|blockquote|table)([^>]*)>(.+?)</\2>\s*(\s*<br\ />\s*)*', re.DOTALL)
+    body = blocksRegex.sub(r'\n<\2\3>\4</\2>\n', body)
+
+    # if aggressive mode = true
+        # blocksRegex = re.compile(r'(\s*<br\ */*>\s*)*\s*<(pre)([^>]*)>(.+?)</\2>\s*(\s*<br\ */*>\s*)*', re.DOTALL)
+        # In aggressive mode, we also check breakes inside blockquotes, meaning we can get orphaned paragraph tags.
+        # body = re.sub(r'<blockquote([^>]*)>(.+?)</blockquote>', r'<blockquote\1><p>\2</p></blockquote>', body, re.DOTALL)
+    # end aggressive mode
+
+    blocks = blocksRegex.finditer(body)
+    # For our replacements to work, we need to work backwards, so we reverse the iterator.
+    blocksList = []
+    for match in blocks:
+        blocksList.insert(0, match)
+
+    for match in blocksList:
+        group4 =  match.group(4).replace(u'<br />', u'{br /}')
+        body = body[:match.start(4)] + group4 + body[match.end(4):]
+
+    # change surrounding div to a p and remove attrs Top surrounding
+    # tag in all cases now should be div, to just strip the first and
+    # last tags.
+    # body = u'<p>' + body + u'</p>'
+
+    # Nuke div tags surrounding a HR tag.
+    body = re.sub(r'<div[^>]+>\s*<hr[^>]+>\s*</div>', r'\n<hr />\n', body)
+
+    # So many people add formatting to their HR tags, and ePub does not allow those, we are supposed to use css.
+    # This nukes the hr tag attributes.
+    body = re.sub(r'\s*<hr[^>]+>\s*', r'\n<hr />\n', body)
+
+    # Remove leading and trailing breaks from HR tags
+    body = re.sub(r'\s*(<br\ \/>)*\s*<hr\ \/>\s*(<br\ \/>)*\s*', r'\n<hr />\n', body)
+    # Nuking breaks leading paragraps that may be in the body. They are eventually treated as <p><br /></p>
+    body = re.sub(r'\s*(<br\ \/>)+\s*<p', r'\n<p></p>\n<p', body)
+    # Nuking breaks trailing paragraps that may be in the body. They are eventually treated as <p><br /></p>
+    body = re.sub(r'</p>\s*(<br\ \/>)+\s*', r'</p>\n<p></p>\n', body)
+
+    # Because a leading or trailing non break tag will break the following code, we have to mess around rather badly for a few lines.
+    body = body.replace(u'[',u'&squareBracketStart;')
+    body = body.replace(u']',u'&squareBracketEnd;')
+    body = body.replace(u'<br />',u'[br /]')
+
+    breaksRegexp = [
+        re.compile(r'([^\]])(\[br\ \/\])([^\[])'),
+        re.compile(r'([^\]])(\[br\ \/\]){2}([^\[])'),
+        re.compile(r'([^\]])(\[br\ \/\]){3}([^\[])'),
+        re.compile(r'([^\]])(\[br\ \/\]){4}([^\[])'),
+        re.compile(r'([^\]])(\[br\ \/\]){5}([^\[])'),
+        re.compile(r'([^\]])(\[br\ \/\]){6}([^\[])'),
+        re.compile(r'([^\]])(\[br\ \/\]){7}([^\[])'),
+        re.compile(r'([^\]])(\[br\ \/\]){8}([^\[])'),
+        re.compile(r'(\[br\ \/\]){9,}')]
+
+    breaksCount = [
+        len(breaksRegexp[0].findall(body)),
+        len(breaksRegexp[1].findall(body)),
+        len(breaksRegexp[2].findall(body)),
+        len(breaksRegexp[3].findall(body)),
+        len(breaksRegexp[4].findall(body)),
+        len(breaksRegexp[5].findall(body)),
+        len(breaksRegexp[6].findall(body)),
+        len(breaksRegexp[7].findall(body))]
+
+    breaksMax = 0
+    breaksMaxIndex = 0;
+
+    for i in range(1,len(breaksCount)):
+        if breaksCount[i] >= breaksMax:
+            breaksMax = breaksCount[i]
+            breaksMaxIndex = i
+
+    lines = body.split(u'[br /]')
+    contentLines = 0;
+    contentLinesSum = 0;
+    longestLineLength = 0;
+    averageLineLength = 0;
+
+    for line in lines:
+        lineLen = len(line.strip())
+        if lineLen > 0:
+            contentLines += 1
+            contentLinesSum += lineLen
+            if lineLen > longestLineLength:
+                longestLineLength = lineLen
+
+    averageLineLength = contentLinesSum/contentLines
+
+    logger.debug(u'---')
+    logger.debug(u'Lines.............: ' + str(len(lines)))
+    logger.debug(u'contentLines......: ' + str(contentLines))
+    logger.debug(u'contentLinesSum...: ' + str(contentLinesSum))
+    logger.debug(u'longestLineLength.: ' + str(longestLineLength))
+    logger.debug(u'averageLineLength.: ' + str(averageLineLength))
+
+    if breaksMaxIndex == len(breaksCount)-1 and breaksMax < 2:
+        breaksMaxIndex = 0
+        breaksMax = breaksCount[0]
+
+    logger.debug(u'---')
+    logger.debug(u'breaks 1: ' + str(breaksCount[0]))
+    logger.debug(u'breaks 2: ' + str(breaksCount[1]))
+    logger.debug(u'breaks 3: ' + str(breaksCount[2]))
+    logger.debug(u'breaks 4: ' + str(breaksCount[3]))
+    logger.debug(u'breaks 5: ' + str(breaksCount[4]))
+    logger.debug(u'breaks 6: ' + str(breaksCount[5]))
+    logger.debug(u'breaks 7: ' + str(breaksCount[6]))
+    logger.debug(u'breaks 8: ' + str(breaksCount[7]))
+    logger.debug(u'----')
+    logger.debug(u'max found: ' + str(breaksMax))
+    logger.debug(u'max Index: ' + str(breaksMaxIndex))
+    logger.debug(u'----')
+
+    if breaksMaxIndex > 0 and breaksCount[0] > breaksMax and averageLineLength < 90:
+        body = breaksRegexp[0].sub(r'\1 \n\3', body)
+
+    # Find all instances of consecutive breaks less than otr equal to the max count use most often
+    #  replase those tags to inverted p tag pairs, those with more connsecutive breaks are replaced them with a horisontal line
+    for i in range(len(breaksCount)):
+        if i <= breaksMaxIndex:
+            logger.debug(str(i) + u' <= breaksMaxIndex (' + str(breaksMaxIndex) + u')')
+            body = breaksRegexp[i].sub(r'\1</p>\n<p>\3', body)
+        elif i == breaksMaxIndex+1:
+            logger.debug(str(i) + u' == breaksMaxIndex+1 (' + str(breaksMaxIndex+1) + u')')
+            body = breaksRegexp[i].sub(r'\1</p>\n<p><br/></p>\n<p>\3', body)
+        else:
+            logger.debug(str(i) + u' > breaksMaxIndex+1 (' + str(breaksMaxIndex+1) + u')')
+            body = breaksRegexp[i].sub(r'\1</p>\n<hr />\n<p>\3', body)
+
+    body = breaksRegexp[8].sub(r'</p>\n<hr />\n<p>', body)
+
+    # Reverting the square brackets
+    body = body.replace(u'[', u'<')
+    body = body.replace(u']', u'>')
+    body = body.replace(u'&squareBracketStart;', u'[')
+    body = body.replace(u'&squareBracketEnd;', u']')
+
+    body = body.replace(u'{p}', u'<p>')
+    body = body.replace(u'{/p}', u'</p>')
+
+    # If for some reason, a third break makes its way inside the paragraph, preplace that with the empty paragraph for the additional linespaing.
+    body = re.sub(r'<p>\s*(<br\ \/>)+', r'<p><br /></p>\n<p>', body)
+
+    # change empty p tags to include a br to force spacing.
+    body = re.sub(r'<p>\s*</p>', r'<p><br/></p>', body)
+
+    # Clean up hr tags, and add inverted p tag pairs
+    body = re.sub(r'(<div[^>]+>)*\s*<hr\ \/>\s*(</div>)*', r'\n<hr />\n', body)
+
+    # Clean up hr tags, and add inverted p tag pairs
+    body = re.sub(r'\s*<hr\ \/>\s*', r'</p>\n<hr />\n<p>', body)
+
+    # Because the previous regexp may cause trouble if the hr tag already had a p tag pair around it, w nee dot repair that.
+    # Repeated opening p tags are condenced to one. As we added the extra leading opening p tags, we can safely assume that
+    #  the last in such a chain must be the original. Lets keep its attributes if they are there.
+    body = re.sub(r'\s*(<p[^>]*>\s*)+<p([^>]*)>\s*', r'\n<p\2>', body)
+    # Repeated closing p tags are condenced to one
+    body = re.sub(r'\s*(<\/\s*p>\s*){2,}', r'</p>\n', body)
+
+    # superflous cleaning, remove whitespaces traling opening p tags. These does affect formatting.
+    body = re.sub(r'\s*<p([^>]*)>\s*', r'\n<p\1>', body)
+    # superflous cleaning, remove whitespaces leading closing p tags. These does not affect formatting.
+    body = re.sub(r'\s*</p>\s*', r'</p>\n', body)
+
+    # Remove empty tag pairs
+    body = re.sub(r'\s*<(\S+)[^>]*>\s*</\1>', r'', body)
+
+    body = body.replace(u'{br /}', u'<br />')
+    body = body.strip()
+
+    # re-wrap in div tag.
+    body = u'<div>\n' + body + u'</div>\n'
+
+    return body
+
+def is_valid_block(block):
+    return str(block).find('<') == 0 and str(block).find('<!') != 0
+
+def soup_up_div(body):
+    blockTags = ['address', 'blockquote', 'del', 'div', 'dl', 'fieldset', 'form', 'ins', 'noscript', 'ol', 'p', 'pre', 'table', 'ul']
+    recurseTags = ['blockquote', 'div', 'noscript']
+
+    tag = body[:body.index('>')+1]
+    tagend = body[body.rindex('<'):]
+
+    body = body.replace(u'<br />', u'[br /]')
+
+    soup = bs.BeautifulSoup(body)
+
+    body = u''
+    lastElement = 1 # 1 = block, 2 = nested, 3 = invalid
+
+    for i in soup.contents[0]:
+        if str(i).strip().__len__() > 0:
+            s = str(i)
+            if  type(i) == bs.Tag:
+                if  i.name in blockTags:
+                    if lastElement > 1:
+                        body = body.strip(r'\s*(\[br\ \/\]\s*)*\s*')
+                        body += u'{/p}'
+
+                    lastElement = 1
+
+                    if i.name in recurseTags:
+                        s = soup_up_div(s)
+
+                    body += s.strip() + '\n'
+                else:
+                    if lastElement == 1:
+                        body = body.strip(r'\s*(\[br\ \/\]\s*)*\s*')
+                        body += u'{p}'
+
+                    lastElement = 2
+                    body += s
+            elif type(i) == bs.Comment:
+                body += s
+            else:
+                if lastElement == 1:
+                    body = body.strip(r'\s*(\[br\ \/\]\s*)*\s*')
+                    body += u'{p}'
+
+                lastElement = 3
+                body += s
+
+    if lastElement > 1:
+        body = body.strip(r'\s*(\[br\ \/\]\s*)*\s*')
+        body += u'{/p}'
+
+    body = body.replace(u'[br /]', u'<br />')
+
+    return tag + body + tagend

index.html

@@ -46,6 +46,17 @@
       {{yourfile}}
       <!--			</div> -->
 
+          <h3>fanfiction.net / fimfiction.net</h3>
+          <p>
+            Fanfiction.net appears to be blocking access from Google
+            App Engine, which prevents this web service.  There's
+            nothing I can do about it.  At the time of writing, the
+            latest CLI and calibre plugin versions worked.
+          </p>
+          <p>It appears that FimFiction.net is also blocking access from Google
+            App Engine now.
+          </p>
+	  
       {% if authorized %}
       <form action="/fdown" method="post">
 	<div id='urlbox'>
@@ -57,18 +68,11 @@
 	  <h3>Changes:</h3>
 	  <p>
 	    <ul>
-              <li>Fixes for dark-solace.org.</li>
+              <li>Fixes for storiesonline.net, thanks davidfor</li>
+              <li>Additional fixes/enhancements for replace_br_with_p heuristic processing feature.  Turn on in personal User Config.  Thanks, Asbjørn Grandt.</li>
 	    </ul>
 	  </p>
           
-          <h3>fanfiction.net</h3>
-          <p>
-            Fanfiction.net appears to be blocking access from Google
-            App Engine, which prevents this web service.  There's
-            nothing I can do about it.  At the time of writing, the
-            latest CLI and calibre plugin versions worked.
-          </p>
-	  
 	  <p>
 	    Questions?  Check out our 
 	    <a href="http://code.google.com/p/fanficdownloader/wiki/FanFictionDownloaderFAQs">FAQs</a>.
@@ -77,7 +81,7 @@
 	    If you have any problems with this application, please
 	    report them in
 	    the <a href="http://groups.google.com/group/fanfic-downloader">FanFictionDownLoader Google Group</a>.  The
-	    <a href="http://4-4-77.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
+	    <a href="http://4-4-81.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
 	  </p>
 	  <div id='error'>
 	    {{ error_message }}

plugin-defaults.ini

@@ -187,6 +187,11 @@ extratags: FanFiction
 ## doesn't work on some devices either.)
 #replace_hr: false
 
+## Some sites/authors/stories use br tags instead of p tags for
+## paragraphs.  This feature uses some heuristics to find and replace
+## br paragraphs with p tags while preserving scene breaks.
+#replace_br_with_p: false
+
 ## If you have the Generate Cover plugin installed, you can use the
 ## generate_cover_settings parameter to intelligently decide which GC
 ## setting to run.  There are three parts 1) a template of which
@@ -778,6 +783,24 @@ dislikes_label:Dislikes
 #username:YourName
 #password:yourpassword
 
+[storiesonline.net]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Clear FanFiction from defaults, site is original fiction.
+extratags:
+
+extra_valid_entries:size,universe,codes
+#extra_titlepage_entries:size,universe,codes
+
+size_label:Size
+universe_label:Universe
+codes_label:Codes
+
 [grangerenchanted.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter
@@ -1140,6 +1163,10 @@ never_make_cover: true
 ## for examples of how to use them.
 extra_valid_entries:reviews,favs,follows
 
+## ffnet uses 'Pairings', not 'Relationship', stating they don't have
+## to be romantic pairings.
+ships_label:Pairings
+
 [www.fanfiktion.de]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In

status.html

@@ -51,13 +51,16 @@
 	      by {{ fic.author }} ({{ fic.format }})
 	  {% endif %}
 	  {% if fic.failure %}
-          <h3>fanfiction.net</h3>
+          <h3>fanfiction.net / fimfiction.net</h3>
           <p>
             FYI, fanfiction.net appears to be blocking access from Google
             App Engine, which prevents this web service.  There's
             nothing I can do about it.  At the time of writing, the
             latest CLI and calibre plugin versions worked.
           </p>
+          <p>It appears that FimFiction.net is also blocking access from Google
+            App Engine now.
+          </p>
 	  <span id='error'>{{ fic.failure }}</span>
 	  {% endif %}
 	  {% if  not fic.completed and not fic.failure %}