Bump versions, make ready for release.

and is now removing ASCII character x!0 (whitespace) as it interfered with
  regexp.
base_adapter called replace_br_with_p with two argument, where teh function
  only support one.

app.yaml

@@ -1,6 +1,6 @@
 # ffd-retief-hrd fanfictiondownloader
 application: fanfictiondownloader
-version: 4-4-77
+version: 4-4-80
 runtime: python27
 api_version: 1
 threadsafe: true

calibre-plugin/__init__.py

@@ -36,7 +36,7 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
     description         = 'UI plugin to download FanFiction stories from various sites.'
     supported_platforms = ['windows', 'osx', 'linux']
     author              = 'Jim Miller'
-    version             = (1, 7, 48)
+    version             = (1, 7, 51)
     minimum_calibre_version = (0, 8, 57)
 
     #: This field defines the GUI plugin class that contains all the code

calibre-plugin/config.py

@@ -129,9 +129,15 @@ class ConfigWidget(QWidget):
         label.setOpenExternalLinks(True)
         self.l.addWidget(label)
 
-        tab_widget = QTabWidget(self)
-        self.l.addWidget(tab_widget)
 
+        self.scroll_area = QScrollArea(self)
+        self.scroll_area.setFrameShape(QScrollArea.NoFrame)
+        self.scroll_area.setWidgetResizable(True)
+        self.l.addWidget(self.scroll_area)
+
+        tab_widget = QTabWidget(self)
+        self.scroll_area.setWidget(tab_widget)
+        
         self.basic_tab = BasicTab(self, plugin_action)
         tab_widget.addTab(self.basic_tab, 'Basic')
 
@@ -446,15 +452,17 @@ class BasicTab(QWidget):
         topl.addWidget(defs_gb)
 
         horz = QHBoxLayout()
-        topl.addLayout(horz)
-        horz.addWidget(cali_gb)
-        horz.addWidget(rej_gb)
-        
-        horz = QHBoxLayout()
-        topl.addLayout(horz)
-        horz.addWidget(gui_gb)
-        horz.addWidget(misc_gb)
 
+        horz.addWidget(cali_gb)
+
+        vert = QVBoxLayout()
+        vert.addWidget(gui_gb)
+        vert.addWidget(misc_gb)
+        vert.addWidget(rej_gb)
+        
+        horz.addLayout(vert)
+        
+        topl.addLayout(horz)
         topl.insertStretch(-1)
         
     def set_collisions(self):

defaults.ini

@@ -220,6 +220,11 @@ extratags: FanFiction
 ## doesn't work on some devices either.)
 #replace_hr: false
 
+## Some sites/authors/stories use br tags instead of p tags for
+## paragraphs.  This feature uses some heuristics to find and replace
+## br paragraphs with p tags while preserving scene breaks.
+#replace_br_with_p: false
+
 ## If set false, the summary will have all html stripped.
 ## Both this and include_images must be true to get images in the
 ## summary.
@@ -793,6 +798,24 @@ dislikes_label:Dislikes
 #username:YourName
 #password:yourpassword
 
+[storiesonline.net]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Clear FanFiction from defaults, site is original fiction.
+extratags:
+
+extra_valid_entries:size,universe,codes
+#extra_titlepage_entries:size,universe,codes
+
+size_label:Size
+universe_label:Universe
+codes_label:Codes
+
 [grangerenchanted.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter
@@ -1155,6 +1178,10 @@ never_make_cover: true
 ## for examples of how to use them.
 extra_valid_entries:reviews,favs,follows
 
+## ffnet uses 'Pairings', not 'Relationship', stating they don't have
+## to be romantic pairings.
+ships_label:Pairings
+
 [www.fanfiktion.de]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In

fanficdownloader/adapters/__init__.py

@@ -120,6 +120,7 @@ import adapter_potterheadsanonymouscom
 import adapter_simplyundeniablecom
 import adapter_scarheadnet
 import adapter_fictionpadcom
+import adapter_storiesonlinenet
 
 ## This bit of complexity allows adapters to be added by just adding
 ## importing.  It eliminates the long if/else clauses we used to need

fanficdownloader/adapters/adapter_archiveofourownorg.py

@@ -303,7 +303,7 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
     def getChapterText(self, url):
         logger.debug('Getting chapter text from: %s' % url)
 		
-        chapter=bs.BeautifulSoup('<div class="story"></div>')
+        chapter=bs.BeautifulSoup('<div class="story"></div>').find('div')
         data = self._fetchUrl(url)
         soup = bs.BeautifulSoup(data,selfClosingTags=('br','hr'))
 		

fanficdownloader/adapters/adapter_darksolaceorg.py

@@ -94,11 +94,10 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
         else:
             params['penname'] = self.getConfig("username")
             params['password'] = self.getConfig("password")
-        params['rememberme'] = '1'
         params['action'] = 'login'
         params['submit'] = 'Submit'
     
-        loginUrl = 'http://' + self.getSiteDomain() + '/elysian/user.php'
+        loginUrl = 'http://www.' + self.getSiteDomain() + '/elysian/user.php'
         logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['penname']))
     

fanficdownloader/adapters/adapter_fanfictionnet.py

@@ -222,14 +222,18 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
             elif  metalist[0].startswith('Words'):
                 self.story.setMetadata('numWords',metalist[0].split(':')[1].strip())
             elif not donechars:
-                self.story.extendList('characters',metalist[0].split('&'))
+                # with 'pairing' support, pairings are bracketed w/o comma after
+                # [Caspian X, Lucy Pevensie] Edmund Pevensie, Peter Pevensie
+                self.story.extendList('characters',metalist[0].replace('[','').replace(']',',').split(','))
+
+                l = metalist[0]
+                while '[' in l:
+                    self.story.addToList('ships',l[l.index('[')+1:l.index(']')].replace(', ','/'))
+                    l = l[l.index(']')+1:]
+
                 donechars = True
             metalist=metalist[1:]
 
-        # next might be characters, otherwise Reviews, Updated, Published, Words
-        # if not ( metalist[0].startswith('Reviews') or metalist[0].startswith('Updated') or metalist[0].startswith('Published') or metalist[0].startswith('Words') or metalist[0].startswith('Chapters') ):
-        #     self.story.extendList('characters',metalist[0].split('&'))
-        
         if 'Status: Complete' in metatext:
             self.story.setMetadata('status', 'Completed')
         else:
@@ -281,7 +285,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         # don't care about anything before "<div class='storytextp"
         # (there's a space after storytextp, so no close quote(')) and
         # this kills any body tags.
-        divstr = "<div role='main' class='storytextp"
+        divstr = "<div role='main'"
         if divstr not in data:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
         else:

fanficdownloader/adapters/adapter_fictionpadcom.py

@@ -180,7 +180,11 @@ class FictionPadSiteAdapter(BaseSiteAdapter):
             
     def getChapterText(self, url):
         logger.debug('Getting chapter text from: %s' % url)
-        soup = bs.BeautifulSoup(self._fetchUrl(url))
+        if not url:
+            data = u"<em>This chapter has no text.</em>"
+        else:
+            data = self._fetchUrl(url)
+        soup = bs.BeautifulSoup(u"<div id='story'>"+data+u"</div>")
         return self.utf8FromSoup(url,soup)
     
 def getClass():

fanficdownloader/adapters/adapter_finestoriescom.py

@@ -93,7 +93,7 @@ class FineStoriesComAdapter(BaseSiteAdapter):
             params['theusername'] = self.getConfig("username")
             params['thepassword'] = self.getConfig("password")
         params['rememberMe'] = '1'
-        params['page'] = 'http://finestories.com/'
+        params['page'] = 'http://'+self.getSiteDomain()+'/'
         params['submit'] = 'Login'
     
         loginUrl = 'http://' + self.getSiteDomain() + '/login.php'
@@ -262,8 +262,6 @@ class FineStoriesComAdapter(BaseSiteAdapter):
                 last[len(last)-1]=last[len(last)-1].append(next)
                 div.append(div1)
             
-            
-            
         # removing all the left-over stuff    
         for a in div.findAll('span'):
             a.extract()

fanficdownloader/adapters/adapter_nickandgregnet.py

@@ -166,10 +166,11 @@ class NickAndGregNetAdapter(BaseSiteAdapter):
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.find('table', {'class' : 'tblborder6'})
 
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
+        # wrap a div around it.
+        divsoup = bs.BeautifulStoneSoup('<div class="story"></div>',
+                                        selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        div = divsoup.find('div')
+        div.append(soup.find('table', {'class' : 'tblborder6'}))
+
         return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_simplyundeniablecom.py

@@ -214,5 +214,6 @@ class SimplyUndeniableComAdapter(BaseSiteAdapter):
 
         if None == div:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+        div.name='div'
     
         return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_siyecouk.py

@@ -241,5 +241,6 @@ class SiyeCoUkAdapter(BaseSiteAdapter): # XXX
         
         if None == story:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+        story.name='div'
     
         return self.utf8FromSoup(url,story)

fanficdownloader/adapters/adapter_storiesonlinenet.py

@@ -0,0 +1,305 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return StoriesOnlineNetAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class StoriesOnlineNetAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url
+        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2].split(':')[0])
+        if 'storyInfo' in self.story.getMetadata('storyId'):
+            self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/s/'+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','strol')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%Y-%m-%d"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'storiesonline.net'
+
+    @classmethod
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/s/1234 http://"+self.getSiteDomain()+"/s/1234:4010"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain())+r"/s/\d+(:\d+)?(;\d+)?$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Free Registration' in data \
+                or "Invalid Password!" in data \
+                or "Invalid User Name!" in data \
+                or "Access to unlinked chapters requires" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['theusername'] = self.username
+            params['thepassword'] = self.password
+        else:
+            params['theusername'] = self.getConfig("username")
+            params['thepassword'] = self.getConfig("password")
+        params['rememberMe'] = '1'
+        params['page'] = 'http://'+self.getSiteDomain()+'/'
+        params['submit'] = 'Login'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/login.php'
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['theusername']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "My Account" not in d : #Member Account
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['theusername']))
+            raise exceptions.FailedToLogin(url,params['theusername'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        #print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('h1')
+        self.story.setMetadata('title',stripHTML(a))
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"/a/\w+"))
+        self.story.setMetadata('authorId',a['href'].split('/')[2])
+        self.story.setMetadata('authorUrl','http://'+self.host+a['href'])
+        self.story.setMetadata('author',stripHTML(a).replace("'s Page",""))
+
+        # Find the chapters:
+        chapters = soup.findAll('a', href=re.compile(r'/s/'+self.story.getMetadata('storyId')+":\d+$"))
+        if len(chapters) != 0:
+            for chapter in chapters:
+                # just in case there's tags, like <i> in chapter titles.
+                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+chapter['href']))
+        else:
+            self.chapterUrls.append((self.story.getMetadata('title'),'http://'+self.host+'/s/'+self.story.getMetadata('storyId')))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # surprisingly, the detailed page does not give enough details, so go to author's page
+        skip=0
+        i=0
+        while i == 0:
+            asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')+"&skip="+str(skip)))
+        
+            a = asoup.findAll('td', {'class' : 'lc2'})
+            for lc2 in a:
+                if lc2.find('a')['href'] == '/s/'+self.story.getMetadata('storyId'):
+                    i=1
+                    break
+                if a[len(a)-1] == lc2:
+                    skip=skip+10
+        
+        for cat in lc2.findAll('div', {'class' : 'typediv'}):
+            self.story.addToList('genre',cat.text)
+
+        # in lieu of word count.
+        self.story.setMetadata('size', lc2.findNext('td', {'class' : 'num'}).text)
+        
+        lc4 = lc2.findNext('td', {'class' : 'lc4'})        
+        
+        try:
+            a = lc4.find('a', href=re.compile(r"/library/show_series.php\?id=\d+"))
+            i = a.parent.text.split('(')[1].split(')')[0]
+            self.setSeries(stripHTML(a), i)
+            self.story.setMetadata('seriesUrl','http://'+self.host+a['href'])
+        except:
+            pass
+        try:
+            a = lc4.find('a', href=re.compile(r"/library/universe.php\?id=\d+"))
+            if a:
+                self.story.setMetadata("universe",stripHTML(a))
+        except:
+            pass
+            
+        for a in lc4.findAll('span', {'class' : 'help'}):
+            a.extract()
+        for a in lc4.findAll('br'):
+            a.extract()
+
+        desc = "%s"%lc4
+        desc = desc[desc.index(">")+1:]
+        desc = desc[:desc.index("<")]
+        self.setDescription('http://'+self.host+'/s/'+self.story.getMetadata('storyId'),desc)
+            
+        for b in lc4.findAll('b'):
+            #logger.debug('Getting metadata: "%s"' % b)
+            label = b.text
+            if label in ['Posted:', 'Concluded:', 'Updated:']:
+                value = b.findNext('noscript').text
+                #logger.debug('Have a date field label: "%s", value: "%s"' % (label, value))
+            else:
+                value = b.nextSibling
+            #logger.debug('label: "%s", value: "%s"' % (label, value))
+            
+            if 'Sex' in label:
+                self.story.setMetadata('rating', value)
+                
+            if 'Codes' in label:
+                for code in value.split(' '):
+                    self.story.addToList('codes',code)
+                    
+            if 'Posted' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+                
+            if 'Concluded' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+                
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+#                
+        status = lc4.find('span', {'class' : 'ab'})
+        if  status != None:
+            self.story.setMetadata('status', 'In-Progress')
+            if "Last Activity" in status.text:
+                self.story.setMetadata('dateUpdated', makeDate(status.text.split('Activity: ')[1].split(')')[0], self.dateformat))
+        else:
+            self.story.setMetadata('status', 'Completed')
+
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+        
+        # some big chapters are split over several pages
+        pager = div.find('span', {'class' : 'pager'})
+        if pager != None:
+            urls=pager.findAll('a')
+            urls=urls[:len(urls)-1]
+            
+            
+            for ur in urls:
+                soup = bs.BeautifulSoup(self._fetchUrl("http://"+self.getSiteDomain()+ur['href']),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+                div1 = soup.find('div', {'id' : 'story'})
+                
+                # appending next section
+                last=div.findAll('p')
+                next=div1.find('span', {'class' : 'conTag'}).nextSibling
+            
+                last[len(last)-1]=last[len(last)-1].append(next)
+                div.append(div1)
+            
+        # removing all the left-over stuff    
+        for a in div.findAll('span'):
+            a.extract() 
+            
+        for a in div.findAll('h1'):
+            a.extract()
+        for a in div.findAll('h2'):
+            a.extract()
+        for a in div.findAll('h3'):
+            a.extract()
+        for a in div.findAll('h4'):
+            a.extract()
+        for a in div.findAll('br'):
+            a.extract()
+        for a in div.findAll('div', {'class' : 'date'}):
+            a.extract()
+            
+        a = div.find('form')
+        if a != None:
+            b = a.nextSibling
+            while b != None:
+                a.extract()
+                a=b
+                b=b.nextSibling
+        
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_test1.py

@@ -323,7 +323,7 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
 </div>
 '''
         elif self.story.getMetadata('storyId') == '0':
-            text=u'''
+            text=u'''<div>
 <h3>45. Pronglet Returns to Hogwarts: Chapter 7</h3>
 <br />
     eyes… but I’m not convinced we should automatically<br />
@@ -332,6 +332,7 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
 <br /><br />
     “Sure, invite her along. Does she have children?”<br />
 <br />
+</div>
 '''
         else:
             if self.story.getMetadata('storyId') == '667':

fanficdownloader/adapters/adapter_whoficcom.py

@@ -228,7 +228,8 @@ class WhoficComSiteAdapter(BaseSiteAdapter):
 
         if None == span:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
+
+        span.name='div'
         return self.utf8FromSoup(url,span)
 
 def getClass():

fanficdownloader/adapters/base_adapter.py

@@ -26,6 +26,7 @@ from functools import partial
 
 from .. import BeautifulSoup as bs
 from ..htmlcleanup import stripHTML
+from ..htmlheuristics import replace_br_with_p
 
 logger = logging.getLogger(__name__)
 
@@ -354,13 +355,8 @@ class BaseSiteAdapter(Configurable):
 	    # removes paired, but empty tags.
             if t.string != None and len(t.string.strip()) == 0 :
                 t.extract()
-                
-        retval = soup.__str__('utf8').decode('utf-8')
 
-        if self.getConfig('replace_hr'):
-            # replacing a self-closing tag with a container tag in the
-            # soup is more difficult than it first appears.  So cheat.
-            retval = retval.replace("<hr />","<div class='center'>* * *</div>")
+        retval = soup.__str__('utf8').decode('utf-8')
 
         if self.getConfig('nook_img_fix'):
             # if the <img> tag doesn't have a div or a p around it,
@@ -371,7 +367,19 @@ class BaseSiteAdapter(Configurable):
             
         # Don't want body tags in chapter html--writers add them.
         # This is primarily for epub updates.
-        return re.sub(r"</?body>\r?\n?","",retval)
+        retval = re.sub(r"</?body>\r?\n?","",retval)
+        
+        if self.getConfig("replace_br_with_p"):
+            # Apply heuristic processing to replace <br> paragraph
+            # breaks with <p> tags.
+            retval = replace_br_with_p(retval)
+            
+        if self.getConfig('replace_hr'):
+            # replacing a self-closing tag with a container tag in the
+            # soup is more difficult than it first appears.  So cheat.
+            retval = retval.replace("<hr />","<div class='center'>* * *</div>")
+
+        return retval
 
 def cachedfetch(realfetch,cache,url):
     if url in cache:

fanficdownloader/htmlheuristics.py

@@ -0,0 +1,127 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import logging
+logger = logging.getLogger(__name__)
+import re
+
+from . import exceptions as exceptions
+
+def replace_br_with_p(body):
+
+    # Ascii character (and Unicode as well) xA0 is a non-breaking space, ascii code 160.
+    # However, Python Regex does not recognize it as a whitespace, so we'll be changing it to a reagular space.
+    body = body.replace(u'\xa0', u' ')
+
+    if body.find('>') == -1 or body.rfind("<") == -1:
+        return body
+
+    # change surrounding div to a p and remove attrs Top surrounding
+    # tag in all cases now should be div, to just strip the first and
+    # last tags.
+    body = u'<p>'+body[body.index('>')+1:body.rindex("<")]+u'</p>'
+
+    # So many people add formatting to their HR tags, and ePub does not allow those, we are supposed to use css.
+    # This nukes the hr tag attributes.
+    body = re.sub(r'\s*<hr[^>]+>\s*', r'\n<hr />\n', body)
+
+    # Need to look at BeautifulSoup to see if it'll even return breaks that aren't properly formatted (<br />).
+    body = re.sub(r'\s*<br[^>]*>\s*', r'<br />', body)
+
+    # Remove leading and trailing breaks from HR tags
+    body = re.sub(r'\s*(<br\ \/>)*\s*<hr\ \/>\s*(<br\ \/>)*\s*', r'\n<hr />\n', body)
+    # Nuking breaks leading paragraps that may be in the body. They are eventually treated as <p><br /></p>
+    body = re.sub(r'\s*(<br\ \/>)+\s*<p', r'\n<p></p>\n<p', body)
+    # Nuking breaks trailing paragraps that may be in the body. They are eventually treated as <p><br /></p>
+    body = re.sub(r'</p>\s*(<br\ \/>)+\s*', r'</p>\n<p></p>\n', body)
+
+    # Because a leading or trailing non break tag will break the following code, we have to mess around rather badly for a few lines.
+    body = body.replace(u'[',u'&squareBracketStart;')
+    body = body.replace(u']',u'&squareBracketEnd;')
+    body = body.replace(u'<br />',u'[br /]')
+
+    breaksRegexp = [
+        re.compile(r'([^\]])(\[br\ \/\])([^\[])'),
+        re.compile(r'([^\]])(\[br\ \/\]){2}([^\[])'),
+        re.compile(r'([^\]])(\[br\ \/\]){3}([^\[])'),
+        re.compile(r'([^\]])(\[br\ \/\]){4}([^\[])'),
+        re.compile(r'([^\]])(\[br\ \/\]){5}([^\[])'),
+        re.compile(r'([^\]])(\[br\ \/\]){6}([^\[])'),
+        re.compile(r'([^\]])(\[br\ \/\]){7}([^\[])'),
+        re.compile(r'([^\]])(\[br\ \/\]){8}([^\[])'),
+        re.compile(r'(\[br\ \/\]){9,}')]
+
+    breaksCount = [
+        len(breaksRegexp[0].findall(body)),
+        len(breaksRegexp[1].findall(body)),
+        len(breaksRegexp[2].findall(body)),
+        len(breaksRegexp[3].findall(body)),
+        len(breaksRegexp[4].findall(body)),
+        len(breaksRegexp[5].findall(body)),
+        len(breaksRegexp[6].findall(body)),
+        len(breaksRegexp[7].findall(body))]
+
+    breaksMax = 0
+    breaksMaxIndex = 0;
+
+    for i in range(len(breaksCount)):
+        if breaksCount[i] > breaksMax:
+            breaksMax = breaksCount[i]
+            breaksMaxIndex = i
+
+    # Find all instances of consecutive breaks less than otr equal to the max count use most often
+    #  replase those tags to inverted p tag pairs, those with more connsecutive breaks are replaced them with a horisontal line
+    for i in range(len(breaksCount)):
+        if i <= breaksMaxIndex:
+            body = breaksRegexp[i].sub(r'\1</p>\n<p>\3', body)
+        else:
+            body = breaksRegexp[i].sub(r'\1</p>\n<hr />\n<p>\3', body)
+
+    body = breaksRegexp[8].sub(r'</p>\n<hr />\n<p>', body)
+
+    # Reverting the square brackets
+    body = body.replace(u'[', u'<')
+    body = body.replace(u']', u'>')
+    body = body.replace(u'&squareBracketStart;', u'[')
+    body = body.replace(u'&squareBracketEnd;', u']')
+
+    # If for some reason, a third break makes its way inside the paragraph, preplace that with the empty paragraph for the additional linespaing.
+    body = re.sub(r'<p>\s*(<br\ \/>)+', r'<p><br /></p>\n<p>', body)
+
+    # change empty p tags to include a br to force spacing.
+    body = re.sub(r'<p>\s*</p>', r'<p><br/></p>', body)
+
+    # Clean up hr tags, and add inverted p tag pairs
+    body = re.sub(r'\s*<hr\ \/>\s*', r'</p>\n<hr />\n<p>', body)
+
+    # Because the previous regexp may cause trouble if the hr tag already had a p tag pair around it, w nee dot repair that.
+    # Repeated opening p tags are condenced to one. As we added the extra leading opening p tags, we can safely assume that
+    #  the last in such a chain must be the original. Lets keep its attributes if they are there.
+    body = re.sub(r'\s*(<p[^>]*>\s*)+<p([^>]*)>\s*', r'\n<p\2>', body)
+    # Repeated closing p tags are condenced to one
+    body = re.sub(r'\s*(<\/\s*p>\s*){2,}', r'</p>\n', body)
+
+    # superflous cleaning, remove whitespaces traling opening p tags. These does affect formatting.
+    body = re.sub(r'<p([^>]*)>\s*', r'<p\1>', body)
+    # superflous cleaning, remove whitespaces leading closing p tags. These does not affect formatting.
+    body = re.sub(r'\s*</p>', r'</p>', body)
+
+    # re-wrap in div tag.
+    body = u'<div>\n' + body + u'\n</div>'
+
+    return body 
+

index.html

@@ -46,6 +46,14 @@
       {{yourfile}}
       <!--			</div> -->
 
+          <h3>fanfiction.net</h3>
+          <p>
+            Fanfiction.net appears to be blocking access from Google
+            App Engine, which prevents this web service.  There's
+            nothing I can do about it.  At the time of writing, the
+            latest CLI and calibre plugin versions worked.
+          </p>
+	  
       {% if authorized %}
       <form action="/fdown" method="post">
 	<div id='urlbox'>
@@ -57,18 +65,12 @@
 	  <h3>Changes:</h3>
 	  <p>
 	    <ul>
-              <li>New site: fictionpad.com</li>
+              <li>New site: storiesonline.net</li>
+              <li>Add replace_br_with_p heuristic processing feature.  Turn on in personal.ini</li>
+              <li>Handling for fictionpad.com empty chapters. "This chapter has no text."</li>
 	    </ul>
 	  </p>
           
-          <h3>fanfiction.net</h3>
-          <p>
-            Fanfiction.net appears to be blocking access from Google
-            App Engine, which prevents this web service.  There's
-            nothing I can do about it.  At the time of writing, the
-            latest CLI and calibre plugin versions worked.
-          </p>
-	  
 	  <p>
 	    Questions?  Check out our 
 	    <a href="http://code.google.com/p/fanficdownloader/wiki/FanFictionDownloaderFAQs">FAQs</a>.
@@ -77,7 +79,7 @@
 	    If you have any problems with this application, please
 	    report them in
 	    the <a href="http://groups.google.com/group/fanfic-downloader">FanFictionDownLoader Google Group</a>.  The
-	    <a href="http://4-4-75.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
+	    <a href="http://4-4-79.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
 	  </p>
 	  <div id='error'>
 	    {{ error_message }}

plugin-defaults.ini

@@ -187,6 +187,11 @@ extratags: FanFiction
 ## doesn't work on some devices either.)
 #replace_hr: false
 
+## Some sites/authors/stories use br tags instead of p tags for
+## paragraphs.  This feature uses some heuristics to find and replace
+## br paragraphs with p tags while preserving scene breaks.
+#replace_br_with_p: false
+
 ## If you have the Generate Cover plugin installed, you can use the
 ## generate_cover_settings parameter to intelligently decide which GC
 ## setting to run.  There are three parts 1) a template of which
@@ -778,6 +783,24 @@ dislikes_label:Dislikes
 #username:YourName
 #password:yourpassword
 
+[storiesonline.net]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Clear FanFiction from defaults, site is original fiction.
+extratags:
+
+extra_valid_entries:size,universe,codes
+#extra_titlepage_entries:size,universe,codes
+
+size_label:Size
+universe_label:Universe
+codes_label:Codes
+
 [grangerenchanted.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter
@@ -1140,6 +1163,10 @@ never_make_cover: true
 ## for examples of how to use them.
 extra_valid_entries:reviews,favs,follows
 
+## ffnet uses 'Pairings', not 'Relationship', stating they don't have
+## to be romantic pairings.
+ships_label:Pairings
+
 [www.fanfiktion.de]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In