Bump versions, update web service to announce fimf blocking.

Also cleaned up paragraphs a little more, mostly cosmetic.

app.yaml

@@ -1,6 +1,6 @@
 # ffd-retief-hrd fanfictiondownloader
 application: fanfictiondownloader
-version: 4-4-80
+version: 4-4-81
 runtime: python27
 api_version: 1
 threadsafe: true

calibre-plugin/__init__.py

@@ -36,7 +36,7 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
     description         = 'UI plugin to download FanFiction stories from various sites.'
     supported_platforms = ['windows', 'osx', 'linux']
     author              = 'Jim Miller'
-    version             = (1, 7, 51)
+    version             = (1, 7, 52)
     minimum_calibre_version = (0, 8, 57)
 
     #: This field defines the GUI plugin class that contains all the code

calibre-plugin/ffdl_plugin.py

@@ -78,7 +78,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
     # (text, icon_path, tooltip, keyboard shortcut)
     # icon_path isn't in the zip--icon loaded below.
     action_spec = (name, None,
-                   'Download FanFiction stories from various web sites', ())
+                   _('Download FanFiction stories from various web sites'), ())
     # None for keyboard shortcut doesn't allow shortcut.  () does, there just isn't one yet
 
     action_type = 'global'
@@ -108,7 +108,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         # are not found in the zip file will result in null QIcons.
         icon = get_icon('images/icon.png')
 
-        self.qaction.setText('FanFictionDL')
+        self.qaction.setText(_('FanFictionDL'))
         
         # The qaction is automatically created from the action_spec defined
         # above
@@ -709,7 +709,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
             options['tdir']=tdir
 
         if 0 < len(filter(lambda x : x['good'], books)):
-            self.gui.status_bar.show_message(_('Started fetching metadata for %s stories.'%len(books)), 3000)
+            self.gui.status_bar.show_message(_('Started fetching metadata for %s stories.')%len(books), 3000)
             LoopProgressDialog(self.gui,
                                books,
                                partial(self.prep_download_loop, options = options, merge=merge),
@@ -1172,7 +1172,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                 # Sort by our marked column to display the books in order
                 self.gui.library_view.sort_by_named_field('marked', True)
                 
-        self.gui.status_bar.show_message(_('Finished Adding/Updating %d books.'%(len(update_list) + len(add_list))), 3000)
+        self.gui.status_bar.show_message(_('Finished Adding/Updating %d books.')%(len(update_list) + len(add_list)), 3000)
         remove_dir(options['tdir'])
         
         if 'Count Pages' in self.gui.iactions and len(prefs['countpagesstats']) and len(all_ids):
@@ -1273,7 +1273,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         good_list = sorted(good_list,key=lambda x : x['listorder'])
         bad_list = sorted(bad_list,key=lambda x : x['listorder'])
         
-        self.gui.status_bar.show_message(_('Merging %s books.'%total_good))
+        self.gui.status_bar.show_message(_('Merging %s books.')%total_good)
 
         
         existingbook = None
@@ -1331,7 +1331,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         custom_columns = self.gui.library_view.model().custom_columns
         if prefs['mark'] or (prefs['errorcol'] != '' and prefs['errorcol'] in custom_columns):
             self.previous = self.gui.library_view.currentIndex() # used by update_books_finish.
-            self.gui.status_bar.show_message(_('Adding/Updating %s BAD books.'%len(book_list)))
+            self.gui.status_bar.show_message(_('Adding/Updating %s BAD books.')%len(book_list))
             label = custom_columns[prefs['errorcol']]['label']
             LoopProgressDialog(self.gui,
                                book_list,

fanficdownloader/adapters/adapter_fimfictionnet.py

@@ -252,7 +252,7 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
         rawGroupList = soup.find('ul', {'id':'story_group_list'})
         if rawGroupList is not None:
             for groupName in rawGroupList.findAll('a', {'href':re.compile('^/group/')}):
-                self.story.addToList("groups",stripHTML(groupName))
+                self.story.addToList("groups",stripHTML(groupName).replace(',', ';'))
             
     def hookForUpdates(self,chaptercount):
         if self.oldchapters and len(self.oldchapters) > self.newestChapterNum:

fanficdownloader/adapters/adapter_storiesonlinenet.py

@@ -71,13 +71,14 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
         return "http://"+self.getSiteDomain()+"/s/1234 http://"+self.getSiteDomain()+"/s/1234:4010"
 
     def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain())+r"/s/\d+(:\d+)?(;\d+)?$"
+        return re.escape("http://"+self.getSiteDomain())+r"/s/\d+((:\d+)?(;\d+)?$|(:i)?$)"
 
     ## Login seems to be reasonably standard across eFiction sites.
     def needToLoginCheck(self, data):
         if 'Free Registration' in data \
                 or "Invalid Password!" in data \
                 or "Invalid User Name!" in data \
+                or "Log In" in data \
                 or "Access to unlinked chapters requires" in data:
             return True
         else:
@@ -119,7 +120,7 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
         logger.debug("URL: "+url)
 
         try:
-            data = self._fetchUrl(url)
+            data = self._fetchUrl(url+":i")
         except urllib2.HTTPError, e:
             if e.code == 404:
                 raise exceptions.StoryDoesNotExist(self.url)
@@ -129,10 +130,12 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
         if self.needToLoginCheck(data):
             # need to log in for this one.
             self.performLogin(url)
-            data = self._fetchUrl(url)
-            
+            data = self._fetchUrl(url+":i")
+        
         if "Access denied. This story has not been validated by the adminstrators of this site." in data:
             raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+        elif "Error! The story you're trying to access is being filtered by your choice of contents filtering." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Error! The story you're trying to access is being filtered by your choice of contents filtering.")
             
         # use BeautifulSoup HTML parser to make everything easier to find.
         soup = bs.BeautifulSoup(data)
@@ -151,7 +154,7 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
         self.story.setMetadata('author',stripHTML(a).replace("'s Page",""))
 
         # Find the chapters:
-        chapters = soup.findAll('a', href=re.compile(r'/s/'+self.story.getMetadata('storyId')+":\d+$"))
+        chapters = soup.findAll('a', href=re.compile(r'^/s/'+self.story.getMetadata('storyId')+":\d+$"))
         if len(chapters) != 0:
             for chapter in chapters:
                 # just in case there's tags, like <i> in chapter titles.
@@ -166,7 +169,7 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
         i=0
         while i == 0:
             asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')+"&skip="+str(skip)))
-        
+
             a = asoup.findAll('td', {'class' : 'lc2'})
             for lc2 in a:
                 if lc2.find('a')['href'] == '/s/'+self.story.getMetadata('storyId'):
@@ -174,7 +177,7 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
                     break
                 if a[len(a)-1] == lc2:
                     skip=skip+10
-        
+
         for cat in lc2.findAll('div', {'class' : 'typediv'}):
             self.story.addToList('genre',cat.text)
 
@@ -197,14 +200,8 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
         except:
             pass
             
-        for a in lc4.findAll('span', {'class' : 'help'}):
-            a.extract()
-        for a in lc4.findAll('br'):
-            a.extract()
 
-        desc = "%s"%lc4
-        desc = desc[desc.index(">")+1:]
-        desc = desc[:desc.index("<")]
+        desc = lc4.contents[0]
         self.setDescription('http://'+self.host+'/s/'+self.story.getMetadata('storyId'),desc)
             
         for b in lc4.findAll('b'):
@@ -238,7 +235,9 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
         if  status != None:
             self.story.setMetadata('status', 'In-Progress')
             if "Last Activity" in status.text:
-                self.story.setMetadata('dateUpdated', makeDate(status.text.split('Activity: ')[1].split(')')[0], self.dateformat))
+                # date is passed as a timestamp and converted in JS.
+                value = status.findNext('noscript').text
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
         else:
             self.story.setMetadata('status', 'Completed')
 
@@ -258,8 +257,7 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
         if pager != None:
             urls=pager.findAll('a')
             urls=urls[:len(urls)-1]
-            
-            
+                        
             for ur in urls:
                 soup = bs.BeautifulSoup(self._fetchUrl("http://"+self.getSiteDomain()+ur['href']),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.

fanficdownloader/adapters/base_adapter.py

@@ -352,8 +352,8 @@ class BaseSiteAdapter(Configurable):
             if t.name in ('center'):
                 t['class']=t.name
                 t.name='div'
-	    # removes paired, but empty tags.
-            if t.string != None and len(t.string.strip()) == 0 :
+	    # removes paired, but empty non paragraph tags.
+            if t.name not in ('p') and t.string != None and len(t.string.strip()) == 0 :
                 t.extract()
 
         retval = soup.__str__('utf8').decode('utf-8')

fanficdownloader/htmlheuristics.py

@@ -30,18 +30,51 @@ def replace_br_with_p(body):
     if body.find('>') == -1 or body.rfind("<") == -1:
         return body
 
+    # logger.debug(u'BODY start.: ' + body[:250])
+    # logger.debug(u'BODY end...: ' + body[-250:])
+    # logger.debug(u'BODY.......: ' + body)
+
     # change surrounding div to a p and remove attrs Top surrounding
     # tag in all cases now should be div, to just strip the first and
     # last tags.
-    body = u'<p>'+body[body.index('>')+1:body.rindex("<")]+u'</p>'
+    body = body[body.index('>')+1:body.rindex("<")]
+
+    # Need to look at BeautifulSoup to see if it'll even return breaks that aren't properly formatted (<br />).
+    body = re.sub(r'\s*<br[^>]*>\s*', r'<br />', body)
+
+    # Find all bexisting blocks with p, pre and blockquote tags, we need to shields break tags inside those.
+    # This is for "lenient" mode, however it is also used to clear break tags before and after the block elements.
+    blocksRegex = re.compile(r'(\s*<br\ */*>\s*)*\s*<(pre|p|blockquote)([^>]*)>(.+?)</\2>\s*(\s*<br\ */*>\s*)*', re.DOTALL)
+    body = blocksRegex.sub(r'\n<\2\3>\4</\2>\n', body)
+
+    # if aggressive mode = true
+        # blocksRegex = re.compile(r'(\s*<br\ */*>\s*)*\s*<(pre)([^>]*)>(.+?)</\2>\s*(\s*<br\ */*>\s*)*', re.DOTALL)
+        # In aggressive mode, we also check breakes inside blockquotes, meaning we can get orphaned paragraph tags.
+        # body = re.sub(r'<blockquote([^>]*)>(.+?)</blockquote>', r'<blockquote\1><p>\2</p></blockquote>', body, re.DOTALL)
+    # end aggressive mode
+
+    blocks = blocksRegex.finditer(body)
+    # For our replacements to work, we need to work backwards, so we reverse the iterator.
+    blocksList = []
+    for match in blocks:
+        blocksList.insert(0, match)
+
+    for match in blocksList:
+        group4 =  match.group(4).replace(u'<br />', u'{br /}')
+        body = body[:match.start(4)] + group4 + body[match.end(4):]
+
+    # change surrounding div to a p and remove attrs Top surrounding
+    # tag in all cases now should be div, to just strip the first and
+    # last tags.
+    body = u'<p>' + body + u'</p>'
+
+    # Nuke div tags surrounding a HR tag.
+    body = re.sub(r'<div[^>]+>\s*<hr[^>]+>\s*</div>', r'\n<hr />\n', body)
 
     # So many people add formatting to their HR tags, and ePub does not allow those, we are supposed to use css.
     # This nukes the hr tag attributes.
     body = re.sub(r'\s*<hr[^>]+>\s*', r'\n<hr />\n', body)
 
-    # Need to look at BeautifulSoup to see if it'll even return breaks that aren't properly formatted (<br />).
-    body = re.sub(r'\s*<br[^>]*>\s*', r'<br />', body)
-
     # Remove leading and trailing breaks from HR tags
     body = re.sub(r'\s*(<br\ \/>)*\s*<hr\ \/>\s*(<br\ \/>)*\s*', r'\n<hr />\n', body)
     # Nuking breaks leading paragraps that may be in the body. They are eventually treated as <p><br /></p>
@@ -78,17 +111,66 @@ def replace_br_with_p(body):
     breaksMax = 0
     breaksMaxIndex = 0;
 
-    for i in range(len(breaksCount)):
-        if breaksCount[i] > breaksMax:
+    for i in range(1,len(breaksCount)):
+        if breaksCount[i] >= breaksMax:
             breaksMax = breaksCount[i]
             breaksMaxIndex = i
 
+    lines = body.split(u'[br /]')
+    contentLines = 0;
+    contentLinesSum = 0;
+    longestLineLength = 0;
+    averageLineLength = 0;
+
+    for line in lines:
+        lineLen = len(line.strip())
+        if lineLen > 0:
+            contentLines += 1
+            contentLinesSum += lineLen
+            if lineLen > longestLineLength:
+                longestLineLength = lineLen
+
+    averageLineLength = contentLinesSum/contentLines
+
+    logger.debug(u'---')
+    logger.debug(u'Lines.............: ' + str(len(lines)))
+    logger.debug(u'contentLines......: ' + str(contentLines))
+    logger.debug(u'contentLinesSum...: ' + str(contentLinesSum))
+    logger.debug(u'longestLineLength.: ' + str(longestLineLength))
+    logger.debug(u'averageLineLength.: ' + str(averageLineLength))
+
+    if breaksMaxIndex == len(breaksCount)-1 and breaksMax < 2:
+        breaksMaxIndex = 0
+        breaksMax = breaksCount[0]
+
+    logger.debug(u'---')
+    logger.debug(u'breaks 1: ' + str(breaksCount[0]))
+    logger.debug(u'breaks 2: ' + str(breaksCount[1]))
+    logger.debug(u'breaks 3: ' + str(breaksCount[2]))
+    logger.debug(u'breaks 4: ' + str(breaksCount[3]))
+    logger.debug(u'breaks 5: ' + str(breaksCount[4]))
+    logger.debug(u'breaks 6: ' + str(breaksCount[5]))
+    logger.debug(u'breaks 7: ' + str(breaksCount[6]))
+    logger.debug(u'breaks 8: ' + str(breaksCount[7]))
+    logger.debug(u'----')
+    logger.debug(u'max found: ' + str(breaksMax))
+    logger.debug(u'max Index: ' + str(breaksMaxIndex))
+    logger.debug(u'----')
+
+    if breaksMaxIndex > 0 and breaksCount[0] > breaksMax and averageLineLength < 90:
+        body = breaksRegexp[0].sub(r'\1 \n\3', body)
+
     # Find all instances of consecutive breaks less than otr equal to the max count use most often
     #  replase those tags to inverted p tag pairs, those with more connsecutive breaks are replaced them with a horisontal line
     for i in range(len(breaksCount)):
         if i <= breaksMaxIndex:
+            logger.debug(str(i) + u' <= breaksMaxIndex (' + str(breaksMaxIndex) + u')')
             body = breaksRegexp[i].sub(r'\1</p>\n<p>\3', body)
+        elif i == breaksMaxIndex+1:
+            logger.debug(str(i) + u' == breaksMaxIndex+1 (' + str(breaksMaxIndex+1) + u')')
+            body = breaksRegexp[i].sub(r'\1</p>\n<p><br/></p>\n<p>\3', body)
         else:
+            logger.debug(str(i) + u' > breaksMaxIndex+1 (' + str(breaksMaxIndex+1) + u')')
             body = breaksRegexp[i].sub(r'\1</p>\n<hr />\n<p>\3', body)
 
     body = breaksRegexp[8].sub(r'</p>\n<hr />\n<p>', body)
@@ -105,6 +187,9 @@ def replace_br_with_p(body):
     # change empty p tags to include a br to force spacing.
     body = re.sub(r'<p>\s*</p>', r'<p><br/></p>', body)
 
+    # Clean up hr tags, and add inverted p tag pairs
+    body = re.sub(r'(<div[^>]+>)*\s*<hr\ \/>\s*(</div>)*', r'\n<hr />\n', body)
+
     # Clean up hr tags, and add inverted p tag pairs
     body = re.sub(r'\s*<hr\ \/>\s*', r'</p>\n<hr />\n<p>', body)
 
@@ -116,12 +201,17 @@ def replace_br_with_p(body):
     body = re.sub(r'\s*(<\/\s*p>\s*){2,}', r'</p>\n', body)
 
     # superflous cleaning, remove whitespaces traling opening p tags. These does affect formatting.
-    body = re.sub(r'<p([^>]*)>\s*', r'<p\1>', body)
+    body = re.sub(r'\s*<p([^>]*)>\s*', r'\n<p\1>', body)
     # superflous cleaning, remove whitespaces leading closing p tags. These does not affect formatting.
-    body = re.sub(r'\s*</p>', r'</p>', body)
+    body = re.sub(r'\s*</p>\s*', r'</p>\n', body)
 
+    # Remove empty tag pairs
+    body = re.sub(r'\s*<(\S+)[^>]*>\s*</\1>', r'', body)
+
+    body = body.replace(u'{br /}', u'<br />')
+    
     # re-wrap in div tag.
-    body = u'<div>\n' + body + u'\n</div>'
+    body = u'<div>\n' + body + u'</div>\n'
 
     return body 
 

index.html

@@ -46,13 +46,16 @@
       {{yourfile}}
       <!--			</div> -->
 
-          <h3>fanfiction.net</h3>
+          <h3>fanfiction.net / fimfiction.net</h3>
           <p>
             Fanfiction.net appears to be blocking access from Google
             App Engine, which prevents this web service.  There's
             nothing I can do about it.  At the time of writing, the
             latest CLI and calibre plugin versions worked.
           </p>
+          <p>It appears that FimFiction.net is also blocking access from Google
+            App Engine now.
+          </p>
 	  
       {% if authorized %}
       <form action="/fdown" method="post">
@@ -65,9 +68,8 @@
 	  <h3>Changes:</h3>
 	  <p>
 	    <ul>
-              <li>New site: storiesonline.net</li>
-              <li>Add replace_br_with_p heuristic processing feature.  Turn on in personal.ini</li>
-              <li>Handling for fictionpad.com empty chapters. "This chapter has no text."</li>
+              <li>Fixes for storiesonline.net, thanks davidfor</li>
+              <li>Additional fixes/enhancements for replace_br_with_p heuristic processing feature.  Turn on in personal User Config.  Thanks, Asbjørn Grandt.</li>
 	    </ul>
 	  </p>
           
@@ -79,7 +81,7 @@
 	    If you have any problems with this application, please
 	    report them in
 	    the <a href="http://groups.google.com/group/fanfic-downloader">FanFictionDownLoader Google Group</a>.  The
-	    <a href="http://4-4-79.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
+	    <a href="http://4-4-80.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
 	  </p>
 	  <div id='error'>
 	    {{ error_message }}

status.html

@@ -51,13 +51,16 @@
 	      by {{ fic.author }} ({{ fic.format }})
 	  {% endif %}
 	  {% if fic.failure %}
-          <h3>fanfiction.net</h3>
+          <h3>fanfiction.net / fimfiction.net</h3>
           <p>
             FYI, fanfiction.net appears to be blocking access from Google
             App Engine, which prevents this web service.  There's
             nothing I can do about it.  At the time of writing, the
             latest CLI and calibre plugin versions worked.
           </p>
+          <p>It appears that FimFiction.net is also blocking access from Google
+            App Engine now.
+          </p>
 	  <span id='error'>{{ fic.failure }}</span>
 	  {% endif %}
 	  {% if  not fic.completed and not fic.failure %}