diff --git a/fanficdownloader/htmlheuristics.py b/fanficdownloader/htmlheuristics.py
index d24c57a..d416385 100644
--- a/fanficdownloader/htmlheuristics.py
+++ b/fanficdownloader/htmlheuristics.py
@@ -18,6 +18,7 @@
import logging
logger = logging.getLogger(__name__)
import re
+import BeautifulSoup as bs
from . import exceptions as exceptions
@@ -34,17 +35,22 @@ def replace_br_with_p(body):
# logger.debug(u'BODY end...: ' + body[-250:])
# logger.debug(u'BODY.......: ' + body)
+ # clean breaks (
), removing whitespaces between them.
+ body = re.sub(r'\s*
]*>\s*', r'
', body)
+
# change surrounding div to a p and remove attrs Top surrounding
# tag in all cases now should be div, to just strip the first and
# last tags.
- body = body[body.index('>')+1:body.rindex("<")]
+ if is_valid_block(body) and body.find('
')+1:body.rindex("<")]
- # Need to look at BeautifulSoup to see if it'll even return breaks that aren't properly formatted (
).
- body = re.sub(r'\s*
]*>\s*', r'
', body)
+ body = soup_up_div(u'
' + body + u'
')
+
+ body = body[body.index('>')+1:body.rindex("<")]
# Find all bexisting blocks with p, pre and blockquote tags, we need to shields break tags inside those.
# This is for "lenient" mode, however it is also used to clear break tags before and after the block elements.
- blocksRegex = re.compile(r'(\s*
\s*)*\s*<(pre|p|blockquote)([^>]*)>(.+?)\2>\s*(\s*
\s*)*', re.DOTALL)
+ blocksRegex = re.compile(r'(\s*
\s*)*\s*<(pre|p|blockquote|table)([^>]*)>(.+?)\2>\s*(\s*
\s*)*', re.DOTALL)
body = blocksRegex.sub(r'\n<\2\3>\4\2>\n', body)
# if aggressive mode = true
@@ -66,7 +72,7 @@ def replace_br_with_p(body):
# change surrounding div to a p and remove attrs Top surrounding
# tag in all cases now should be div, to just strip the first and
# last tags.
- body = u'
' + body + u'
'
+ # body = u'
' + body + u'
'
# Nuke div tags surrounding a HR tag.
body = re.sub(r'
]+>\s*
]+>\s*', r'\n
\n', body)
@@ -181,6 +187,9 @@ def replace_br_with_p(body):
body = body.replace(u'&squareBracketStart;', u'[')
body = body.replace(u'&squareBracketEnd;', u']')
+ body = body.replace(u'{p}', u'
')
+ body = body.replace(u'{/p}', u'
')
+
# If for some reason, a third break makes its way inside the paragraph, preplace that with the empty paragraph for the additional linespaing.
body = re.sub(r'
\s*(
)+', r'
\n
', body)
@@ -209,9 +218,64 @@ def replace_br_with_p(body):
body = re.sub(r'\s*<(\S+)[^>]*>\s*\1>', r'', body)
body = body.replace(u'{br /}', u'
')
-
+ body = body.strip()
+
# re-wrap in div tag.
body = u'
\n' + body + u'
\n'
- return body
+ return body
+def is_valid_block(block):
+ return str(block).find('<') == 0
+
+def soup_up_div(body):
+ blockTags = ['address', 'blockquote', 'del', 'div', 'dl', 'fieldset', 'form', 'ins', 'noscript', 'ol', 'p', 'pre', 'table', 'ul']
+ recurseTags = ['blockquote', 'div', 'noscript']
+
+ tag = body[:body.index('>')+1]
+ tagend = body[body.rindex('<'):]
+
+ body = body.replace(u'
', u'[br /]')
+
+ soup = bs.BeautifulSoup(body)
+
+ body = u''
+ lastElement = 1 # 1 = block, 2 = nested, 3 = invalid
+
+ for i in soup.contents[0]:
+ if str(i).strip().__len__() > 0:
+ s = str(i)
+ if is_valid_block(i):
+ if i.name in blockTags:
+ if lastElement > 1:
+ body = body.strip(r'\s*(\[br\ \/\]\s*)*\s*')
+ body += u'{/p}'
+
+ lastElement = 1
+
+ if i.name in recurseTags:
+ s = soup_up_div(s)
+
+ body += s.strip() + '\n'
+ else:
+ if lastElement == 1:
+ body = body.strip(r'\s*(\[br\ \/\]\s*)*\s*')
+ body += u'{p}'
+
+ lastElement = 2
+ body += s
+ else:
+ if lastElement == 1:
+ body = body.strip(r'\s*(\[br\ \/\]\s*)*\s*')
+ body += u'{p}'
+
+ lastElement = 3
+ body += s
+
+ if lastElement > 1:
+ body = body.strip(r'\s*(\[br\ \/\]\s*)*\s*')
+ body += u'{/p}'
+
+ body = body.replace(u'[br /]', u'
')
+
+ return tag + body + tagend