diff --git a/fanficdownloader/htmlheuristics.py b/fanficdownloader/htmlheuristics.py index d33e2b2..d24c57a 100644 --- a/fanficdownloader/htmlheuristics.py +++ b/fanficdownloader/htmlheuristics.py @@ -23,7 +23,6 @@ from . import exceptions as exceptions def replace_br_with_p(body): - # Ascii character (and Unicode as well) xA0 is a non-breaking space, ascii code 160. # However, Python Regex does not recognize it as a whitespace, so we'll be changing it to a reagular space. body = body.replace(u'\xa0', u' ') @@ -33,6 +32,7 @@ def replace_br_with_p(body): # logger.debug(u'BODY start.: ' + body[:250]) # logger.debug(u'BODY end...: ' + body[-250:]) + # logger.debug(u'BODY.......: ' + body) # change surrounding div to a p and remove attrs Top surrounding # tag in all cases now should be div, to just strip the first and @@ -42,10 +42,17 @@ def replace_br_with_p(body): # Need to look at BeautifulSoup to see if it'll even return breaks that aren't properly formatted (
). body = re.sub(r'\s*]*>\s*', r'
', body) - # Find all bexisting blocks with p, pre and blockquote tags, we need to leave those alone. + # Find all bexisting blocks with p, pre and blockquote tags, we need to shields break tags inside those. + # This is for "lenient" mode, however it is also used to clear break tags before and after the block elements. blocksRegex = re.compile(r'(\s*\s*)*\s*<(pre|p|blockquote)([^>]*)>(.+?)\s*(\s*\s*)*', re.DOTALL) body = blocksRegex.sub(r'\n<\2\3>\4\n', body) + # if aggressive mode = true + # blocksRegex = re.compile(r'(\s*\s*)*\s*<(pre)([^>]*)>(.+?)\s*(\s*\s*)*', re.DOTALL) + # In aggressive mode, we also check breakes inside blockquotes, meaning we can get orphaned paragraph tags. + # body = re.sub(r']*)>(.+?)', r'

\2

', body, re.DOTALL) + # end aggressive mode + blocks = blocksRegex.finditer(body) # For our replacements to work, we need to work backwards, so we reverse the iterator. blocksList = [] @@ -194,9 +201,9 @@ def replace_br_with_p(body): body = re.sub(r'\s*(<\/\s*p>\s*){2,}', r'

\n', body) # superflous cleaning, remove whitespaces traling opening p tags. These does affect formatting. - body = re.sub(r']*)>\s*', r'', body) + body = re.sub(r'\s*]*)>\s*', r'\n', body) # superflous cleaning, remove whitespaces leading closing p tags. These does not affect formatting. - body = re.sub(r'\s*

', r'

', body) + body = re.sub(r'\s*

\s*', r'

\n', body) # Remove empty tag pairs body = re.sub(r'\s*<(\S+)[^>]*>\s*', r'', body) @@ -204,7 +211,7 @@ def replace_br_with_p(body): body = body.replace(u'{br /}', u'
') # re-wrap in div tag. - body = u'
\n' + body + u'\n
' + body = u'
\n' + body + u'
\n' return body