diff --git a/fanficdownloader/HtmlTagStack.py b/fanficdownloader/HtmlTagStack.py new file mode 100644 index 0000000..f166ff3 --- /dev/null +++ b/fanficdownloader/HtmlTagStack.py @@ -0,0 +1,57 @@ +# coding: utf-8 + +import re +import codecs + +stack = [] + +def get_end_tag(tag): + if len(tag) > 0 and tag.find(u'<') > -1 and tag.rfind(u'>') > -1: + return re.sub(r'.*<([^\ >]+).*', r'', tag) + return u'' + +def get_tag_name(tag): + if len(tag) > 0 and tag.find(u'<') > -1 and tag.rfind(u'>') > -1: + return re.sub(r']+).*', r'\1', tag) + return u'' + +def push(tag): + if len(tag) > 0 and tag.find(u'<') > -1 and tag.rfind(u'>') > -1: + stack.append(tag) + +def pop(): + if len(stack) > 0: + return stack.pop() + return u'' + +def pop_end_tag(): + return unicode(get_end_tag(pop())) + +def spool_end(): + html = u'' + for tag in reversed(stack): + html += get_end_tag(tag) + return html + +def spool_start(): + html = u'' + for item in stack: + html += item + return html + +def has_elements(): + return len(stack) > 0 + +def get_last(): + # t = pop() + # push(t) + # return t + if len(stack) > 0: + return stack[len(stack)-1] + return u'' + +def flush(): + del stack[:] + +def get_stack(): + return stack \ No newline at end of file diff --git a/fanficdownloader/htmlheuristics.py b/fanficdownloader/htmlheuristics.py index 54fef4d..f803198 100644 --- a/fanficdownloader/htmlheuristics.py +++ b/fanficdownloader/htmlheuristics.py @@ -18,7 +18,9 @@ import logging logger = logging.getLogger(__name__) import re +import codecs import BeautifulSoup as bs +import HtmlTagStack as stack from . import exceptions as exceptions @@ -28,7 +30,7 @@ def replace_br_with_p(body): # However, Python Regex does not recognize it as a whitespace, so we'll be changing it to a reagular space. body = body.replace(u'\xa0', u' ') - if body.find('>') == -1 or body.rfind("<") == -1: + if body.find('>') == -1 or body.rfind('<') == -1: return body # logger.debug(u'BODY start.: ' + body[:250]) @@ -42,11 +44,11 @@ def replace_br_with_p(body): # tag in all cases now should be div, to just strip the first and # last tags. if is_valid_block(body) and body.find('')+1:body.rindex("<")] + body = body[body.index('>')+1:body.rindex('<')] body = soup_up_div(u'
' + body + u'
') - body = body[body.index('>')+1:body.rindex("<")] + body = body[body.index('>')+1:body.rindex('<')] # Find all bexisting blocks with p, pre and blockquote tags, we need to shields break tags inside those. # This is for "lenient" mode, however it is also used to clear break tags before and after the block elements. @@ -169,6 +171,7 @@ def replace_br_with_p(body): # Find all instances of consecutive breaks less than otr equal to the max count use most often # replase those tags to inverted p tag pairs, those with more connsecutive breaks are replaced them with a horisontal line for i in range(len(breaksCount)): + # if i > 0 or breaksMaxIndex == 0: if i <= breaksMaxIndex: logger.debug(str(i) + u' <= breaksMaxIndex (' + str(breaksMaxIndex) + u')') body = breaksRegexp[i].sub(r'\1

\n

\3', body) @@ -223,7 +226,8 @@ def replace_br_with_p(body): # re-wrap in div tag. body = u'

\n' + body + u'
\n' - return body + # return body + return tag_sanitizer(body) def is_valid_block(block): return str(block).find('<') == 0 and str(block).find('') return tag + body + tagend + + +def is_end_tag(tag): + return re.match(r']+)>', tag) != None + +def is_comment_tag(tag): + return re.match(r'<\!\-\-([^>]+)>', tag) != None + +def is_closed_tag(tag): + return re.match(r'<(.+?)/>', tag) != None + +def tag_sanitizer(html): + blockTags = ['address', 'blockquote', 'del', 'div', 'dl', 'fieldset', 'form', 'ins', 'noscript', 'ol', 'pre', 'table', 'ul'] + + body = u'' + tags = re.findall(r'(<[^>]+>)([^<]*)', html) + + for rTag in tags: + name = stack.get_tag_name(rTag[0]) + is_end = is_end_tag(rTag[0]) + is_closed = is_closed_tag(rTag[0]) or is_comment_tag(rTag[0]) + + # is_comment = is_comment_tag(rTag[0]) + # logger.debug(u'%s > isEnd: %s > isClosed: %s > isComment: %s'%(name, str(is_end), str(is_closed), str(is_comment))) + # logger.debug(u'> %s%s\n'%(rTag[0], rTag[1])) + + if name in blockTags: + body += rTag[0] + body += rTag[1] + elif name == u'p': + if is_end: + body += stack.spool_end() + body += rTag[0] + body += rTag[1] + elif is_closed: + body += rTag[0] + body += rTag[1] + else: + body += rTag[0] + body += stack.spool_start() + body += rTag[1] + else: + if is_end: + t = stack.get_last() + tn = stack.get_tag_name(t) + rTn = stack.get_tag_name(rTag[0]) + if tn == rTn: + body += rTag[0] + stack.pop() + elif not is_closed: + stack.push(rTag[0]) + body += rTag[0] + else: + body += rTag[0] + + body += rTag[1] + stack.flush() + return body