diff --git a/fanficdownloader/BeautifulSoup.py b/fanficdownloader/BeautifulSoup.py
index 4b17b85..5414aa8 100644
--- a/fanficdownloader/BeautifulSoup.py
+++ b/fanficdownloader/BeautifulSoup.py
@@ -114,6 +114,8 @@ class PageElement(object):
"""Contains the navigational information for some part of the page
(either a tag or a piece of text)"""
+ bs3=True
+
def setup(self, parent=None, previous=None):
"""Sets up the initial relations between this element and
other elements."""
diff --git a/fanficdownloader/adapters/adapter_tthfanficorg.py b/fanficdownloader/adapters/adapter_tthfanficorg.py
index 8bb9130..bfd6bdd 100644
--- a/fanficdownloader/adapters/adapter_tthfanficorg.py
+++ b/fanficdownloader/adapters/adapter_tthfanficorg.py
@@ -286,8 +286,9 @@ class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter):
self.story.addToList('category','Buffy: The Vampire Slayer')
pseries = soup.find('p', {'style':'margin-top:0px'})
- m = re.match('This story is No\. (?P tag doesn't have a div or a p around it,
diff --git a/fanficdownloader/epubutils.py b/fanficdownloader/epubutils.py
index 6d2e6ff..2e1b041 100644
--- a/fanficdownloader/epubutils.py
+++ b/fanficdownloader/epubutils.py
@@ -14,7 +14,7 @@ import re, os, traceback
from zipfile import ZipFile
from xml.dom.minidom import parseString
-from . import BeautifulSoup as bs
+import bs4 as bs
def get_dcsource(inputio):
return get_update_data(inputio,getfilecount=False,getsoups=False)[0]
diff --git a/fanficdownloader/htmlcleanup.py b/fanficdownloader/htmlcleanup.py
index 09d2985..1544c81 100644
--- a/fanficdownloader/htmlcleanup.py
+++ b/fanficdownloader/htmlcleanup.py
@@ -56,10 +56,14 @@ def _replaceNotEntities(data):
return p.sub(r'&\1', data)
def stripHTML(soup):
- return removeAllEntities(re.sub(r'<[^>]+>','',"%s" % soup)).strip()
+ if isinstance(soup,basestring) or soup.has_attr('bs3'):
+ return removeAllEntities(re.sub(r'<[^>]+>','',"%s" % soup)).strip()
+ else:
+ # bs4 already converts all the entities to UTF8 chars.
+ return soup.get_text(strip=True)
def conditionalRemoveEntities(value):
- if isinstance(value,str) or isinstance(value,unicode) :
+ if isinstance(value,basestring):
return removeEntities(value).strip()
else:
return value
diff --git a/fanficdownloader/writers/writer_epub.py b/fanficdownloader/writers/writer_epub.py
index fb405a3..3b1b19d 100644
--- a/fanficdownloader/writers/writer_epub.py
+++ b/fanficdownloader/writers/writer_epub.py
@@ -662,7 +662,7 @@ div { margin: 0pt; padding: 0pt; }
# The replaces above added tons of extra newlines
# during *each* epub update. The regexp version adds
# only one and removes any extra.
- fullhtml = re.sub(r'(