From 89873d153ed7f933ccd47cd26277d55775a87946 Mon Sep 17 00:00:00 2001
From: Jim Miller <retiefjimm@gmail.com>
Date: Tue, 2 Dec 2014 15:52:03 -0600
Subject: [PATCH] More changes for bs4.

---
 fanficdownloader/BeautifulSoup.py                 | 2 ++
 fanficdownloader/adapters/adapter_tthfanficorg.py | 5 +++--
 fanficdownloader/adapters/base_adapter.py         | 5 +++--
 fanficdownloader/epubutils.py                     | 2 +-
 fanficdownloader/htmlcleanup.py                   | 8 ++++++--
 fanficdownloader/writers/writer_epub.py           | 2 +-
 6 files changed, 16 insertions(+), 8 deletions(-)

diff --git a/fanficdownloader/BeautifulSoup.py b/fanficdownloader/BeautifulSoup.py
index 4b17b85..5414aa8 100644
--- a/fanficdownloader/BeautifulSoup.py
+++ b/fanficdownloader/BeautifulSoup.py
@@ -114,6 +114,8 @@ class PageElement(object):
     """Contains the navigational information for some part of the page
     (either a tag or a piece of text)"""
 
+    bs3=True
+
     def setup(self, parent=None, previous=None):
         """Sets up the initial relations between this element and
         other elements."""
diff --git a/fanficdownloader/adapters/adapter_tthfanficorg.py b/fanficdownloader/adapters/adapter_tthfanficorg.py
index 8bb9130..bfd6bdd 100644
--- a/fanficdownloader/adapters/adapter_tthfanficorg.py
+++ b/fanficdownloader/adapters/adapter_tthfanficorg.py
@@ -286,8 +286,9 @@ class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter):
             self.story.addToList('category','Buffy: The Vampire Slayer')
             
         pseries = soup.find('p', {'style':'margin-top:0px'})
-        m = re.match('This story is No\. (?P<num>\d+) in the series &quot;(?P<series>.+)&quot;\.',
-                     pseries.text)
+        print("pseries:%s"%pseries.get_text())
+        m = re.match('This story is No\. (?P<num>\d+) in the series "(?P<series>.+)"\.',
+                     pseries.get_text())
         if m:
             self.setSeries(m.group('series'),m.group('num'))
             self.story.setMetadata('seriesUrl',"http://"+self.host+pseries.find('a')['href'])
diff --git a/fanficdownloader/adapters/base_adapter.py b/fanficdownloader/adapters/base_adapter.py
index af3ca1f..4f21ebc 100644
--- a/fanficdownloader/adapters/base_adapter.py
+++ b/fanficdownloader/adapters/base_adapter.py
@@ -468,7 +468,8 @@ class BaseSiteAdapter(Configurable):
         #print("\n\nsvalue:\n%s\n"%svalue)
         if self.getConfig('keep_summary_html'):
             if isinstance(svalue,basestring):
-                # bs4/html5lib add html, header and body tags.
+                # bs4/html5lib add html, header and body tags, which
+                # we don't want.
                 svalue = bs.BeautifulSoup(svalue,"html5lib").body
                 svalue.name='span'
             self.story.setMetadata('description',self.utf8FromSoup(url,svalue))
@@ -540,7 +541,7 @@ class BaseSiteAdapter(Configurable):
         except AttributeError, ae:
             print("Error parsing HTML, probably poor input HTML. %s"%ae)
         
-        retval = unicode(soup) #__str__().decode('utf-8')
+        retval = unicode(soup)
 
         if self.getConfig('nook_img_fix') and not self.getConfig('replace_br_with_p'):
             # if the <img> tag doesn't have a div or a p around it,
diff --git a/fanficdownloader/epubutils.py b/fanficdownloader/epubutils.py
index 6d2e6ff..2e1b041 100644
--- a/fanficdownloader/epubutils.py
+++ b/fanficdownloader/epubutils.py
@@ -14,7 +14,7 @@ import re, os, traceback
 from zipfile import ZipFile
 from xml.dom.minidom import parseString
 
-from . import BeautifulSoup as bs
+import bs4 as bs
 
 def get_dcsource(inputio):
     return get_update_data(inputio,getfilecount=False,getsoups=False)[0]
diff --git a/fanficdownloader/htmlcleanup.py b/fanficdownloader/htmlcleanup.py
index 09d2985..1544c81 100644
--- a/fanficdownloader/htmlcleanup.py
+++ b/fanficdownloader/htmlcleanup.py
@@ -56,10 +56,14 @@ def _replaceNotEntities(data):
     return p.sub(r'&\1', data)
 
 def stripHTML(soup):
-    return removeAllEntities(re.sub(r'<[^>]+>','',"%s" % soup)).strip()
+    if isinstance(soup,basestring) or soup.has_attr('bs3'):
+        return removeAllEntities(re.sub(r'<[^>]+>','',"%s" % soup)).strip()
+    else:
+        # bs4 already converts all the entities to UTF8 chars.
+        return soup.get_text(strip=True)
 
 def conditionalRemoveEntities(value):
-    if isinstance(value,str) or isinstance(value,unicode) :
+    if isinstance(value,basestring):
         return removeEntities(value).strip()
     else:
         return value
diff --git a/fanficdownloader/writers/writer_epub.py b/fanficdownloader/writers/writer_epub.py
index fb405a3..3b1b19d 100644
--- a/fanficdownloader/writers/writer_epub.py
+++ b/fanficdownloader/writers/writer_epub.py
@@ -662,7 +662,7 @@ div { margin: 0pt; padding: 0pt; }
                 # The replaces above added tons of extra newlines
                 # during *each* epub update.  The regexp version adds
                 # only one and removes any extra.
-                fullhtml = re.sub(r'(</p>|<br />)\n*',r'\1\n',fullhtml)
+                fullhtml = re.sub(r'(</p>|<br ?/>)\n*',r'\1\n',fullhtml)
 
                 outputepub.writestr("OEBPS/file%04d.xhtml"%(index+1),fullhtml.encode('utf-8'))
                 del fullhtml