More changes for bs4.

fanficdownloader/BeautifulSoup.py

@@ -114,6 +114,8 @@ class PageElement(object):
     """Contains the navigational information for some part of the page
     (either a tag or a piece of text)"""
 
+    bs3=True
+
     def setup(self, parent=None, previous=None):
         """Sets up the initial relations between this element and
         other elements."""

fanficdownloader/adapters/adapter_tthfanficorg.py

@@ -286,8 +286,9 @@ class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter):
             self.story.addToList('category','Buffy: The Vampire Slayer')
             
         pseries = soup.find('p', {'style':'margin-top:0px'})
-        m = re.match('This story is No\. (?P<num>\d+) in the series &quot;(?P<series>.+)&quot;\.',
-                     pseries.text)
+        print("pseries:%s"%pseries.get_text())
+        m = re.match('This story is No\. (?P<num>\d+) in the series "(?P<series>.+)"\.',
+                     pseries.get_text())
         if m:
             self.setSeries(m.group('series'),m.group('num'))
             self.story.setMetadata('seriesUrl',"http://"+self.host+pseries.find('a')['href'])

fanficdownloader/adapters/base_adapter.py

@@ -468,7 +468,8 @@ class BaseSiteAdapter(Configurable):
         #print("\n\nsvalue:\n%s\n"%svalue)
         if self.getConfig('keep_summary_html'):
             if isinstance(svalue,basestring):
-                # bs4/html5lib add html, header and body tags.
+                # bs4/html5lib add html, header and body tags, which
+                # we don't want.
                 svalue = bs.BeautifulSoup(svalue,"html5lib").body
                 svalue.name='span'
             self.story.setMetadata('description',self.utf8FromSoup(url,svalue))
@@ -540,7 +541,7 @@ class BaseSiteAdapter(Configurable):
         except AttributeError, ae:
             print("Error parsing HTML, probably poor input HTML. %s"%ae)
         
-        retval = unicode(soup) #__str__().decode('utf-8')
+        retval = unicode(soup)
 
         if self.getConfig('nook_img_fix') and not self.getConfig('replace_br_with_p'):
             # if the <img> tag doesn't have a div or a p around it,

fanficdownloader/epubutils.py

@@ -14,7 +14,7 @@ import re, os, traceback
 from zipfile import ZipFile
 from xml.dom.minidom import parseString
 
-from . import BeautifulSoup as bs
+import bs4 as bs
 
 def get_dcsource(inputio):
     return get_update_data(inputio,getfilecount=False,getsoups=False)[0]

fanficdownloader/htmlcleanup.py

@@ -56,10 +56,14 @@ def _replaceNotEntities(data):
     return p.sub(r'&\1', data)
 
 def stripHTML(soup):
-    return removeAllEntities(re.sub(r'<[^>]+>','',"%s" % soup)).strip()
+    if isinstance(soup,basestring) or soup.has_attr('bs3'):
+        return removeAllEntities(re.sub(r'<[^>]+>','',"%s" % soup)).strip()
+    else:
+        # bs4 already converts all the entities to UTF8 chars.
+        return soup.get_text(strip=True)
 
 def conditionalRemoveEntities(value):
-    if isinstance(value,str) or isinstance(value,unicode) :
+    if isinstance(value,basestring):
         return removeEntities(value).strip()
     else:
         return value

fanficdownloader/writers/writer_epub.py

@@ -662,7 +662,7 @@ div { margin: 0pt; padding: 0pt; }
                 # The replaces above added tons of extra newlines
                 # during *each* epub update.  The regexp version adds
                 # only one and removes any extra.
-                fullhtml = re.sub(r'(</p>|<br />)\n*',r'\1\n',fullhtml)
+                fullhtml = re.sub(r'(</p>|<br ?/>)\n*',r'\1\n',fullhtml)
 
                 outputepub.writestr("OEBPS/file%04d.xhtml"%(index+1),fullhtml.encode('utf-8'))
                 del fullhtml