Add '\n' after each </p> and <br /> tag. Extremely long(200k+) single line chapters (like ffnet gives) causes problems for nook.

Also removed a little dead code from ffnet.py.
This commit is contained in:
retiefjimm
2010-10-11 14:05:57 -05:00
parent dafa33c64e
commit 7f8d7a7dbb
2 changed files with 15 additions and 13 deletions
+2 -7
View File
@@ -117,17 +117,12 @@ class FFNet(FanfictionSiteAdapter):
def getText(self, url):
data = self._fetchUrl(url)
lines = data.split('\n')
textbuf = ''
emit = False
olddata = data
try:
data = data.decode('utf8')
except:
data = olddata
try:
soup = bs.BeautifulStoneSoup(data)
except:
@@ -195,4 +190,4 @@ class FFA_UnitTests(unittest.TestCase):
text = f.getText(url)
self.assertTrue(text.find('Kale Resgerald at your service" He answered, "So, can we go now? Or do you want to') != -1)
if __name__ == '__main__':
unittest.main()
unittest.main()
+13 -6
View File
@@ -136,7 +136,7 @@ class EPubFanficWriter(FanficWriter):
self.storyTitle = removeEntities(name)
self.name = makeAcceptableFilename(name)
self.directory = self.basePath + '/' + self.name
self.authorName = removeEntities(author)
self.authorName = removeEntities(author+" aa")
self.inmemory = inmemory
@@ -195,14 +195,21 @@ class EPubFanficWriter(FanficWriter):
if t.name in ('center'):
t['class']=t.name
t.name='div'
allPs = self.soup.findAll(recursive=True)
for p in allPs:
if p.string != None and len(p.string.strip()) == 0 :
p.extract()
# removes paired, but empty tags.
if t.string != None and len(t.string.strip()) == 0 :
t.extract()
text = self.soup.__str__('utf8')
# ffnet(& maybe others) gives the whole chapter text
# as one line. This causes problems for nook(at
# least) when the chapter size starts getting big
# (200k+) Using Soup's prettify() messes up italics
# and such. Done after soup extract so <p> and <br>
# tags are normalized. Doing it here seems less evil
# than hacking BeautifulSoup, but it's debatable.
text = text.replace('</p>','</p>\n').replace('<br />','<br />\n')
self._writeFile(fn, XHTML_START % (title, title))
self._writeFile(fn, text)
self._writeFile(fn, XHTML_END)