Trying to fix outputting some utf-8 text. Added the utf-8 header to all the source code.

Ended up modifying the removeEntities function to do a weird decode/encode step on the text passed in. This seems to at least stop things from crashing..
2026-09-10 11:40:24 +08:00 · 2010-11-10 13:47:13 -06:00
parent 3f40df82dd
commit ed4ff4b6ab
12 changed files with 33 additions and 4 deletions
@@ -1,3 +1,5 @@
+# -*- coding: utf-8 -*-
+
 """Beautiful Soup
 Elixir and Tonic
 "The Screen-Scraper's Friend"
@@ -0,0 +1 @@
+# -*- coding: utf-8 -*-
@@ -1,3 +1,5 @@
+# -*- coding: utf-8 -*-
+
 class LoginRequiredException(Exception):
 	def __init__(self, url):
 		self.url = url
@@ -27,7 +27,7 @@ h7 { text-align: left; font-size: large; font-weight: bold; }

 MIMETYPE = '''application/epub+zip'''

-TITLE_HEADER = '''<html xmlns="http://www.w3.org/1999/xhtml" xmlns:xlink="http://www.w3.org/1999/xlink"><head><meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>
+TITLE_HEADER = '''<?xml version="1.0" encoding="utf-8"?><html xmlns="http://www.w3.org/1999/xhtml" xmlns:xlink="http://www.w3.org/1999/xlink"><head><meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>
 <title>%s - %s</title><link href="stylesheet.css" type="text/css" charset="UTF-8" rel="stylesheet"/></head><body>
 <p><h7 id="lnks"><b><a id="StoryLink" href="%s">%s</a></b> by <b><a id="AuthorLink" href="%s">%s</a></b></h7></p>
 <table class="full">
@@ -41,7 +41,7 @@ TITLE_FOOTER = '''</table>
 </body></html>
 '''

-CONTAINER = '''<?xml version="1.0"?>
+CONTAINER = '''<?xml version="1.0" encoding="utf-8"?>
 <container version="1.0" xmlns="urn:oasis:names:tc:opendocument:xmlns:container">
  <rootfiles>
    <rootfile full-path="OEBPS/content.opf" media-type="application/oebps-package+xml"/>
@@ -1,3 +1,5 @@
+# -*- coding: utf-8 -*-
+
 import os
 import re
 import sys
@@ -1,3 +1,5 @@
+# -*- coding: utf-8 -*-
+
 import os
 import re
 import sys
@@ -1,4 +1,6 @@
 #!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
 """html2text: Turn HTML into equivalent Markdown-structured text."""
 __version__ = "2.37"
 __author__ = "Aaron Swartz (me@aaronsw.com)"
@@ -1,3 +1,5 @@
+# -*- coding: utf-8 -*-
+
 XHTML_START = '''<?xml version="1.0" encoding="UTF-8"?>
 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
 <html xmlns="http://www.w3.org/1999/xhtml">
@@ -5,6 +5,7 @@ import re
 import sys
 import cgi
 import uuid
+import unicodedata
 import codecs
 import shutil
 import string
@@ -123,7 +124,7 @@ class EPubFanficWriter(FanficWriter):
 			if self.inmemory:
 				self.files[fileName] = StringIO.StringIO()
 			else:
-				self.files[fileName] = open(self.directory + '/' + fileName, 'w')
+				self.files[fileName] = open(self.directory + '/' + fileName, encoding='utf-8', mode='w')
 			
 			self._writeFile(fileName, data)
 		
@@ -182,7 +183,7 @@ class EPubFanficWriter(FanficWriter):
 		# BeautifulStoneSoup doesn't have any selfClosingTags by default.  
 		# hr & br needs to be if they're going to work.
 		# Some stories do use multiple br tags as their section breaks...
-		self.soup = bs.BeautifulStoneSoup(text.decode('utf-8'), selfClosingTags=('br','hr'))
+		self.soup = bs.BeautifulStoneSoup(text, selfClosingTags=('br','hr'))

 		allTags = self.soup.findAll(recursive=True)
 		for t in allTags:
@@ -349,6 +350,15 @@ def replaceNumberEntities(data):

 def removeEntities(text):
 	# replace numeric versions of [&<>] with named versions.
+	
+	try:
+		t = text.decode('utf-8')
+	except UnicodeEncodeError, e:
+		try:
+			t = text.encode ('ascii', 'xmlcharrefreplace') 
+		except UnicodeEncodeError, e:
+			t = text
+	text = t 
 	text = re.sub(r'&#0*38;','&amp;',text)
 	text = re.sub(r'&#0*60;','&lt;',text)
 	text = re.sub(r'&#0*62;','&gt;',text)
@@ -1,3 +1,5 @@
+# -*- coding: utf-8 -*-
+
 import os
 import re
 import sys
@@ -1,2 +1,4 @@
+# -*- coding: utf-8 -*-
+
 # This is really for the web version.  downalod.py will ask.
 password='somepass'
@@ -1,3 +1,5 @@
+# -*- coding: utf-8 -*-
+
 import os
 import zipfile
 import logging