Another work around for SGMLParser's entity handling. &#8212again v &#8212a;gain

calibre-plugin/__init__.py

@@ -27,7 +27,7 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
     description         = 'UI plugin to download FanFiction stories from various sites.'
     supported_platforms = ['windows', 'osx', 'linux']
     author              = 'Jim Miller'
-    version             = (1, 5, 10)
+    version             = (1, 5, 11)
     minimum_calibre_version = (0, 8, 30)
 
     #: This field defines the GUI plugin class that contains all the code

fanficdownloader/adapters/base_adapter.py

@@ -331,10 +331,6 @@ class BaseSiteAdapter(Configurable):
         # This is primarily for epub updates.
         return re.sub(r"</?body>\r?\n?","",retval)
 
-fullmon = {"January":"01", "February":"02", "March":"03", "April":"04", "May":"05",
-           "June":"06","July":"07", "August":"08", "September":"09", "October":"10",
-           "November":"11", "December":"12" }
-
 def cachedfetch(realfetch,cache,url):
     if url in cache:
         print("cache hit")
@@ -342,6 +338,9 @@ def cachedfetch(realfetch,cache,url):
     else:
         return realfetch(url)
     
+fullmon = {"January":"01", "February":"02", "March":"03", "April":"04", "May":"05",
+           "June":"06","July":"07", "August":"08", "September":"09", "October":"10",
+           "November":"11", "December":"12" }
 
 def makeDate(string,format):
     # Surprise!  Abstracting this turned out to be more useful than

fanficdownloader/htmlcleanup.py

@@ -23,11 +23,16 @@ def _unirepl(match):
         radix=16
     else:
         radix=10
-    value = int(match.group(2), radix )
-    return unichr(value)
+    value = int(match.group(2), radix)
+    return "%s%s"%(unichr(value),match.group(3))
 
 def _replaceNumberEntities(data):
-    p = re.compile(r'&#(x?)([0-9a-fA-F]+);')
+    # The same brokenish entity parsing in SGMLParser that inserts ';'
+    # after non-entities will also insert ';' incorrectly after number
+    # entities, including part of the next word if it's a-z.
+    # "Don't&#8212ever&#8212do&#8212that&#8212again," becomes
+    # "Don't&#8212e;ver&#8212d;o—that&#8212a;gain,"
+    p = re.compile(r'&#(x?)([0-9a-fA-F]{,4})([0-9a-fA-F]*);')
     return p.sub(_unirepl, data)
 
 def _replaceNotEntities(data):
@@ -50,9 +55,6 @@ def removeAllEntities(text):
     return removeEntities(text).replace('&lt;', '<').replace('&gt;', '>').replace('&amp;', '&')
 
 def removeEntities(text):
-
-    # replace numeric versions of [&<>] with named versions,
-    # then replace named versions with actual characters,
     
     if text is None:
         return ""
@@ -67,6 +69,8 @@ def removeEntities(text):
         except UnicodeEncodeError, e:
             t = text
     text = t 
+    # replace numeric versions of [&<>] with named versions,
+    # then replace named versions with actual characters,
     text = re.sub(r'&#0*38;','&amp;',text)
     text = re.sub(r'&#0*60;','&lt;',text)
     text = re.sub(r'&#0*62;','&gt;',text)

fanficdownloader/story.py

@@ -216,7 +216,7 @@ class Story:
         for (p,v) in self.replacements:
             if (isinstance(value,str) or isinstance(value,unicode)) and re.match(p,value):
                 value = re.sub(p,v,value)                
-        return value;
+        return value
         
     def getMetadata(self, key, removeallentities=False):
         value = None