From 2a86c3c9abbcb0286eafd83920b84690add25298 Mon Sep 17 00:00:00 2001
From: Jim Miller <retiefjimm@gmail.com>
Date: Sun, 15 Jan 2012 23:14:54 -0600
Subject: [PATCH] Adding 'characters' parsing to other existing sites.  Similar
 'fix' as twiwrite for plugin for some stories across several sites.  Suspect
 different SGMLParser in calibre stumbling on something common to many
 eFiction sites.

---
 calibre-plugin/ffdl_plugin.py                 |  5 +++++
 .../adapters/adapter_adastrafanficcom.py      | 19 +++++++++++++++-
 .../adapters/adapter_castlefansorg.py         |  6 +++++
 .../adapters/adapter_ficwadcom.py             |  6 +++++
 .../adapter_harrypotterfanfictioncom.py       |  5 +++++
 .../adapters/adapter_potionsandsnitchesnet.py | 10 +++++++++
 fanficdownloader/adapters/adapter_siyecouk.py |  5 ++++-
 .../adapters/adapter_tenhawkpresentscom.py    |  6 +++++
 .../adapter_thewriterscoffeeshopcom.py        | 19 +++++++++++++++-
 .../adapters/adapter_twilightednet.py         | 21 +++++++++++++++++-
 .../adapters/adapter_twiwritenet.py           | 22 ++++++++++++++-----
 .../adapters/adapter_whoficcom.py             |  9 ++++++++
 12 files changed, 124 insertions(+), 9 deletions(-)

diff --git a/calibre-plugin/ffdl_plugin.py b/calibre-plugin/ffdl_plugin.py
index 0a9a069..d677ea1 100644
--- a/calibre-plugin/ffdl_plugin.py
+++ b/calibre-plugin/ffdl_plugin.py
@@ -109,6 +109,11 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
 
         self.menus_lock = threading.RLock()
 
+    def initialization_complete(self):
+        # otherwise configured hot keys won't work until the menu's
+        # been displayed once.
+        self.rebuild_menus()
+
     def about_to_show_menu(self):
         self.rebuild_menus()
 
diff --git a/fanficdownloader/adapters/adapter_adastrafanficcom.py b/fanficdownloader/adapters/adapter_adastrafanficcom.py
index bc908a2..14fe48e 100644
--- a/fanficdownloader/adapters/adapter_adastrafanficcom.py
+++ b/fanficdownloader/adapters/adapter_adastrafanficcom.py
@@ -79,6 +79,11 @@ class AdAstraFanficComSiteAdapter(BaseSiteAdapter):
         if "Content is only suitable for mature adults. May contain explicit language and adult themes. Equivalent of NC-17." in data:
             raise exceptions.AdultCheckRequired(self.url)
             
+        # problems with some stories, but only in calibre.  I suspect
+        # issues with different SGML parsers in python.  This is a
+        # nasty hack, but it works.
+        data = data[data.index("<body"):]
+        
         # use BeautifulSoup HTML parser to make everything easier to find.
         soup = bs.BeautifulSoup(data)
 
@@ -142,6 +147,12 @@ class AdAstraFanficComSiteAdapter(BaseSiteAdapter):
                 for cat in catstext:
                     self.story.addToList('category',cat.string)
 
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                charstext = [char.string for char in chars]
+                for char in charstext:
+                    self.story.addToList('characters',char.string)
+
             if 'Genre' in label:
                 genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
                 genrestext = [genre.string for genre in genres]
@@ -175,7 +186,13 @@ class AdAstraFanficComSiteAdapter(BaseSiteAdapter):
 
         logging.debug('Getting chapter text from: %s' % url)
 
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+        data = self._fetchUrl(url)
+        # problems with some stories, but only in calibre.  I suspect
+        # issues with different SGML parsers in python.  This is a
+        # nasty hack, but it works.
+        data = data[data.index("<body"):]
+        
+        soup = bs.BeautifulStoneSoup(data,
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
         
         span = soup.find('div', {'id' : 'story'})
diff --git a/fanficdownloader/adapters/adapter_castlefansorg.py b/fanficdownloader/adapters/adapter_castlefansorg.py
index 907d78e..9184b9f 100644
--- a/fanficdownloader/adapters/adapter_castlefansorg.py
+++ b/fanficdownloader/adapters/adapter_castlefansorg.py
@@ -232,6 +232,12 @@ class CastleFansOrgAdapter(BaseSiteAdapter): # XXX
                 for cat in catstext:
                     self.story.addToList('category',cat.string)
 
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                charstext = [char.string for char in chars]
+                for char in charstext:
+                    self.story.addToList('characters',char.string)
+
             ## Not all sites use Genre, but there's no harm to
             ## leaving it in.  Check to make sure the type_id number
             ## is correct, though--it's site specific.
diff --git a/fanficdownloader/adapters/adapter_ficwadcom.py b/fanficdownloader/adapters/adapter_ficwadcom.py
index 8d63aba..affd747 100644
--- a/fanficdownloader/adapters/adapter_ficwadcom.py
+++ b/fanficdownloader/adapters/adapter_ficwadcom.py
@@ -152,6 +152,12 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
             for g in m.group(1).split(','):
                 self.story.addToList('genre',g)
         
+        m = re.match(r".*?Characters: (.*?) -.*?",metastr)
+        if m:
+            for g in m.group(1).split(','):
+                if g:
+                    self.story.addToList('characters',g)
+        
         m = re.match(r".*?Published: ([0-9/]+?) -.*?",metastr)
         if m:
             self.story.setMetadata('datePublished',makeDate(m.group(1), "%Y/%m/%d"))
diff --git a/fanficdownloader/adapters/adapter_harrypotterfanfictioncom.py b/fanficdownloader/adapters/adapter_harrypotterfanfictioncom.py
index 2d44f9b..5682b9e 100644
--- a/fanficdownloader/adapters/adapter_harrypotterfanfictioncom.py
+++ b/fanficdownloader/adapters/adapter_harrypotterfanfictioncom.py
@@ -159,6 +159,11 @@ class HarryPotterFanFictionComSiteAdapter(BaseSiteAdapter):
                     for g in m.group(1).split(','):
                         self.story.addToList('genre',g)
         
+                m = re.match(r".*?Characters: (.+?) Genre.*?",metastr)
+                if m:
+                    for g in m.group(1).split(','):
+                        self.story.addToList('characters',g)
+        
                 m = re.match(r".*?Warnings: (.+).*?",metastr)
                 if m:
                     for w in m.group(1).split(','):
diff --git a/fanficdownloader/adapters/adapter_potionsandsnitchesnet.py b/fanficdownloader/adapters/adapter_potionsandsnitchesnet.py
index de9a722..4d79413 100644
--- a/fanficdownloader/adapters/adapter_potionsandsnitchesnet.py
+++ b/fanficdownloader/adapters/adapter_potionsandsnitchesnet.py
@@ -145,6 +145,16 @@ class PotionsAndSnitchesNetSiteAdapter(BaseSiteAdapter):
                 for cat in catstext:
                     self.story.addToList('category',cat.string)
 
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                charstext = [char.string for char in chars]
+                for char in charstext:
+                    if char == "!Snape and Harry (required)":
+                        self.story.addToList('characters',"Snape")
+                        self.story.addToList('characters',"Harry")
+                    else:
+                        self.story.addToList('characters',char.string)
+
             if 'Genre' in label:
                 genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class'))
                 genrestext = [genre.string for genre in genres]
diff --git a/fanficdownloader/adapters/adapter_siyecouk.py b/fanficdownloader/adapters/adapter_siyecouk.py
index a7066a0..23b3716 100644
--- a/fanficdownloader/adapters/adapter_siyecouk.py
+++ b/fanficdownloader/adapters/adapter_siyecouk.py
@@ -203,7 +203,10 @@ class SiyeCoUkAdapter(BaseSiteAdapter): # XXX
             if part.startswith("Characters:"):
                 part = part[part.find(':')+1:]
                 for item in part.split(','):
-                    if item.strip() != "None":
+                    if item.strip() == "Harry/Ginny":
+                        self.story.addToList('characters',"Harry")
+                        self.story.addToList('characters',"Ginny")
+                    elif item.strip() not in ("None","All"):
                         self.story.addToList('characters',item)
 
             if part.startswith("Genres:"):
diff --git a/fanficdownloader/adapters/adapter_tenhawkpresentscom.py b/fanficdownloader/adapters/adapter_tenhawkpresentscom.py
index a4922d2..4d4df83 100644
--- a/fanficdownloader/adapters/adapter_tenhawkpresentscom.py
+++ b/fanficdownloader/adapters/adapter_tenhawkpresentscom.py
@@ -178,6 +178,12 @@ class TenhawkPresentsComSiteAdapter(BaseSiteAdapter):
                 for cat in catstext:
                     self.story.addToList('category',cat.string)
 
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                charstext = [char.string for char in chars]
+                for char in charstext:
+                    self.story.addToList('characters',char.string)
+
             if 'Genre' in label:
                 genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class'))
                 genrestext = [genre.string for genre in genres]
diff --git a/fanficdownloader/adapters/adapter_thewriterscoffeeshopcom.py b/fanficdownloader/adapters/adapter_thewriterscoffeeshopcom.py
index fecc2bd..f15db2c 100644
--- a/fanficdownloader/adapters/adapter_thewriterscoffeeshopcom.py
+++ b/fanficdownloader/adapters/adapter_thewriterscoffeeshopcom.py
@@ -123,6 +123,11 @@ class TheWritersCoffeeShopComSiteAdapter(BaseSiteAdapter):
         if "Access denied. This story has not been validated by the adminstrators of this site." in data:
             raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
             
+        # problems with some stories, but only in calibre.  I suspect
+        # issues with different SGML parsers in python.  This is a
+        # nasty hack, but it works.
+        data = data[data.index("<body"):] 
+        
         # use BeautifulSoup HTML parser to make everything easier to find.
         soup = bs.BeautifulSoup(data)
 
@@ -175,6 +180,12 @@ class TheWritersCoffeeShopComSiteAdapter(BaseSiteAdapter):
                 for cat in catstext:
                     self.story.addToList('category',cat.string)
 
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                charstext = [char.string for char in chars]
+                for char in charstext:
+                    self.story.addToList('characters',char.string)
+
             if 'Genre' in label:
                 genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class'))
                 genrestext = [genre.string for genre in genres]
@@ -201,7 +212,13 @@ class TheWritersCoffeeShopComSiteAdapter(BaseSiteAdapter):
 
         logging.debug('Getting chapter text from: %s' % url)
 
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+        data = self._fetchUrl(url)
+        # problems with some stories, but only in calibre.  I suspect
+        # issues with different SGML parsers in python.  This is a
+        # nasty hack, but it works.
+        data = data[data.index("<body"):]
+        
+        soup = bs.BeautifulStoneSoup(data,
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
         
         span = soup.find('div', {'id' : 'story'})
diff --git a/fanficdownloader/adapters/adapter_twilightednet.py b/fanficdownloader/adapters/adapter_twilightednet.py
index 675f305..9895be2 100644
--- a/fanficdownloader/adapters/adapter_twilightednet.py
+++ b/fanficdownloader/adapters/adapter_twilightednet.py
@@ -118,6 +118,12 @@ class TwilightedNetSiteAdapter(BaseSiteAdapter):
         if "Access denied. This story has not been validated by the adminstrators of this site." in data:
             raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
             
+        # problems with some stories, but only in calibre.  I suspect
+        # issues with different SGML parsers in python.  This is a
+        # nasty hack, but it works.
+        # twilighted isn't writing <body> ??? wtf?
+        data = "<html><body>"+data[data.index("</head>"):] 
+        
         # use BeautifulSoup HTML parser to make everything easier to find.
         soup = bs.BeautifulSoup(data)
 
@@ -170,6 +176,12 @@ class TwilightedNetSiteAdapter(BaseSiteAdapter):
                 for cat in catstext:
                     self.story.addToList('category',cat.string)
 
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                charstext = [char.string for char in chars]
+                for char in charstext:
+                    self.story.addToList('characters',char.string)
+
             ## twilighted.net doesn't use genre.
             # if 'Genre' in label:
             #     genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class'))
@@ -197,7 +209,14 @@ class TwilightedNetSiteAdapter(BaseSiteAdapter):
 
         logging.debug('Getting chapter text from: %s' % url)
 
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+        data = self._fetchUrl(url)
+        # problems with some stories, but only in calibre.  I suspect
+        # issues with different SGML parsers in python.  This is a
+        # nasty hack, but it works.
+        # twilighted isn't writing <body> ??? wtf?
+        data = "<html><body>"+data[data.index("</head>"):] 
+        
+        soup = bs.BeautifulStoneSoup(data,
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
         
         span = soup.find('div', {'id' : 'story'})
diff --git a/fanficdownloader/adapters/adapter_twiwritenet.py b/fanficdownloader/adapters/adapter_twiwritenet.py
index 9473d99..871ad9a 100644
--- a/fanficdownloader/adapters/adapter_twiwritenet.py
+++ b/fanficdownloader/adapters/adapter_twiwritenet.py
@@ -117,10 +117,13 @@ class TwiwriteNetSiteAdapter(BaseSiteAdapter):
 
         if "Access denied. This story has not been validated by the adminstrators of this site." in data:
             raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
+
+        # problems with some stories, but only in calibre.  I suspect
+        # issues with different SGML parsers in python.  This is a
+        # nasty hack, but it works.
+        data = data[data.index("<body"):] 
+        
         # use BeautifulSoup HTML parser to make everything easier to find.
-        data = data[data.index("<body"):] # desperate--strip before <body
-        # in calibre plugin only, soup wasn't parsing the html properly.
         soup = bs.BeautifulSoup(data)
 
         ## Title
@@ -180,6 +183,12 @@ class TwiwriteNetSiteAdapter(BaseSiteAdapter):
                 for cat in catstext:
                     self.story.addToList('category',cat.string)
 
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                charstext = [char.string for char in chars]
+                for char in charstext:
+                    self.story.addToList('characters',char.string)
+
             if 'Genre' in label:
                 genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=3'))
                 genrestext = [genre.string for genre in genres]
@@ -214,8 +223,11 @@ class TwiwriteNetSiteAdapter(BaseSiteAdapter):
         logging.debug('Getting chapter text from: %s' % url)
 
         data = self._fetchUrl(url)
-        data = data[data.index("<body"):] # desperate--strip before <body
-        # in calibre plugin only, soup wasn't parsing the html properly.
+        # problems with some stories, but only in calibre.  I suspect
+        # issues with different SGML parsers in python.  This is a
+        # nasty hack, but it works.
+        data = data[data.index("<body"):]
+        
         soup = bs.BeautifulStoneSoup(data,
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
         
diff --git a/fanficdownloader/adapters/adapter_whoficcom.py b/fanficdownloader/adapters/adapter_whoficcom.py
index 40baa6a..6262c22 100644
--- a/fanficdownloader/adapters/adapter_whoficcom.py
+++ b/fanficdownloader/adapters/adapter_whoficcom.py
@@ -158,6 +158,15 @@ class WhoficComSiteAdapter(BaseSiteAdapter):
         for g in genre.split(r', '):
             self.story.addToList('genre',g)
 
+        # line 3 is characters.
+        chars = metadatachunks[3]
+        charsearch="<i>Characters:</i>"
+        if charsearch in chars:
+            chars = chars[metadatachunks[3].index(charsearch)+len(charsearch):]
+            for c in chars.split(','):
+                if c.strip() != u'None':
+                    self.story.addToList('characters',c)
+            
         # the next line is stuff with ' - ' separators *and* names--with tags.
         moremeta = metadatachunks[5]
         moremeta = re.sub(r'<[^>]+>','',moremeta) # strip tags.