Bump versions.

app.yaml

@@ -1,6 +1,6 @@
 # ffd-retief-hrd fanfictiondownloader
 application: fanfictiondownloader
-version: 4-4-65
+version: 4-4-67
 runtime: python27
 api_version: 1
 threadsafe: true

calibre-plugin/__init__.py

@@ -26,7 +26,7 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
     description         = 'UI plugin to download FanFiction stories from various sites.'
     supported_platforms = ['windows', 'osx', 'linux']
     author              = 'Jim Miller'
-    version             = (1, 7, 31)
+    version             = (1, 7, 34)
     minimum_calibre_version = (0, 8, 57)
 
     #: This field defines the GUI plugin class that contains all the code

calibre-plugin/config.py

@@ -322,10 +322,10 @@ class BasicTab(QWidget):
         self.updateepubcover.setChecked(prefs['updateepubcover'])
         self.l.addWidget(self.updateepubcover)
 
+        self.smarten_punctuation = QCheckBox('Smarten Punctuation (EPUB only)',self)
+        self.smarten_punctuation.setToolTip("Run Smarten Punctuation from Calibre's Polish Book feature on each EPUB download and update.")
+        self.smarten_punctuation.setChecked(prefs['smarten_punctuation'])
         if calibre_version >= (0, 9, 39):
-            self.smarten_punctuation = QCheckBox('Smarten Punctuation (EPUB only)',self)
-            self.smarten_punctuation.setToolTip("Run Smarten Punctuation from Calibre's Polish Book feature on each EPUB download and update.")
-            self.smarten_punctuation.setChecked(prefs['smarten_punctuation'])
             self.l.addWidget(self.smarten_punctuation)
 
         cali_gb = groupbox = QGroupBox("Updating Calibre Options")

defaults.ini

@@ -1355,6 +1355,11 @@ titlepage_entries: series,seriesUrl,category,genre,language,status,datePublished
 ## Site dedicated to these categories/characters/ships
 extracategories:Lord of the Rings
 
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
 [www.thepetulantpoetess.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -1476,7 +1481,8 @@ extracategories:Stargate: Atlantis
 [teststory:defaults]
 valid_entries:title,author_list,authorId_list,authorUrl_list,storyUrl,
  datePublished,dateUpdated,numWords,status,language,series,seriesUrl,
- rating,category_list,genre_list,warnings_list,characters_list
+ rating,category_list,genre_list,warnings_list,characters_list,ships_list,
+ description,site,extratags
 
 # {{storyId}} is a special case--it's the only one that works.
 title:Test Story Title {{storyId}}

fanficdownloader/adapters/adapter_darksolaceorg.py

@@ -184,31 +184,31 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
         # first a tag in pagetitle is title
         self.story.setMetadata('title',stripHTML(div.find('a')))
 
-        # Find the chapters:
-        # chapters=soup.find('select', {'name' : 'chapter'})
-        # if chapters != None:
-        #     for chapter in chapters.findAll('option'):
-        #         self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/elysian/viewstory.php?sid='+self.story.getMetadata('storyId')+'&chapter='+chapter['value']))
-        # else:
-        #     self.chapterUrls.append((self.story.getMetadata('title'),url+"&chapter=1"))
-
         for chapa in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+
                                                        self.story.getMetadata('storyId')+'&chapter=\d+')):
             self.chapterUrls.append((stripHTML(chapa),'http://'+self.host+'/elysian/'+chapa['href']))
         
         self.story.setMetadata('numChapters',len(self.chapterUrls))
-        
+
         asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
+        storylink = asoup.find('a', href=re.compile(r'viewstory.php\?sid='+
+                                                    self.story.getMetadata('storyId')+'($|[^\d])'))
+        # author's story list is paginated if there's a pagelinks div.
+        # Only need to look in it if the story wasn't on the first page.
+        pagelinks = asoup.find('div',{'id':'pagelinks'})
+        if pagelinks and storylink==None:
+            authpageslist = pagelinks.findAll('a',href=re.compile(r'action=storiesby'))
+            for page in authpageslist[1:]: # skip first, already checked above.
+                asoup = bs.BeautifulSoup(self._fetchUrl('http://'+self.host+'/elysian/'+page['href']))
+                storylink = asoup.find('a', href=re.compile(r'viewstory.php\?sid='+
+                                                            self.story.getMetadata('storyId')+'($|[^\d])'))
+                if storylink:
+                    break
+
+        if not storylink:
+            raise exceptions.FailedToDownload("Unable to find story metadata on author's page(s)")
         
-        # for metalist in asoup.findAll('div', {'class' : re.compile('listbox\s+')}):
-        #     a = metalist.find('a', href=re.compile(r'viewstory.php\?sid='))
-        #     if a != None:
-        #         if 'viewstory.php?sid='+self.story.getMetadata('storyId') in a['href']:
-        #             break
-
-        metalist = asoup.find('a', href=re.compile(r'viewstory.php\?sid='+
-                                                   self.story.getMetadata('storyId')+'($|[^\d])')).parent.parent
-
+        metalist = storylink.parent.parent
         # eFiction sites don't help us out a lot with their meta data
         # formating, so it's a little ugly.
 
@@ -229,7 +229,7 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
             if 'Summary' in label:
                 ## Everything until the next span class='label'
                 svalue = ""
-                while not (defaultGetattr(value,'class') == 'label' or "Chapters: " in stripHTML(value)):
+                while value and not (defaultGetattr(value,'class') == 'label' or "Chapters: " in stripHTML(value)):
                     svalue += str(value)
                     value = value.nextSibling
                 self.setDescription(url,svalue)
@@ -284,20 +284,40 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
             # use BeautifulSoup HTML parser to make everything easier to find.
             seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
             # can't use ^viewstory...$ in case of higher rated stories with javascript href.
-            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
-            i=1
-            for a in storyas:
-                # skip 'report this' and 'TOC' links
-                if 'contact.php' not in a['href'] and 'index' not in a['href']:
-                    if ('viewstory.php?sid='+self.story.getMetadata('storyId')) in a['href']:
-                        self.setSeries(series_name, i)
-                        self.story.setMetadata('seriesUrl',series_url)
+            storylink = seriessoup.find('a', href=re.compile(r'viewstory.php\?sid='+
+                                                             self.story.getMetadata('storyId')+'($|[^\d])'))
+            if storylink and storylink.parent and storylink.parent['class'] != 'title': # in case of links inside story summaries.
+                storylink = None
+
+            offset = 0
+            # series story list is paginated if there's a pagelinks div.
+            # Only need to look in it if the story wasn't on the first page.
+            pagelinks = seriessoup.find('div',{'id':'pagelinks'})
+            if pagelinks and storylink==None:
+                authpageslist = pagelinks.findAll('a',href=re.compile(r'offset='))
+                for page in authpageslist[1:]: # skip first, already checked above.
+                    seriessoup = bs.BeautifulSoup(self._fetchUrl('http://'+self.host+'/elysian/'+page['href']))
+                    storylink = seriessoup.find('a', href=re.compile(r'viewstory.php\?sid='+
+                                                                     self.story.getMetadata('storyId')+'($|[^\d])'))
+                    if storylink and storylink.parent and storylink.parent['class'] != 'title': # in case of links inside story summaries.
+                        storylink = None
+                    if storylink:
+                        offset = int(page['href'].split('=')[-1]) # offset is last.
                         break
-                    i+=1
+
+            # for reasons I don't understand, searching for story
+            # links by regex wasn't working reliably.  It was missing
+            # the javascript links sometimes.  This is cleaner anyway.
+            for i, div in enumerate(seriessoup.findAll('div', {'class':'title'})):
+                a = div.find('a') # first a is story link.
+                # skip 'report this' and 'TOC' links
+                if a == storylink:
+                    self.setSeries(series_name, 1+i+offset)
+                    self.story.setMetadata('seriesUrl',series_url)
+                    break
             
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
+        except Exception, e:
+            print("Series parsing failed: %s"%e)
             
     # grab the text for an individual chapter.
     def getChapterText(self, url):

fanficdownloader/adapters/adapter_fanfiktionde.py

@@ -69,10 +69,10 @@ class FanFiktionDeAdapter(BaseSiteAdapter):
 
     @classmethod
     def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/s/46ccbef30000616306614050"
+        return "http://"+self.getSiteDomain()+"/s/46ccbef30000616306614050 http://"+self.getSiteDomain()+"/s/46ccbef30000616306614050/1 http://"+self.getSiteDomain()+"/s/46ccbef30000616306614050/1/story-name"
 
     def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/s/")+r"\w+(/\d+)?$"
+        return re.escape("http://"+self.getSiteDomain()+"/s/")+r"\w+(/\d+)?"
         
         ## Login seems to be reasonably standard across eFiction sites.
     def needToLoginCheck(self, data):
@@ -138,15 +138,15 @@ class FanFiktionDeAdapter(BaseSiteAdapter):
         # Now go hunting for all the meta data and the chapter list.
         
         ## Title
-        a = soup.find('a', href=re.compile(r'/s/'+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        a = soup.find('a', href=re.compile(r'/s/'+self.story.getMetadata('storyId')+"/"))
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         head = soup.find('div', {'class' : 'story-metadata-left-top'})
         a = head.find('a')
         self.story.setMetadata('authorId',a['href'].split('/')[2])
         self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
+        self.story.setMetadata('author',stripHTML(a))
 
         # Find the chapters:
         for chapter in soup.find('select').findAll('option'):
@@ -173,14 +173,14 @@ class FanFiktionDeAdapter(BaseSiteAdapter):
         tr=asoup.findAll('tr')
         for i in range(1,len(tr)):
             a = tr[i].find('a')
-            if a['href'] == '/s/'+self.story.getMetadata('storyId'):
+            if '/s/'+self.story.getMetadata('storyId')+'/1/' in a['href']:
                 break
         self.setDescription(url,a['onmouseover'].split("', '")[1])
         
         td = tr[i].findAll('td')
         self.story.addToList('category',stripHTML(td[1]))
-        self.story.setMetadata('rating', td[4].string)
-        self.story.setMetadata('numWords', td[5].string)
+        self.story.setMetadata('rating', stripHTML(td[4]))
+        self.story.setMetadata('numWords', stripHTML(td[5]))
  
             
     # grab the text for an individual chapter.

fanficdownloader/adapters/adapter_hpfanficarchivecom.py

@@ -37,6 +37,11 @@ class HPFanficArchiveComAdapter(BaseSiteAdapter):
     def __init__(self, config, url):
         BaseSiteAdapter.__init__(self, config, url)
 
+        # hpfanficarchive.com blocks the default user-agent.  However,
+        # when asked, they said it was just general anti-spam, not
+        # targeted at us.  That lets me do this in good conscience:
+        self.opener.addheaders = [('User-agent', 'FFDL/1.7')]
+        
         self.decode = ["Windows-1252",
                        "utf8"] # 1252 is a superset of iso-8859-1.
                                # Most sites that claim to be

fanficdownloader/adapters/adapter_ncisfictionnet.py

@@ -37,10 +37,10 @@ class NCISFictionNetAdapter(BaseSiteAdapter):
     def __init__(self, config, url):
         BaseSiteAdapter.__init__(self, config, url)
 
-        # ncisfiction.net blocks IPs the default user-agent.  However,
+        # ncisfiction.net blocks the default user-agent.  However,
         # when asked, they said it was just general anti-spam, not
         # targeted at us.  That lets me do this in good conscience:
-        self.opener.addheaders = [('User-agent', 'FFDL/1.6')]
+        self.opener.addheaders = [('User-agent', 'FFDL/1.7')]
         
         self.decode = ["iso-8859-1",
                        "Windows-1252"] # 1252 is a superset of iso-8859-1.

fanficdownloader/adapters/adapter_storiesofardacom.py

@@ -137,8 +137,20 @@ class StoriesOfArdaComAdapter(BaseSiteAdapter):
 
         logger.debug('Getting chapter text from: %s' % url)
 
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+        if self.getConfig('is_adult'):
+            params = {'confirmAge':'1'}
+            data = self._postUrl(url,params)
+        else:
+            data = self._fetchUrl(url)
+
+        data = data[data.index('<table width="90%" align="center">'):]
+        data.replace("<body","<notbody").replace("<BODY","<NOTBODY")
+            
+        soup = bs.BeautifulStoneSoup(data,
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+
+        if "Please indicate that you are an adult by selecting the appropriate choice below" in data:
+            raise exceptions.FailedToDownload("Chapter requires you be an adult.  Set is_adult in personal.ini (chapter url:%s)" % url)
         
         div = soup.find('table', {'width' : '90%'}).find('td')
         div.name='div'

fanficdownloader/adapters/adapter_thehexfilesnet.py

@@ -103,7 +103,7 @@ class TheHexFilesNetAdapter(BaseSiteAdapter):
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
         self.story.setMetadata('authorId',a['href'].split('=')[1])
         self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
+        self.story.setMetadata('author',stripHTML(a))
         asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
 		
         try:
@@ -118,7 +118,7 @@ class TheHexFilesNetAdapter(BaseSiteAdapter):
         for info in asoup.findAll('table', {'cellspacing' : '4'}):
             a = info.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
             if a != None:
-                self.story.setMetadata('title',a.string)
+                self.story.setMetadata('title',stripHTML(a))
                 break
 		
 

index.html

@@ -57,7 +57,7 @@
 	  <h3>Changes:</h3>
 	  <p>
 	    <ul>
-	      <li>Several new ini parameters for metadata.  See the Plugin forum for details.</li>
+	      <li>Fixes fanfiktion.de, storiesofarda.com, hpfanficarchive.com, and dark-solace.org/elysian sites.</li>
 	    </ul>
 	  </p>
 	  
@@ -69,7 +69,7 @@
 	    If you have any problems with this application, please
 	    report them in
 	    the <a href="http://groups.google.com/group/fanfic-downloader">FanFictionDownLoader Google Group</a>.  The
-	    <a href="http://4-4-64.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
+	    <a href="http://4-4-66.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
 	  </p>
 	  <div id='error'>
 	    {{ error_message }}

plugin-defaults.ini

@@ -1337,6 +1337,11 @@ titlepage_entries: series,seriesUrl,category,genre,language,status,datePublished
 ## Site dedicated to these categories/characters/ships
 extracategories:Lord of the Rings
 
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
 [www.thepetulantpoetess.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -1458,7 +1463,8 @@ extracategories:Stargate: Atlantis
 [teststory:defaults]
 valid_entries:title,author_list,authorId_list,authorUrl_list,storyUrl,
  datePublished,dateUpdated,numWords,status,language,series,seriesUrl,
- rating,category_list,genre_list,warnings_list,characters_list
+ rating,category_list,genre_list,warnings_list,characters_list,ships_list,
+ description,site,extratags
 
 # {{storyId}} is a special case--it's the only one that works.
 title:Test Story Title {{storyId}}