Bump versions, update index.html.

meta section, fix highbyte chars in description.

app.yaml

@@ -1,6 +1,6 @@
 # ffd-retief-hrd fanfictiondownloader
 application: fanfictiondownloader
-version: 4-4-58
+version: 4-4-60
 runtime: python27
 api_version: 1
 threadsafe: true

calibre-plugin/__init__.py

@@ -26,7 +26,7 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
     description         = 'UI plugin to download FanFiction stories from various sites.'
     supported_platforms = ['windows', 'osx', 'linux']
     author              = 'Jim Miller'
-    version             = (1, 7, 25)
+    version             = (1, 7, 27)
     minimum_calibre_version = (0, 8, 57)
 
     #: This field defines the GUI plugin class that contains all the code

calibre-plugin/ffdl_plugin.py

@@ -862,11 +862,25 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                                                "<p>Click '<b>Yes</b>' to %se book with new URL.</p>"%updat+
                                                "<p>Click '<b>No</b>' to skip %sing this book.</p>"%updat,
                                                show_copy_button=False):
-                            book['comment'] = "Update declined by user due to differing story URL(%s)"%liburl
-                            book['good']=False
-                            book['icon']='rotate-right.png'
-                            book['status'] = 'Different URL'
-                            return
+                            if question_dialog(self.gui, 'Download as New Book?',
+                                               '<h3>Download as New Book?</h3>'+
+                                               '<p><b>%s</b> by <b>%s</b> is already in your library with a different source URL.</p>'%
+                                               (mi.title,', '.join(mi.author))+
+                                               '<p>You chose not to update the existing book.  Do you want to add a new book for this URL?</p>'+
+                                               '<p>New URL: <a href="%(newurl)s">%(newurl)s</a></p>'%
+                                               {'newurl':book['url']}+
+                                               "<p>Click '<b>Yes</b>' to a new book with new URL.</p>"+
+                                               "<p>Click '<b>No</b>' to skip URL.</p>",
+                                               show_copy_button=False):
+                                book_id = None
+                                mi = None
+                                book['calibre_id'] = None
+                            else:
+                                book['comment'] = "Update declined by user due to differing story URL(%s)"%liburl
+                                book['good']=False
+                                book['icon']='rotate-right.png'
+                                book['status'] = 'Different URL'
+                                return
         
             if book_id != None and collision != ADDNEW:
                 if collision in (CALIBREONLY):
@@ -1309,15 +1323,6 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                 except:
                     print("Failed to set_cover, skipping")
 
-        # set author link if found.  All current adapters have authorUrl, except anonymous on AO3.
-        if 'authorUrl' in book['all_metadata']:
-            authurls = book['all_metadata']['authorUrl'].split(", ")
-            for i, auth in enumerate(book['author']):
-                #print("===Update author url for %s to %s"%(auth,authurls[i]))
-                autid=db.get_author_id(auth)
-                db.set_link_field_for_author(autid, unicode(authurls[i]),
-                                             commit=False, notify=False)
-
         # implement 'newonly' flags here by setting to the current
         # value again.
         if not book['added']:
@@ -1395,7 +1400,11 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
 
                     if flag == 'r' or book['added']: # flag 'n' isn't actually needed--*always* set if configured and new book.
                         if coldef['datatype'] in ('int','float'): # for favs, etc--site specific metadata.
-                            val = unicode(book['all_metadata'][meta]).replace(",","")
+                            if 'anthology_meta_list' in book and meta in book['anthology_meta_list']:
+                                # re-split list, strip commas, convert to floats, sum up.
+                                val = sum([ float(x.replace(",","")) for x in book['all_metadata'][meta].split(", ") ])
+                            else:
+                                val = unicode(book['all_metadata'][meta]).replace(",","")
                         else:
                             val = book['all_metadata'][meta]
                         db.set_custom(book_id, val, label=label, commit=False)
@@ -1416,6 +1425,16 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                             
                         db.set_custom(book_id, ", ".join(vallist), label=label, commit=False)
 
+        # set author link if found.  All current adapters have authorUrl, except anonymous on AO3.
+        # Moved down so author's already in the DB.
+        if 'authorUrl' in book['all_metadata']:
+            authurls = book['all_metadata']['authorUrl'].split(", ")
+            for i, auth in enumerate(book['author']):
+                #print("===Update author url for %s to %s"%(auth,authurls[i]))
+                autid=db.get_author_id(auth)
+                db.set_link_field_for_author(autid, unicode(authurls[i]),
+                                             commit=False, notify=False)
+
         db.commit()
 
         if 'Generate Cover' in self.gui.iactions and (book['added'] or not prefs['gcnewonly']):
@@ -1674,6 +1693,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         book['tags'] = []
         book['url'] = ''
         book['all_metadata'] = {}
+        book['anthology_meta_list'] = {}
         book['comment'] = ''
         book['added'] = True
         book['good'] = True
@@ -1740,6 +1760,10 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                         book['all_metadata'][k]=book['all_metadata'][k]+"\n\n"+v
                     else:
                         book['all_metadata'][k]=book['all_metadata'][k]+", "+v
+                        # flag psuedo list element.  Used so numeric
+                        # cust cols can convert back to numbers and
+                        # add.
+                        book['anthology_meta_list'][k]=True
     
         if existingbook:
             book['title'] = deftitle = existingbook['title']

defaults.ini

@@ -622,7 +622,7 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 ## composite metadata entries.  dramione.org, for example, adds
 ## 'cliches' and then defines as the composite of hermiones,dracos in
 ## include_in_cliches.
-extra_valid_entries:themes,hermiones,dracos,timeline,cliches
+extra_valid_entries:themes,hermiones,dracos,timeline,cliches,read,reviews
 include_in_cliches:hermiones,dracos
 
 ## For another example, you could, by uncommenting this line, include
@@ -722,6 +722,10 @@ extracharacters:Hermione Granger
 ## personal.ini, not defaults.ini.
 #is_adult:true
 
+## Extra metadata that this adapter knows about.  See [dramione.org]
+## for examples of how to use them.
+extra_valid_entries:read,reviews
+
 [hlfiction.net]
 ## Site dedicated to these categories/characters/ships
 extracategories:Highlander
@@ -1179,6 +1183,11 @@ cover_exclusion_regexp:/stories/999/images/.*?_trophy.png
 ## Site dedicated to these categories/characters/ships
 extracategories:NCIS
 
+[www.nickandgreg.net]
+## Site dedicated to these categories/characters/ships
+extracategories:CSI
+extraships:Nick Stokes/Greg Sanders
+
 [www.phoenixsong.net]
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter

fanficdownloader/adapters/__init__.py

@@ -115,6 +115,7 @@ import adapter_hennethannunnet
 import adapter_tokrafandomnetcom
 import adapter_netraptororg
 import adapter_asr3slashzoneorg
+import adapter_nickandgregnet
 
 ## This bit of complexity allows adapters to be added by just adding
 ## importing.  It eliminates the long if/else clauses we used to need

fanficdownloader/adapters/adapter_dramioneorg.py

@@ -237,6 +237,9 @@ class DramioneOrgAdapter(BaseSiteAdapter):
             if 'Word count' in label:
                 self.story.setMetadata('numWords', value)
 
+            if 'Read' in label:
+                self.story.setMetadata('read', value)
+
             if 'Categories' in label:
                 cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
                 for cat in cats:
@@ -282,6 +285,14 @@ class DramioneOrgAdapter(BaseSiteAdapter):
             # I find it hard to care if the series parsing fails
             pass
             
+        try:
+            self.story.setMetadata('reviews',
+                                   stripHTML(soup.find('h2',{'id':'pagetitle'}).
+                                             findAll('a', href=re.compile(r'^reviews.php'))[1]))
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 

fanficdownloader/adapters/adapter_fanfictionnet.py

@@ -135,12 +135,25 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         categories = soup.find('div',{'id':'pre_story_links'}).findAll('a',{'class':'xcontrast_txt'})
         #print("xcontrast_txt a:%s"%categories)
         if len(categories) > 1:
+            # Strangely, the ones with *two* links are the
+            # non-crossover categories.  Each is in a category itself
+            # of Book, Movie, etc.
             self.story.addToList('category',stripHTML(categories[1]))
         elif 'Crossover' in categories[0]['href']:
             caturl = "http://%s%s"%(self.getSiteDomain(),categories[0]['href'])
             catsoup = bs.BeautifulSoup(self._fetchUrl(caturl))
             for a in catsoup.findAll('a',href=re.compile(r"^/crossovers/")):
                 self.story.addToList('category',stripHTML(a))
+            else:
+                # Fall back.  I ran across a story with a Crossver
+                # category link to a broken page once.
+                # http://www.fanfiction.net/s/2622060/1/
+                # Naruto + Harry Potter Crossover
+                logger.info("Fall back category collection")
+                for c in stripHTML(categories[0]).replace(" Crossover","").split(' + '):
+                    self.story.addToList('category',c)
+                
+                
             
         a = soup.find('a', href='http://www.fictionratings.com/')
         rating = a.string

fanficdownloader/adapters/adapter_grangerenchantedcom.py

@@ -227,6 +227,9 @@ class GrangerEnchantedCom(BaseSiteAdapter):
             if 'Word count' in label:
                 self.story.setMetadata('numWords', value)
 
+            if 'Read' in label:
+                self.story.setMetadata('read', value)
+
             if 'Categories' in label:
                 cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
                 for cat in cats:
@@ -278,7 +281,14 @@ class GrangerEnchantedCom(BaseSiteAdapter):
                         self.story.setMetadata('seriesUrl',series_url)
                         break
                     i+=1
-            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+
+        try:
+            self.story.setMetadata('reviews',
+                                   stripHTML(soup.find('div',{'id':'sort'}).
+                                             findAll('a', href=re.compile(r'^reviews.php'))[1]))
         except:
             # I find it hard to care if the series parsing fails
             pass

fanficdownloader/adapters/adapter_nhamagicalworldsus.py

@@ -112,7 +112,8 @@ class NHAMagicalWorldsUsAdapter(BaseSiteAdapter):
 		
         for info in asoup.findAll('table', {'width' : '100%', 'bordercolor' : re.compile(r'#')}):
             a = info.find('a')
-            if ('viewstory.php?sid='+self.story.getMetadata('storyId')) in a['href']:
+            if 'viewstory.php?sid='+self.story.getMetadata('storyId') == a['href'] or \
+                    ('viewstory.php?sid='+self.story.getMetadata('storyId')+'&') in a['href']:
                 self.story.setMetadata('title',a.string)
                 break
 		
@@ -142,14 +143,14 @@ class NHAMagicalWorldsUsAdapter(BaseSiteAdapter):
         for cat in cats:
             self.story.addToList('category',cat.string)
 					
-        a = info.find('a', href=re.compile(r'reviews.php\?sid='+self.story.getMetadata('storyId')))
+        a = info.find('a', href=re.compile(r'viewuser.php'))
         val = a.nextSibling
         svalue = ""
         while not defaultGetattr(val) == 'br':
             val = val.nextSibling
         val = val.nextSibling
         while not defaultGetattr(val) == 'br':
-            svalue += str(val)
+            svalue += unicode(val)
             val = val.nextSibling
         self.setDescription(url,svalue)
         

fanficdownloader/adapters/adapter_nickandgregnet.py

@@ -0,0 +1,174 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return NickAndGregNetAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class NickAndGregNetAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
+        self._setURL('http://' + self.getSiteDomain() + '/desert_archive/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','nag')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%Y/%m/%d"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.nickandgreg.net'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/desert_archive/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/desert_archive/viewstory.php?sid=")+r"\d+$"
+    
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&i=1'
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/desert_archive/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        chapters = soup.find('select')
+        for chapter in chapters.findAll('option'):
+            if chapter.text != 'Story Index' and chapter.text != 'Chapters':
+                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/desert_archive/'+chapter['value']))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+        
+        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
+        
+        for div in asoup.findAll('td', {'class' : 'tblborder6'}):
+            a = div.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+            if a != None:
+                break
+        
+        self.setDescription(url,div.find('br').nextSibling)
+        
+        a=div.text.split('Rating:')
+        if len(a) == 2: self.story.setMetadata('rating', a[1].split(' -')[0])
+        
+        a=div.text.split('Characters:')
+        if len(a) == 2: 
+        	for char in a[1].split(' -')[0].split(', '):
+        	    self.story.addToList('characters',char)
+        	    
+        a=div.text.split('Genres:')
+        if len(a) == 2: 
+        	for genre in a[1].split(' -')[0].split(', '):
+        	    self.story.addToList('genre',genre)
+        	    
+        a=div.text.split('Warnings:')
+        if len(a) == 2: 
+        	for warn in a[1].split(' -')[0].split(', '):
+        	    if 'none' not in warn:
+        	        self.story.addToList('warnings',warn)
+        	    
+        a=div.text.split('Completed:')
+        if len(a) ==2:
+            if 'Yes' in a[1]:
+                self.story.setMetadata('status', 'Completed')
+            else:
+                self.story.setMetadata('status', 'In-Progress')
+                
+        a=div.text.split('Published:')
+        if len(a) == 2: self.story.setMetadata('datePublished', makeDate(stripHTML(a[1].split(' -')[0]), self.dateformat))
+        
+        a=div.text.split('Updated:')
+        if len(a) == 2: self.story.setMetadata('dateUpdated', makeDate(stripHTML(a[1].split(' -')[0]), self.dateformat))
+
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('table', {'class' : 'tblborder6'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_scarvesandcoffeenet.py

@@ -141,7 +141,7 @@ class ScarvesAndCoffeeNetAdapter(BaseSiteAdapter):
         self.story.setMetadata('title',a.string)
         
         # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        a = soup.find('div',{"id":"pagetitle"}).find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
         self.story.setMetadata('authorId',a['href'].split('=')[1])
         self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
         self.story.setMetadata('author',a.string)

fanficdownloader/geturls.py

@@ -54,9 +54,14 @@ def get_urls_from_page(url,configuration=None,normalize=False):
         opener = u2.build_opener(u2.HTTPCookieProcessor(),GZipProcessor())
         data = opener.open(url).read()
 
-    return get_urls_from_html(data,url,configuration,normalize)
+    # kludge because I don't see it on enough sites to be worth generalizing yet.
+    restrictsearch=None
+    if 'scarvesandcoffee.net' in url:
+        restrictsearch=('div',{'id':'mainpage'})
 
-def get_urls_from_html(data,url=None,configuration=None,normalize=False):
+    return get_urls_from_html(data,url,configuration,normalize,restrictsearch)
+
+def get_urls_from_html(data,url=None,configuration=None,normalize=False,restrictsearch=None):
 
     normalized = [] # normalized url
     retlist = [] # orig urls.
@@ -65,6 +70,9 @@ def get_urls_from_html(data,url=None,configuration=None,normalize=False):
         configuration = Configuration("test1.com","EPUB")
 
     soup = BeautifulSoup(data)
+    if restrictsearch:
+        soup = soup.find(*restrictsearch)
+        print("restrict search:%s"%soup)
     
     for a in soup.findAll('a'):
         if a.has_key('href'):

index.html

@@ -57,7 +57,8 @@
 	  <h3>Changes:</h3>
 	  <p>
 	    <ul>
-	      <li>Don't strip lead/trail whitespace from replace_metadata, add feature \s->' ' in replace_metadata replacements.</li>
+	      <li>Fixes for nha.magical-worlds.us and scarvesandcoffee.net.</li>
+	      <li>Fall back category parsing for fanfiction.net when story has a broken crossover category link.</li>
 	    </ul>
 	  </p>
 	  <p>
@@ -68,7 +69,7 @@
 	    If you have any problems with this application, please
 	    report them in
 	    the <a href="http://groups.google.com/group/fanfic-downloader">FanFictionDownLoader Google Group</a>.  The
-	    <a href="http://4-4-56.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
+	    <a href="http://4-4-59.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
 	  </p>
 	  <div id='error'>
 	    {{ error_message }}
@@ -612,6 +613,11 @@
 	    Use the URL of the story's chapter list, such as
 	    <br /><a href="http://netraptor.org/archive/viewstory.php?sid=1234">http://netraptor.org/archive/viewstory.php?sid=1234</a>
 	  </dd>
+	  <dt>nickandgreg.net</dt>
+	  <dd>
+	    Use the URL of the story's chapter list, such as
+	    <br /><a href="http://www.nickandgreg.net/desert_archive/viewstory.php?sid=1234">http://www.nickandgreg.net/desert_archive/viewstory.php?sid=1234</a>
+	  </dd>
 	</dl>
 
 	<p>

plugin-defaults.ini

@@ -593,7 +593,7 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 ## composite metadata entries.  dramione.org, for example, adds
 ## 'cliches' and then defines as the composite of hermiones,dracos in
 ## include_in_cliches.
-extra_valid_entries:themes,hermiones,dracos,timeline,cliches
+extra_valid_entries:themes,hermiones,dracos,timeline,cliches,read,reviews
 include_in_cliches:hermiones,dracos
 
 ## For another example, you could, by uncommenting this line, include
@@ -709,6 +709,10 @@ extracharacters:Hermione Granger
 ## personal.ini, not defaults.ini.
 #is_adult:true
 
+## Extra metadata that this adapter knows about.  See [dramione.org]
+## for examples of how to use them.
+extra_valid_entries:read,reviews
+
 [hlfiction.net]
 ## Site dedicated to these categories/characters/ships
 extracategories:Highlander
@@ -1163,6 +1167,11 @@ cover_exclusion_regexp:/stories/999/images/.*?_trophy.png
 ## Site dedicated to these categories/characters/ships
 extracategories:NCIS
 
+[www.nickandgreg.net]
+## Site dedicated to these categories/characters/ships
+extracategories:CSI
+extraships:Nick Stokes/Greg Sanders
+
 [www.phoenixsong.net]
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter