Bump cli/web version too.

AO3 doesn't reliably report chapter author.

app.yaml

@@ -1,6 +1,6 @@
 # ffd-retief-hrd fanfictiondownloader
 application: fanfictiondownloader
-version: 4-4-49
+version: 4-4-52
 runtime: python27
 api_version: 1
 threadsafe: true

calibre-plugin/__init__.py

@@ -27,7 +27,7 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
     description         = 'UI plugin to download FanFiction stories from various sites.'
     supported_platforms = ['windows', 'osx', 'linux']
     author              = 'Jim Miller'
-    version             = (1, 7, 16)
+    version             = (1, 7, 19)
     minimum_calibre_version = (0, 8, 57)
 
     #: This field defines the GUI plugin class that contains all the code

calibre-plugin/dialogs.py

@@ -318,15 +318,16 @@ class AddNewDialog(SizePersistedDialog):
         if not self.merge:
             self.fileform.setCurrentIndex(self.fileform.findText(self.prefs['fileform']))
 
-        if self.merge and not self.newmerge:
-            self.set_collisions()
-            i = self.collision.findText(self.prefs['collision'])
-            if i > -1:
-                self.collision.setCurrentIndex(i)
-            self.updatemeta.setChecked(self.prefs['updatemeta'])
-
-            if not self.merge:
-                self.updateepubcover.setChecked(self.prefs['updateepubcover'])
+        # add collision options
+        self.set_collisions()
+        
+        i = self.collision.findText(self.prefs['collision'])
+        if i > -1:
+            self.collision.setCurrentIndex(i)
+        self.updatemeta.setChecked(self.prefs['updatemeta'])
+            
+        if not self.merge:
+            self.updateepubcover.setChecked(self.prefs['updateepubcover'])
                 
         self.url.setText(url_list_text)
         if url_list_text:

calibre-plugin/ffdl_plugin.py

@@ -102,7 +102,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         # are not found in the zip file will result in null QIcons.
         icon = get_icon('images/icon.png')
 
-        #self.qaction.setText('FFDL')
+        self.qaction.setText('FanFictionDL')
         
         # The qaction is automatically created from the action_spec defined
         # above
@@ -140,7 +140,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
             file_path = os.path.join(calibre_config_dir,
                                      *("plugins/fanfictiondownloader_macmenuhack.txt".split('/')))
             file_path = os.path.abspath(file_path)
-            print("macmenuhack file_path:%s"%file_path)
+            print("Plugin %s macmenuhack file_path:%s"%(self.name,file_path))
             self.macmenuhack = os.access(file_path, os.F_OK)
             return self.macmenuhack
 
@@ -1533,7 +1533,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
     def convert_url_to_book(self, url):
         book = self.make_book()
         # look here for [\d,\d] at end of url, and remove?
-        mc = re.match(r"^(?P<url>.*?)(?:\[(?P<begin>\d+)?(?P<comma>,)?(?P<end>\d+)?\])?$",url)
+        mc = re.match(r"^(?P<url>.*?)(?:\[(?P<begin>\d+)?(?P<comma>[,-])?(?P<end>\d+)?\])?$",url)
         #print("url:(%s) begin:(%s) end:(%s)"%(mc.group('url'),mc.group('begin'),mc.group('end')))
         url = mc.group('url')
         book['begin'] = mc.group('begin')

calibre-plugin/prefs.py

@@ -61,6 +61,13 @@ default_prefs['allow_custcol_from_ini'] = True
 
 default_prefs['std_cols_newonly'] = {}
 
+# This is where all preferences for this plugin *were* stored
+# Remember that this name (i.e. plugins/fanfictiondownloader_plugin) is also
+# in a global namespace, so make it as unique as possible.
+# You should always prefix your config file name with plugins/,
+# so as to ensure you dont accidentally clobber a calibre config file
+old_prefs = JSONConfig('plugins/fanfictiondownloader_plugin')
+
 def set_library_config(library_config,db):
     db.prefs.set_namespaced(PREFS_NAMESPACE,
                             PREFS_KEY_SETTINGS,
@@ -74,7 +81,7 @@ def get_library_config(db):
     if library_id in old_prefs:
         #print("get prefs from old_prefs")
         library_config = old_prefs[library_id]
-        set_library_config(library_config)
+        set_library_config(library_config,db)
         old_prefs["migrated to library db %s"%library_id] = old_prefs[library_id]
         del old_prefs[library_id]
 
@@ -84,13 +91,6 @@ def get_library_config(db):
                                                  copy.deepcopy(default_prefs))
     return library_config
 
-# This is where all preferences for this plugin *were* stored
-# Remember that this name (i.e. plugins/fanfictiondownloader_plugin) is also
-# in a global namespace, so make it as unique as possible.
-# You should always prefix your config file name with plugins/,
-# so as to ensure you dont accidentally clobber a calibre config file
-old_prefs = JSONConfig('plugins/fanfictiondownloader_plugin')
-
 # fake out so I don't have to change the prefs calls anywhere.  The
 # Java programmer in me is offended by op-overloading, but it's very
 # tidy.

downloader.py

@@ -94,6 +94,9 @@ def main(argv,
    parser.add_option("-l", "--list",
                      action="store_true", dest="list",
                      help="Get list of valid story URLs from page given.",)
+   parser.add_option("-n", "--normalize-list",
+                     action="store_true", dest="normalize",default=False,
+                     help="Get list of valid story URLs from page given, but normalized to standard forms.",)
    parser.add_option("-d", "--debug",
                      action="store_true", dest="debug",
                      help="Show debug output while downloading.",)
@@ -169,8 +172,8 @@ def main(argv,
            (var,val) = opt.split('=')
            configuration.set("overrides",var,val)
 
-   if options.list:
-       retlist = get_urls_from_page(args[0], configuration)
+   if options.list or options.normalize:
+       retlist = get_urls_from_page(args[0], configuration, normalize=options.normalize)
        print "\n".join(retlist)
                
        return

fanficdownloader/adapters/adapter_fanfictionnet.py

@@ -132,9 +132,10 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         ## For 1, use the second link.
         ## For 2, fetch the crossover page and pull the two categories from there.
 
-        categories = soup.findAll('a',{'class':'xcontrast_txt'})
+        categories = soup.find('div',{'id':'pre_story_links'}).findAll('a',{'class':'xcontrast_txt'})
+        #print("xcontrast_txt a:%s"%categories)
         if len(categories) > 1:
-            self.story.addToList('category',stripHTML(categories[-1]))
+            self.story.addToList('category',stripHTML(categories[1]))
         elif 'Crossover' in categories[0]['href']:
             caturl = "http://%s%s"%(self.getSiteDomain(),categories[0]['href'])
             catsoup = bs.BeautifulSoup(self._fetchUrl(caturl))
@@ -158,12 +159,17 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         if summarydiv:
             self.setDescription(url,stripHTML(summarydiv))
             
-        
-        metatext = stripHTML(gui_table1i.find('div', {'class':'xgray'})).replace('Hurt/Comfort','Hurt-Comfort')
+
+        grayspan = gui_table1i.find('span', {'class':'xgray xcontrast_txt'})
+        # for b in grayspan.findAll('button'):
+        #     b.extract()
+        metatext = stripHTML(grayspan).replace('Hurt/Comfort','Hurt-Comfort')
+        #logger.debug("metatext:(%s)"%metatext)
         metalist = metatext.split(" - ")
-        logger.debug("metatext:(%s)"%metalist)
+        #logger.debug("metalist:(%s)"%metalist)
 
         # Rated: Fiction K - English - Words: 158,078 - Published: 02-04-11
+        # Rated: Fiction T - English - Adventure/Sci-Fi - Naruto U. - Chapters: 22 - Words: 114,414 - Reviews: 395 - Favs: 779 - Follows: 835 - Updated: 03-21-13 - Published: 04-28-12 - id: 8067258 
 
         # rating is obtained above more robustly.
         if metalist[0].startswith('Rated:'):

fanficdownloader/adapters/adapter_tthfanficorg.py

@@ -163,17 +163,44 @@ class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter):
                 infodata = self._fetchUrl(infourl)
                 infosoup = bs.BeautifulSoup(infodata)
 
-                for a in infosoup.findAll('a',href=re.compile(r"^/Author-\d+")):
-                    self.story.addToList('authorId',a['href'].split('/')[1].split('-')[1])
-                    self.story.addToList('authorUrl','http://'+self.host+a['href'].replace("/Author-","/AuthorStories-"))
-                    self.story.addToList('author',stripHTML(a))                
+                # for a in infosoup.findAll('a',href=re.compile(r"^/Author-\d+")):
+                #     self.story.addToList('authorId',a['href'].split('/')[1].split('-')[1])
+                #     self.story.addToList('authorUrl','http://'+self.host+a['href'].replace("/Author-","/AuthorStories-"))
+                #     self.story.addToList('author',stripHTML(a))
+
+                # second verticaltable is the chapter list.
+                table = infosoup.findAll('table',{'class':'verticaltable'})[1]
+                for a in table.findAll('a',href=re.compile(r"^/Story-"+self.story.getMetadata('storyId'))):
+                    autha = a.findNext('a',href=re.compile(r"^/Author-\d+"))
+                    self.story.addToList('authorId',autha['href'].split('/')[1].split('-')[1])
+                    self.story.addToList('authorUrl','http://'+self.host+autha['href'].replace("/Author-","/AuthorStories-"))
+                    self.story.addToList('author',stripHTML(autha))
+                    # include leading number to match 1. ... 2. ...
+                    self.chapterUrls.append(("%d. %s by %s"%(len(self.chapterUrls)+1,
+                                                             stripHTML(a),
+                                                             stripHTML(autha)),'http://'+self.host+a['href']))
                                             
             except urllib2.HTTPError, e:
                 if e.code == 404:
                     raise exceptions.StoryDoesNotExist(url)
                 else:
                     raise e
+        else: # single author:
+            # Find the chapter selector 
+            select = soup.find('select', { 'name' : 'chapnav' } )
+        	 
+            if select is None:
+        	   # no selector found, so it's a one-chapter story.
+        	   self.chapterUrls.append((self.story.getMetadata('title'),url))
+            else:
+                allOptions = select.findAll('option')
+                for o in allOptions:
+                    url = "http://"+self.host+o['value']
+                    # just in case there's tags, like <i> in chapter titles.
+                    self.chapterUrls.append((stripHTML(o),url))
     
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
         try:
             # going to pull part of the meta data from *primary* author list page.
             logger.debug("**AUTHOR** URL: "+authorurl)
@@ -235,21 +262,6 @@ class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter):
         if BtVS:
             self.story.addToList('category','Buffy: The Vampire Slayer')
             
-        # Find the chapter selector 
-        select = soup.find('select', { 'name' : 'chapnav' } )
-    	 
-        if select is None:
-    	   # no selector found, so it's a one-chapter story.
-    	   self.chapterUrls.append((self.story.getMetadata('title'),url))
-        else:
-            allOptions = select.findAll('option')
-            for o in allOptions:
-                url = "http://"+self.host+o['value']
-                # just in case there's tags, like <i> in chapter titles.
-                self.chapterUrls.append((stripHTML(o),url))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
         pseries = soup.find('p', {'style':'margin-top:0px'})
         m = re.match('This story is No\. (?P<num>\d+) in the series &quot;(?P<series>.+)&quot;\.',
                      pseries.text)

fanficdownloader/adapters/adapter_wraithbaitcom.py

@@ -126,8 +126,13 @@ class WraithBaitComAdapter(BaseSiteAdapter):
 
         # Find the chapters:
         for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # include author on chapters if multiple authors.
+            if len(alist) > 1:
+                add = " by %s"%stripHTML(chapter.findNext('a', href=re.compile(r"viewuser.php\?uid=\d+")))
+            else:
+                add = ""
             # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+            self.chapterUrls.append((stripHTML(chapter)+add,'http://'+self.host+'/'+chapter['href']+addurl))
 
         self.story.setMetadata('numChapters',len(self.chapterUrls))
 

fanficdownloader/geturls.py

@@ -25,7 +25,7 @@ from gziphttp import GZipProcessor
 import adapters
 from configurable import Configuration
 
-def get_urls_from_page(url,configuration=None):
+def get_urls_from_page(url,configuration=None,normalize=False):
 
     if not configuration:
         configuration = Configuration("test1.com","EPUB")
@@ -54,11 +54,11 @@ def get_urls_from_page(url,configuration=None):
         opener = u2.build_opener(u2.HTTPCookieProcessor(),GZipProcessor())
         data = opener.open(url).read()
 
-    return get_urls_from_html(data,url)
+    return get_urls_from_html(data,url,configuration,normalize)
 
-def get_urls_from_html(data,url=None,configuration=None):
+def get_urls_from_html(data,url=None,configuration=None,normalize=False):
 
-    normalized = set() # normalized url
+    normalized = [] # normalized url
     retlist = [] # orig urls.
     
     if not configuration:
@@ -82,16 +82,19 @@ def get_urls_from_html(data,url=None,configuration=None):
                 href = href.replace('&index=1','')
                 adapter = adapters.getAdapter(configuration,href)
                 if adapter.story.getMetadata('storyUrl') not in normalized:
-                    normalized.add(adapter.story.getMetadata('storyUrl'))
+                    normalized.append(adapter.story.getMetadata('storyUrl'))
                     retlist.append(href)
             except:
                 pass
 
-    return retlist
+    if normalize:
+        return normalized
+    else:
+        return retlist
 
-def get_urls_from_text(data,configuration=None):
+def get_urls_from_text(data,configuration=None,normalize=False):
 
-    normalized = set() # normalized url
+    normalized = [] # normalized url
     retlist = [] # orig urls.
     
     if not configuration:
@@ -109,12 +112,15 @@ def get_urls_from_text(data,configuration=None):
             href = href.replace('&index=1','')
             adapter = adapters.getAdapter(configuration,href)
             if adapter.story.getMetadata('storyUrl') not in normalized:
-                normalized.add(adapter.story.getMetadata('storyUrl'))
+                normalized.append(adapter.story.getMetadata('storyUrl'))
                 retlist.append(href)
         except:
             pass
 
-    return retlist
+    if normalize:
+        return normalized
+    else:
+        return retlist
 
 def form_url(parenturl,url):
      url = url.strip() # ran across an image with a space in the

index.html

@@ -57,9 +57,9 @@
 	  <h3>Changes:</h3>
 	  <p>
 	    <ul>
-	      <li>Fix for changes to fanfiction.net.</li>
-	      <li>Add seriesUrl (and generated seriesHTML) as valid metadata entries. The default series on title_page is now a link.</li>
-	      <li>Add user/pass for dokuga.com.</li>
+	      <li>Another fix for fanfiction.net changes.</li>
+	      <li>Add author to chapter TOC for multi-author stories on TtH and WraithBait.<br />
+		  (AO3 doesn't reliably report chapter author.)</li>
 	    </ul>
 	  </p>
 	  <p>
@@ -70,7 +70,7 @@
 	    If you have any problems with this application, please
 	    report them in
 	    the <a href="http://groups.google.com/group/fanfic-downloader">FanFictionDownLoader Google Group</a>.  The
-	    <a href="http://4-4-47.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
+	    <a href="http://4-4-51.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
 	  </p>
 	  <div id='error'>
 	    {{ error_message }}