Fix to handle extratags correctly, kludge for ffnet for update emails that come

before the site is completely updated, fix plugin keeptags.

calibre-plugin/__init__.py

@@ -27,7 +27,7 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
     description         = 'UI plugin to download FanFiction stories from various sites.'
     supported_platforms = ['windows', 'osx', 'linux']
     author              = 'Jim Miller'
-    version             = (1, 2, 1)
+    version             = (1, 2, 2)
     minimum_calibre_version = (0, 8, 30)
 
     #: This field defines the GUI plugin class that contains all the code

calibre-plugin/ffdl_plugin.py

@@ -700,16 +700,15 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
 
     def _update_metadata(self, db, book_id, book, mi):
         if prefs['keeptags']:
-            mi = copy.deepcopy(mi)
             old_tags = db.get_tags(book_id)
             # remove old Completed/In-Progress only if there's a new one.
             if 'Completed' in mi.tags or 'In-Progress' in mi.tags:
                 old_tags = filter( lambda x : x not in ('Completed', 'In-Progress'), old_tags)
                 # remove old Last Update tags if there are new ones.
-                if len(filter( lambda x : not x.startswith("Last Update"), mi.tags)) > 0:
-                    old_tags = filter( lambda x : not x.startswith("Last Update"), old_tags)
-                    # mi.tags needs to be list, but set kills dups.
-                mi.tags = list(set(list(old_tags)+mi.tags)) 
+            if len(filter( lambda x : not x.startswith("Last Update"), mi.tags)) > 0:
+                old_tags = filter( lambda x : not x.startswith("Last Update"), old_tags)
+            # mi.tags needs to be list, but set kills dups.
+            mi.tags = list(set(list(old_tags)+mi.tags)) 
         # Set language english, but only if not already set.
         oldmi = db.get_metadata(book_id,index_is_id=True)
         if not oldmi.languages:

fanficdownloader/adapters/adapter_fanfictionnet.py

@@ -70,7 +70,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
 
         url = self.origurl
         logging.debug("URL: "+url)
-        
+
         # use BeautifulSoup HTML parser to make everything easier to find.
         try:
             data = self._fetchUrl(url)
@@ -86,7 +86,26 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
             
         if "Chapter not found. Please check to see you are not using an outdated url." in data:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  'Chapter not found. Please check to see you are not using an outdated url.'" % url)
-            
+
+        try:
+            # rather nasty way to check for a newer chapter.  ffnet has a
+            # tendency to send out update notices in email before all
+            # their servers are showing the update on the first chapter.
+            chapcount = len(soup.find('select', { 'name' : 'chapter' } ).findAll('option'))
+            # get chapter part of url.
+            chapter = url.split('/',)[5]
+            tryurl = "http://%s/s/%s/%d/"%(self.getSiteDomain(),
+                                           self.story.getMetadata('storyId'),
+                                           chapcount+1)
+            #print('=Trying newer chapter: %s' % tryurl)
+            newdata = self._fetchUrl(tryurl)
+            if "Chapter not found. Please check to see you are not using an outdated url." \
+                    not in newdata:
+                #print('=======Found newer chapter: %s' % tryurl)
+                soup = bs.BeautifulSoup(newdata)
+        except:
+            pass
+        
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"^/u/\d+"))
         self.story.setMetadata('authorId',a['href'].split('/')[2])
@@ -219,7 +238,6 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         
         return
 
-
     def getChapterText(self, url):
         logging.debug('Getting chapter text from: %s' % url)
         time.sleep(0.5) ## ffnet(and, I assume, fpcom) tends to fail

fanficdownloader/writers/base_writer.py

@@ -101,6 +101,9 @@ class BaseStoryWriter(Configurable):
             }
         self.story.setMetadata('formatname',self.getFormatName())
         self.story.setMetadata('formatext',self.getFormatExt())
+        
+        for tag in self.getConfigList("extratags"):
+            self.story.addToList("extratags",tag)
 
     def getMetadata(self,key):
         return stripHTML(self.story.getMetadata(key))
@@ -184,8 +187,6 @@ class BaseStoryWriter(Configurable):
 
     # if no outstream is given, write to file.
     def writeStory(self,outstream=None, metaonly=False, outfilename=None, forceOverwrite=False):
-        for tag in self.getConfigList("extratags"):
-            self.story.addToList("extratags",tag)
 
         self.metaonly = metaonly
         if outfilename == None:
@@ -266,9 +267,6 @@ class BaseStoryWriter(Configurable):
             if name in self.getConfigList("include_subject_tags"):
                 for tag in lst:
                     subjectset.add(tag)
-                    
-        for tag in self.getConfigList("extratags"):
-            subjectset.add(tag)
 
         return list(subjectset)