Changes for ffnet--longer sleep, config UA, bump versions.

app.yaml

@@ -1,6 +1,6 @@
 # ffd-retief-hrd fanfictiondownloader
 application: fanfictiondownloader
-version: 4-4-68
+version: 4-4-71
 runtime: python27
 api_version: 1
 threadsafe: true

calibre-plugin/__init__.py

@@ -26,7 +26,7 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
     description         = 'UI plugin to download FanFiction stories from various sites.'
     supported_platforms = ['windows', 'osx', 'linux']
     author              = 'Jim Miller'
-    version             = (1, 7, 37)
+    version             = (1, 7, 42)
     minimum_calibre_version = (0, 8, 57)
 
     #: This field defines the GUI plugin class that contains all the code

calibre-plugin/ffdl_plugin.py

@@ -7,13 +7,14 @@ __license__   = 'GPL v3'
 __copyright__ = '2012, Jim Miller'
 __docformat__ = 'restructuredtext en'
 
-import time, os, copy, threading, re, platform
+import time, os, copy, threading, re, platform, sys
 from StringIO import StringIO
 from functools import partial
 from datetime import datetime
 from string import Template
 import urllib
 import email
+import traceback
 
 from PyQt4.Qt import (QApplication, QMenu, QToolButton, QTimer)
 
@@ -1003,7 +1004,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                 mi = db.get_metadata(book_id,index_is_id=True)
                 if not book['series'] and mi.series != None:
                     book['calibre_series'] = (mi.series,mi.series_index)
-                    print("calibre_series:%s [%s]"%book['calibre_series'])
+                    #print("calibre_series:%s [%s]"%book['calibre_series'])
                 
         if book['good']: # there shouldn't be any !'good' books at this point.
             # if still 'good', make a temp file to write the output to.
@@ -1108,7 +1109,17 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
 
         if options['collision'] == CALIBREONLY or \
                 ( (options['updatemeta'] or book['added']) and book['good'] ):
-            self.update_metadata(db, book['calibre_id'], book, mi, options)
+            try:
+                self.update_metadata(db, book['calibre_id'], book, mi, options)
+            except:
+                det_msg = "".join(traceback.format_exception(*sys.exc_info()))+"\nStory Details:\n%s"%pretty_book(book)
+                print("Error Updating Metadata:\n%s"%det_msg)
+                error_dialog(self.gui,
+                             "Error Updating Metadata",
+                             "<p>An error has occurred while FFDL was updating calibre's metadata for <a href='%s'>%s</a>.</p>"%(book['url'],book['title'])+
+                             "The ebook has been updated, but the metadata has not.",
+                             det_msg=det_msg,
+                             show=True)
 
     def update_books_finish(self, book_list, options={}, showlist=True):
         '''Notify calibre about updated rows, update external plugins
@@ -1159,11 +1170,20 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                 self.gui.library_view.sort_by_named_field('marked', True)
                 
         self.gui.status_bar.show_message(_('Finished Adding/Updating %d books.'%(len(update_list) + len(add_list))), 3000)
-            
+
+        1+1
+        
         print("all done, remove temp dir.")
+
+        1+1
+        
         remove_dir(options['tdir'])
 
-        print("Count Pages:prefs:%s\nCP plugin available %s\nids:%s"%(prefs['countpagesstats'],'Count Pages' in self.gui.iactions,all_ids))
+        1+1
+        
+        print("removed temp dir.")
+
+        
         if 'Count Pages' in self.gui.iactions and len(prefs['countpagesstats']) and len(all_ids):
             cp_plugin = self.gui.iactions['Count Pages']
             cp_plugin.count_statistics(all_ids,prefs['countpagesstats'])
@@ -1371,8 +1391,8 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         oldmi = db.get_metadata(book_id,index_is_id=True)
         if prefs['keeptags']:
             old_tags = db.get_tags(book_id)
-            print("old_tags:%s"%old_tags)
-            print("mi.tags:%s"%mi.tags)
+            #print("old_tags:%s"%old_tags)
+            #print("mi.tags:%s"%mi.tags)
             # remove old Completed/In-Progress only if there's a new one.
             if 'Completed' in mi.tags or 'In-Progress' in mi.tags:
                 old_tags = filter( lambda x : x not in ('Completed', 'In-Progress'), old_tags)
@@ -1386,9 +1406,8 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
             for t in list(mi.tags) + list(old_tags):
                 foldedcase_tags[t.lower()] = t
 
-            mi.tags = foldedcase_tags.values()
-                
-            print("mi.tags:%s"%mi.tags)
+            mi.tags = foldedcase_tags.values()   
+            #print("mi.tags:%s"%mi.tags)
 
         if book['all_metadata']['langcode']:
             mi.languages=[book['all_metadata']['langcode']]
@@ -1421,12 +1440,14 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                             pass
 
         db.set_metadata(book_id,mi)
+        # mi.authors gets run through the string_to_authors and split on '&' ',' 'and' and 'with'
+        db.set_authors(book_id,book['author']) # author is a list.
 
         # do configured column updates here.
         #print("all_metadata: %s"%book['all_metadata'])
         custom_columns = self.gui.library_view.model().custom_columns
 
-        print("prefs['custom_cols'] %s"%prefs['custom_cols'])
+        #print("prefs['custom_cols'] %s"%prefs['custom_cols'])
         for col, meta in prefs['custom_cols'].iteritems():
             #print("setting %s to %s"%(col,meta))
             if col not in custom_columns:
@@ -1445,18 +1466,15 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                 continue
             label = coldef['label']
             if coldef['datatype'] in ('enumeration','text','comments','datetime','series'):
-                print("1 db.set_custom(%s,%s,%s)"%(book_id, book['all_metadata'][meta], label))
                 db.set_custom(book_id, book['all_metadata'][meta], label, commit=False)
             elif coldef['datatype'] in ('int','float'):
                 num = unicode(book['all_metadata'][meta]).replace(",","")
-                print("2 db.set_custom(%s,%s,%s)"%(book_id, num, label))
                 db.set_custom(book_id, num, label=label, commit=False)
             elif coldef['datatype'] == 'bool' and meta.startswith('status-'):
                 if meta == 'status-C':
                     val = book['all_metadata']['status'] == 'Completed'
                 if meta == 'status-I':
                     val = book['all_metadata']['status'] == 'In-Progress'
-                print("3 db.set_custom(%s,%s,%s)"%(book_id, val, label))
                 db.set_custom(book_id, val, label=label, commit=False)
 
         configuration = None
@@ -1471,14 +1489,11 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                     if "," in custcol:
                         (custcol,flag) = map( lambda x: x.strip(), custcol.split(",") )
 
-                    print("meta:(%s) => custcol:(%s), flag(%s) "%(meta,custcol,flag))
-                    
                     if meta not in book['all_metadata']:
                         print("No value for %s, skipping custom column(%s) update."%(meta,custcol))
                         continue
                     
                     if custcol not in custom_columns:
-                        print("No custom column(%s), skipping."%(custcol))
                         continue
                     else:
                         coldef = custom_columns[custcol]
@@ -1493,7 +1508,6 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                                 val = unicode(book['all_metadata'][meta]).replace(",","")
                         else:
                             val = book['all_metadata'][meta]
-                        print("4 db.set_custom(%s,%s,%s)"%(book_id, val, label))
                         db.set_custom(book_id, val, label=label, commit=False)
 
                     if flag == 'a':
@@ -1510,22 +1524,33 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                         if book['all_metadata'][meta]:
                             vallist = [book['all_metadata'][meta]]
                             
-                        print("5 db.set_custom(%s,%s,%s)"%(book_id, ", ".join(vallist), label))
                         db.set_custom(book_id, ", ".join(vallist), label=label, commit=False)
 
         # set author link if found.  All current adapters have authorUrl, except anonymous on AO3.
         # Moved down so author's already in the DB.
         if 'authorUrl' in book['all_metadata']:
             authurls = book['all_metadata']['authorUrl'].split(", ")
-            for i, auth in enumerate(book['author']):
-                #print("===Update author url for %s to %s"%(auth,authurls[i]))
-                autid=db.get_author_id(auth)
-                db.set_link_field_for_author(autid, unicode(authurls[i]),
-                                             commit=False, notify=False)
+            authorlist = [ a.replace('&',';') for a in book['author'] ]
+            if hasattr(db, 'new_api'): # new_api starts in calibre 1.0.0
+                authorids = db.new_api.get_item_ids('authors',authorlist)
+                authordata = db.new_api.author_data(authorids.values())
+                # print("\n\nauthorids:%s"%authorids)
+                # print("authordata:%s"%authordata)
 
-        print("About to call db.commit()")
+                author_id_to_link_map = dict()
+                for i, author in enumerate(authorlist):
+                    author_id_to_link_map[authorids[author]] = authurls[i]
+
+                # print("author_id_to_link_map:%s\n\n"%author_id_to_link_map)
+                db.new_api.set_link_for_authors(author_id_to_link_map)
+            else:
+                # keep for pre-calibre 1.0.0
+                for i, auth in enumerate(authorlist):
+                    #print("===Update author url for %s to %s"%(auth,authurls[i]))
+                    autid=db.get_author_id(auth)
+                    db.set_link_field_for_author(autid, unicode(authurls[i]),
+                                             commit=False, notify=False)
         db.commit()
-        print("After call db.commit()")
 
         if 'Generate Cover' in self.gui.iactions and (book['added'] or not prefs['gcnewonly']):
             
@@ -1917,3 +1942,17 @@ def split_text_to_urls(urls):
 def escapehtml(txt):
     return txt.replace("&","&amp;").replace(">","&gt;").replace("<","&lt;")
         
+def pretty_book(d, indent=0, spacer='     '):
+    kindent = spacer * indent
+
+    # if isinstance(d, list):
+    #     return '\n'.join([(pretty_book(v, indent, spacer)) for v in d])
+
+    if isinstance(d, dict):
+        for k in ('password','username'):
+            if k in d and d[k]:
+                d[k]='<was set, removed for security>'
+        return '\n'.join(['%s%s:\n%s' % (kindent, k, pretty_book(v, indent + 1, spacer)) 
+                          for k, v in d.items()])
+    return "%s%s"%(kindent, d)
+

defaults.ini

@@ -16,7 +16,22 @@
 [defaults]
 
 ## [defaults] section applies to all formats and sites but may be
-## overridden at several levels
+## overridden at several levels.  Example:
+
+## [defaults]
+## titlepage_entries: category,genre, status
+## [www.whofic.com]
+## # overrides defaults.
+## titlepage_entries: category,genre, status,dateUpdated,rating
+## [epub]
+## # overrides defaults & site section
+## titlepage_entries: category,genre, status,datePublished,dateUpdated,dateCreated
+## [www.whofic.com:epub]
+## # overrides defaults, site section & format section
+## titlepage_entries: category,genre, status,datePublished
+## [overrides]
+## # overrides all other sections
+## titlepage_entries: category
 
 ## Some sites also require the user to confirm they are adult for
 ## adult content.  Uncomment by removing '#' in front of is_adult.
@@ -152,6 +167,9 @@ extratags: FanFiction
 #extraships:
 #extrawarnings:
 
+## Add this to genre if there's more than one category.
+#add_genre_when_multi_category: Crossover
+
 ## number of seconds to sleep between calls to the story site.  May by
 ## useful if pulling large numbers of stories or if the site is slow.
 #slow_down_sleep_time:0.5
@@ -259,6 +277,8 @@ sort_ships:false
 ## you added calibre_author: keep_in_order_calibre_author:true
 #keep_in_order_author:true
 
+## User-agent
+#user_agent:FFDL/1.7
 
 ## Each output format has a section that overrides [defaults]
 [html]
@@ -447,8 +467,8 @@ nook_img_fix:true
 ## mobi TOC cannot be turned off right now.
 #include_tocpage: true
 
-## Each site has a section that overrides [defaults] *and* the format
-## sections test1.com specifically is not a real story site.  Instead,
+## Each site has a section that overrides [defaults].
+## test1.com specifically is not a real story site.  Instead,
 ## it is a fake site for testing configuration and output.  It uses
 ## URLs like: http://test1.com?sid=12345
 [test1.com]
@@ -505,7 +525,8 @@ extratags: FanFiction,Testing,HTML
 #is_adult:true
 
 ## AO3 adapter defines a few extra metadata entries.
-extra_valid_entries:fandoms,freeformtags,freefromtags,ao3categories,comments,kudos,hits,bookmarks,collections
+## If there's ever more than 4 series, add series04,series04Url etc.
+extra_valid_entries:fandoms,freeformtags,freefromtags,ao3categories,comments,kudos,hits,bookmarks,collections,series00,series01,series02,series03,series00Url,series01Url,series02Url,series03Url
 fandoms_label:Fandoms
 freeformtags_label:Freeform Tags
 freefromtags_label:Freeform Tags
@@ -521,7 +542,7 @@ bookmarks_label:Bookmarks
 include_in_freefromtags:freeformtags
 
 ## adds to titlepage_entries instead of replacing it.
-#extra_titlepage_entries: fandoms,freeformtags,ao3categories,comments,kudos,hits,bookmarks
+#extra_titlepage_entries: fandoms,freeformtags,ao3categories,comments,kudos,hits,bookmarks,series00,series01,series02,series03,series00Url,series01Url,series02Url,series03Url
 
 ## adds to include_subject_tags instead of replacing it.
 #extra_subject_tags:fandoms,freeformtags,ao3categories

fanficdownloader/adapters/adapter_archiveofourownorg.py

@@ -210,10 +210,7 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
         if a != None:
             warnings = a.findAll('a',{'class':"tag"})
             for warning in warnings:
-                if warning.string == "Author Chose Not To Use Archive Warnings":
-                    warning.string = "No Archive Warnings Apply"
-                if warning.string != "No Archive Warnings Apply":
-                    self.story.addToList('warnings',warning.string)
+                self.story.addToList('warnings',warning.string)
 		
         a = metasoup.find('dd',{'class':"freeform tags"})
         if a != None:
@@ -288,19 +285,19 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
                 self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
 
 		
-        try:
-            # Find Series name from series URL.
-            a = metasoup.find('dd',{'class':"series"})
-            b = a.find('a', href=re.compile(r"/series/\d+"))
-            series_name = b.string
-            series_url = 'http://'+self.host+b['href']
-            series_index = int(a.text.split(' ')[1])
-            self.setSeries(series_name, series_index)
-            self.story.setMetadata('seriesUrl',series_url)
-            
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
+        # Find Series name from series URL.
+        ddseries = metasoup.find('dd',{'class':"series"})
+
+        if ddseries:
+            for i, a in enumerate(ddseries.findAll('a', href=re.compile(r"/series/\d+"))):
+                series_name = stripHTML(a)
+                series_url = 'http://'+self.host+a['href']
+                series_index = int(stripHTML(a.previousSibling).replace(', ','').split(' ')[1]) # "Part # of" or ", Part #"
+                self.story.setMetadata('series%02d'%i,"%s [%s]"%(series_name,series_index))
+                self.story.setMetadata('series%02dUrl'%i,series_url)
+                if i == 0:
+                    self.setSeries(series_name, series_index)
+                    self.story.setMetadata('seriesUrl',series_url)
 
     # grab the text for an individual chapter.
     def getChapterText(self, url):

fanficdownloader/adapters/adapter_fanfictionnet.py

@@ -199,7 +199,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         for g in genrelist:
             #logger.debug("g:(%s)"%g)
             if g.strip() not in ffnetgenres:
-                logger.info("g not in ffnetgenres")
+                #logger.info("g not in ffnetgenres")
                 goodgenres=False
         if goodgenres:
             self.story.extendList('genre',genrelist)
@@ -266,11 +266,11 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         return
 
     def getChapterText(self, url):
-        logger.debug('Getting chapter text from: %s' % url)
-        time.sleep(0.5) ## ffnet(and, I assume, fpcom) tends to fail
+        time.sleep(5.0) ## ffnet(and, I assume, fpcom) tends to fail
                         ## more if hit too fast.  This is in
                         ## additional to what ever the
                         ## slow_down_sleep_time setting is.
+        logger.debug('Getting chapter text from: %s' % url)
         data = self._fetchUrl(url)
 
         if "Please email this error message in full to <a href='mailto:support@fanfiction.com'>support@fanfiction.com</a>" in data:

fanficdownloader/adapters/adapter_harrypotterfanfictioncom.py

@@ -131,55 +131,55 @@ class HarryPotterFanFictionComSiteAdapter(BaseSiteAdapter):
 
         ## Finding the metadata is a bit of a pain.  Most of the meta
         ## data is in a center.table without a bgcolor.
-        for center in soup.findAll('center'):
-            table = center.find('table',{'bgcolor':None})
-            if table:
-                metastr = stripHTML(str(table)).replace('\n',' ').replace('\t',' ')
-                # Rating: 12+ Story Reviews: 3
-                # Chapters: 3
-                # Characters: Andromeda, Ted, Bellatrix, R. Lestrange, Lucius, Narcissa, OC
-                # Genre(s): Fluff, Romance, Young Adult Era: OtherPairings: Other Pairing, Lucius/Narcissa
-                # Status: Completed
-                # First Published: 2010.09.02
-                # Last Published Chapter: 2010.09.28
-                # Last Updated: 2010.09.28
-                # Favorite Story Of: 1 users
-                # Warnings: Scenes of a Mild Sexual Nature
+        #for center in soup.findAll('center'):
+        table = soup.find('table',{'class':'storymaininfo'})
+        if table:
+            metastr = stripHTML(str(table)).replace('\n',' ').replace('\t',' ')
+            # Rating: 12+ Story Reviews: 3
+            # Chapters: 3
+            # Characters: Andromeda, Ted, Bellatrix, R. Lestrange, Lucius, Narcissa, OC
+            # Genre(s): Fluff, Romance, Young Adult Era: OtherPairings: Other Pairing, Lucius/Narcissa
+            # Status: Completed
+            # First Published: 2010.09.02
+            # Last Published Chapter: 2010.09.28
+            # Last Updated: 2010.09.28
+            # Favorite Story Of: 1 users
+            # Warnings: Scenes of a Mild Sexual Nature
 
-                m = re.match(r".*?Status: Completed.*?",metastr)
-                if m:
-                    self.story.setMetadata('status','Completed')
-                else:
-                    self.story.setMetadata('status','In-Progress')
+            m = re.match(r".*?Status: Completed.*?",metastr)
+            if m:
+                self.story.setMetadata('status','Completed')
+            else:
+                self.story.setMetadata('status','In-Progress')
 
-                m = re.match(r".*?Rating: (.+?) Story Reviews.*?",metastr)
-                if m:
-                    self.story.setMetadata('rating', m.group(1))
+            m = re.match(r".*?Rating: (.+?) Story Reviews.*?",metastr)
+            if m:
+                self.story.setMetadata('rating', m.group(1))
 
-                m = re.match(r".*?Genre\(s\): (.+?) Era.*?",metastr)
-                if m:
-                    for g in m.group(1).split(','):
-                        self.story.addToList('genre',g)
-        
-                m = re.match(r".*?Characters: (.+?) Genre.*?",metastr)
-                if m:
-                    for g in m.group(1).split(','):
-                        self.story.addToList('characters',g)
-        
-                m = re.match(r".*?Warnings: (.+).*?",metastr)
-                if m:
-                    for w in m.group(1).split(','):
-                        if w != 'Now Warnings':
-                            self.story.addToList('warnings',w)
-        
-                m = re.match(r".*?First Published: ([0-9\.]+).*?",metastr)
-                if m:
-                    self.story.setMetadata('datePublished',makeDate(m.group(1), "%Y.%m.%d"))
+            m = re.match(r".*?Genre\(s\): (.+?) Era.*?",metastr)
+            if m:
+                for g in m.group(1).split(','):
+                    self.story.addToList('genre',g)
+    
+            m = re.match(r".*?Characters: (.+?) Genre.*?",metastr)
+            if m:
+                for g in m.group(1).split(','):
+                    self.story.addToList('characters',g)
+    
+            m = re.match(r".*?Warnings: (.+).*?",metastr)
+            if m:
+                for w in m.group(1).split(','):
+                    if w != 'Now Warnings':
+                        self.story.addToList('warnings',w)
+    
+            m = re.match(r".*?First Published: ([0-9\.]+).*?",metastr)
+            if m:
+                self.story.setMetadata('datePublished',makeDate(m.group(1), "%Y.%m.%d"))
 
-                # Updated can have more than one space after it. <shrug>
-                m = re.match(r".*?Last Updated: ([0-9\.]+).*?",metastr)
-                if m:
-                    self.story.setMetadata('dateUpdated',makeDate(m.group(1), "%Y.%m.%d"))
+            # Updated can have more than one space after it. <shrug>
+            m = re.match(r".*?Last Updated: ([0-9\.]+).*?",metastr)
+            if m:
+                self.story.setMetadata('dateUpdated',makeDate(m.group(1), "%Y.%m.%d"))
 
     def getChapterText(self, url):
 

fanficdownloader/adapters/adapter_hpfanficarchivecom.py

@@ -37,11 +37,6 @@ class HPFanficArchiveComAdapter(BaseSiteAdapter):
     def __init__(self, config, url):
         BaseSiteAdapter.__init__(self, config, url)
 
-        # hpfanficarchive.com blocks the default user-agent.  However,
-        # when asked, they said it was just general anti-spam, not
-        # targeted at us.  That lets me do this in good conscience:
-        self.opener.addheaders = [('User-agent', 'FFDL/1.7')]
-        
         self.decode = ["Windows-1252",
                        "utf8"] # 1252 is a superset of iso-8859-1.
                                # Most sites that claim to be

fanficdownloader/adapters/adapter_ncisfictionnet.py

@@ -37,11 +37,6 @@ class NCISFictionNetAdapter(BaseSiteAdapter):
     def __init__(self, config, url):
         BaseSiteAdapter.__init__(self, config, url)
 
-        # ncisfiction.net blocks the default user-agent.  However,
-        # when asked, they said it was just general anti-spam, not
-        # targeted at us.  That lets me do this in good conscience:
-        self.opener.addheaders = [('User-agent', 'FFDL/1.7')]
-        
         self.decode = ["iso-8859-1",
                        "Windows-1252"] # 1252 is a superset of iso-8859-1.
                                # Most sites that claim to be

fanficdownloader/adapters/base_adapter.py

@@ -77,6 +77,8 @@ class BaseSiteAdapter(Configurable):
         self.is_adult=False
         
         self.opener = u2.build_opener(u2.HTTPCookieProcessor(),GZipProcessor())
+        ## Specific UA because too many sites are blocking the default python UA.
+        self.opener.addheaders = [('User-agent', self.getConfig('user_agent'))]
         self.storyDone = False
         self.metadataDone = False
         self.story = Story(configuration)

fanficdownloader/story.py

@@ -411,6 +411,9 @@ class Story(Configurable):
         if not value in self.metadata[listname]:
             self.metadata[listname].append(value)
 
+        if listname == 'category' and self.getConfig('add_genre_when_multi_category') and len(self.metadata[listname]) > 1:
+            self.addToList('genre',self.getConfig('add_genre_when_multi_category'))
+
     def isList(self,listname):
         'Everything set with an include_in_* is considered a list.'
         return self.hasConfig("include_in_"+listname) or \

index.html

@@ -53,13 +53,26 @@
 	    <p>Hi, {{ nickname }}! This is FanFictionDownLoader, which makes reading stories from various websites
 	      much easier. </p>
 	  </div>
-	  <!-- put announcements here, h3 is a good title size. -->
+	  <!-- put announcements here, h3 is a good title size.
 	  <h3>Changes:</h3>
 	  <p>
 	    <ul>
-	      <li>Minor optimizations.</li>
+              <li>Better doc section override order in ini files.</li>
+              <li>Additional series as site specific data for AO3.</li>
+              <li>Fix for AO3 stories without series.</li>
+              <li>Fixes for changes to harrypotterfanfictioncom.</li>
+              <li>Add User-agent="FFDL/1.7" for all adapters for fanfiction.net changes.<br>
+                (Remove from specific adapters.)</li>
 	    </ul>
 	  </p>
+           -->
+          <h3>fanfiction.net</h3>
+          <p>
+            Fanfiction.net appears to be blocking access from Google
+            App Engine IPs, which prevents this web service.  There's
+            nothing I can do about it.  At the time of writing, the
+            latest CLI and calibre plugin versions worked.
+          </p>
 	  
 	  <p>
 	    Questions?  Check out our 

plugin-defaults.ini

@@ -16,7 +16,22 @@
 [defaults]
 
 ## [defaults] section applies to all formats and sites but may be
-## overridden at several levels
+## overridden at several levels.  Example:
+
+## [defaults]
+## titlepage_entries: category,genre, status
+## [www.whofic.com]
+## # overrides defaults.
+## titlepage_entries: category,genre, status,dateUpdated,rating
+## [epub]
+## # overrides defaults & site section
+## titlepage_entries: category,genre, status,datePublished,dateUpdated,dateCreated
+## [www.whofic.com:epub]
+## # overrides defaults, site section & format section
+## titlepage_entries: category,genre, status,datePublished
+## [overrides]
+## # overrides all other sections
+## titlepage_entries: category
 
 ## Some sites also require the user to confirm they are adult for
 ## adult content.  Uncomment by removing '#' in front of is_adult.
@@ -124,6 +139,9 @@ extratags: FanFiction
 #extraships:
 #extrawarnings:
 
+## Add this to genre if there's more than one category.
+#add_genre_when_multi_category: Crossover
+
 ## number of seconds to sleep between calls to the story site.  May by
 ## useful if pulling large numbers of stories or if the site is slow.
 #slow_down_sleep_time:0.5
@@ -246,6 +264,9 @@ sort_ships:false
 ## you added calibre_author: keep_in_order_calibre_author:true
 #keep_in_order_author:true
 
+## User-agent
+#user_agent:FFDL/1.7
+
 
 ## Each output format has a section that overrides [defaults]
 [html]
@@ -416,8 +437,8 @@ nook_img_fix:true
 ## mobi TOC cannot be turned off right now.
 #include_tocpage: true
 
-## Each site has a section that overrides [defaults] *and* the format
-## sections test1.com specifically is not a real story site.  Instead,
+## Each site has a section that overrides [defaults].
+## test1.com specifically is not a real story site.  Instead,
 ## it is a fake site for testing configuration and output.  It uses
 ## URLs like: http://test1.com?sid=12345
 [test1.com]
@@ -474,7 +495,8 @@ extratags: FanFiction,Testing,HTML
 #is_adult:true
 
 ## AO3 adapter defines a few extra metadata entries.
-extra_valid_entries:fandoms,freeformtags,freefromtags,ao3categories,comments,kudos,hits,bookmarks,collections
+## If there's ever more than 4 series, add series04,series04Url etc.
+extra_valid_entries:fandoms,freeformtags,freefromtags,ao3categories,comments,kudos,hits,bookmarks,collections,series00,series01,series02,series03,series00Url,series01Url,series02Url,series03Url
 fandoms_label:Fandoms
 freeformtags_label:Freeform Tags
 freefromtags_label:Freeform Tags
@@ -490,7 +512,7 @@ bookmarks_label:Bookmarks
 include_in_freefromtags:freeformtags
 
 ## adds to titlepage_entries instead of replacing it.
-#extra_titlepage_entries: fandoms,freeformtags,ao3categories,comments,kudos,hits,bookmarks
+#extra_titlepage_entries: fandoms,freeformtags,ao3categories,comments,kudos,hits,bookmarks,series00,series01,series02,series03,series00Url,series01Url,series02Url,series03Url
 
 ## adds to include_subject_tags instead of replacing it.
 #extra_subject_tags:fandoms,freeformtags,ao3categories

plugin-example.ini

@@ -2,6 +2,24 @@
 ## like.  Uncomment options by removing the '#' in front of them.
 
 [defaults]
+## [defaults] section applies to all formats and sites but may be
+## overridden at several levels.  Example:
+
+## [defaults]
+## titlepage_entries: category,genre, status
+## [www.whofic.com]
+## # overrides defaults.
+## titlepage_entries: category,genre, status,dateUpdated,rating
+## [epub]
+## # overrides defaults & site section
+## titlepage_entries: category,genre, status,datePublished,dateUpdated,dateCreated
+## [www.whofic.com:epub]
+## # overrides defaults, site section & format section
+## titlepage_entries: category,genre, status,datePublished
+## [overrides]
+## # overrides all other sections
+## titlepage_entries: category
+
 ## Some sites also require the user to confirm they are adult for
 ## adult content.  Uncomment by removing '#' in front of is_adult.
 #is_adult:true
@@ -15,6 +33,9 @@
 #strip_chapter_numbers:true
 #add_chapter_numbers:true
 
+## Add this to genre if there's more than one category.
+#add_genre_when_multi_category: Crossover
+
 [epub]
 ## include images from img tags in the body and summary of stories.
 ## Images will be converted to jpg for size if possible.  Images work