Bump versions.

app.yaml

@@ -1,6 +1,6 @@
 # ffd-retief-hrd fanfictiondownloader
 application: fanfictiondownloader
-version: 4-4-70
+version: 4-4-75
 runtime: python27
 api_version: 1
 threadsafe: true

calibre-plugin/__init__.py

@@ -26,7 +26,7 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
     description         = 'UI plugin to download FanFiction stories from various sites.'
     supported_platforms = ['windows', 'osx', 'linux']
     author              = 'Jim Miller'
-    version             = (1, 7, 41)
+    version             = (1, 7, 46)
     minimum_calibre_version = (0, 8, 57)
 
     #: This field defines the GUI plugin class that contains all the code

calibre-plugin/ffdl_plugin.py

@@ -1469,7 +1469,8 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                 db.set_custom(book_id, book['all_metadata'][meta], label, commit=False)
             elif coldef['datatype'] in ('int','float'):
                 num = unicode(book['all_metadata'][meta]).replace(",","")
-                db.set_custom(book_id, num, label=label, commit=False)
+                if num != '':
+                    db.set_custom(book_id, num, label=label, commit=False)
             elif coldef['datatype'] == 'bool' and meta.startswith('status-'):
                 if meta == 'status-C':
                     val = book['all_metadata']['status'] == 'Completed'
@@ -1508,7 +1509,8 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                                 val = unicode(book['all_metadata'][meta]).replace(",","")
                         else:
                             val = book['all_metadata'][meta]
-                        db.set_custom(book_id, val, label=label, commit=False)
+                        if val != '':
+                            db.set_custom(book_id, val, label=label, commit=False)
 
                     if flag == 'a':
                         vallist = []

calibre-plugin/jobs.py

@@ -198,7 +198,10 @@ def do_download_for_worker(book,options,notification=lambda x,y:x):
             # dup handling from ffdl_plugin needed for anthology updates.
             if chaptercount > urlchaptercount:
                 raise NotGoingToDownload("Existing epub contains %d chapters, web site only has %d. Use Overwrite to force update." % (chaptercount,urlchaptercount),'dialog_error.png')
-                            
+
+            if adapter.getConfig("do_update_hook"):
+                chaptercount = adapter.hookForUpdates(chaptercount)
+
             print("Do update - epub(%d) vs url(%d)" % (chaptercount, urlchaptercount))
             print("write to %s"%outfile)
 

defaults.ini

@@ -170,6 +170,14 @@ extratags: FanFiction
 ## Add this to genre if there's more than one category.
 #add_genre_when_multi_category: Crossover
 
+## default_value_(entry) can be used to set the value for a metadata
+## entry when no value has been found on the site.  For example, some
+## sites doesn't have a status metadatum.  If uncommented, this will
+## use 'Unknown' for status when no status is found.
+#default_value_status:Unknown
+## Can also be used for other metadata values
+#default_value_category:FanFiction
+
 ## number of seconds to sleep between calls to the story site.  May by
 ## useful if pulling large numbers of stories or if the site is slow.
 #slow_down_sleep_time:0.5
@@ -277,6 +285,8 @@ sort_ships:false
 ## you added calibre_author: keep_in_order_calibre_author:true
 #keep_in_order_author:true
 
+## User-agent
+user_agent:FFDL/1.7
 
 ## Each output format has a section that overrides [defaults]
 [html]
@@ -808,6 +818,10 @@ extracategories:Highlander
 ## Site dedicated to these categories/characters/ships
 extracategories:In Death
 
+## some sites include images that we don't ever want becoming the
+## cover image.  This lets you exclude them.
+cover_exclusion_regexp:/public/style_emoticons/.*
+
 [ksarchive.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:Star Trek
@@ -1115,6 +1129,7 @@ context_label:Context
 type_label:Type of Couple
 
 [www.fanfiction.net]
+user_agent:
 ## fanfiction.net's 'cover' images are really just tiny thumbnails.
 ## Change this to false to use them anyway.
 never_make_cover: true
@@ -1145,7 +1160,12 @@ extracategories:Harry Potter
 ## fictionally.org storyIds are not unique.  Combine with authorId.
 output_filename: ${title}-${siteabbrev}_${authorId}_${storyId}${formatext}
 
+## fictionalley.org doesn't have a status metadatum.  If uncommented,
+## this will be used for status.
+#default_value_status:Unknown
+
 [www.fictionpress.com]
+user_agent:
 ## Clear FanFiction from defaults, fictionpress.com is original fiction.
 extratags:
 
@@ -1167,12 +1187,13 @@ extracategories:My Little Pony: Friendship is Magic
 
 ## Extra metadata that this adapter knows about.  See [dramione.org]
 ## for examples of how to use them.
-extra_valid_entries:likes,dislikes,views,total_views,short_description
+extra_valid_entries:likes,dislikes,views,total_views,short_description,groups
 likes_label:Likes
 dislikes_label:Dislikes
 views_label:Highest Single Chapter Views
 total_views_label:Total Views
 short_description_label:Short Summary
+groups_label:Groups
 
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
@@ -1184,6 +1205,13 @@ short_description_label:Short Summary
 ## when a password is required rather than prompting every time.
 #fail_on_password: false
 
+## fimfiction.net stories allow chapters to be added out of order.  So
+## the newest chapter may not be the last one.  FFDL update doesn't
+## like that.  If do_update_hook is uncommented and set true, the
+## adapter will discard all existing chapters from the newest one on
+## when updating to enforce accurate chapters.
+#do_update_hook:false
+
 [www.harrypotterfanfiction.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter

downloader.py

@@ -270,7 +270,6 @@ def main(argv,
            elif chaptercount == 0:
                print "%s doesn't contain any recognizable chapters, probably from a different source.  Not updating." % (output_filename)
            else:
-               print "Do update - epub(%d) vs url(%d)" % (chaptercount, urlchaptercount)
                if not options.metaonly:
 
                    # update now handled by pre-populating the old
@@ -284,6 +283,11 @@ def main(argv,
                     adapter.calibrebookmark,
                     adapter.logfile) = get_update_data(output_filename)
 
+                   print "Do update - epub(%d) vs url(%d)" % (chaptercount, urlchaptercount)
+
+                   if adapter.getConfig("do_update_hook"):
+                       chaptercount = adapter.hookForUpdates(chaptercount)
+
                    writeStory(configuration,adapter,"epub")
                    
        else:

fanficdownloader/adapters/adapter_fanfictionnet.py

@@ -199,7 +199,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         for g in genrelist:
             #logger.debug("g:(%s)"%g)
             if g.strip() not in ffnetgenres:
-                logger.info("g not in ffnetgenres")
+                #logger.info("g not in ffnetgenres")
                 goodgenres=False
         if goodgenres:
             self.story.extendList('genre',genrelist)
@@ -266,11 +266,11 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         return
 
     def getChapterText(self, url):
-        logger.debug('Getting chapter text from: %s' % url)
-        time.sleep(0.5) ## ffnet(and, I assume, fpcom) tends to fail
+        time.sleep(5.0) ## ffnet(and, I assume, fpcom) tends to fail
                         ## more if hit too fast.  This is in
                         ## additional to what ever the
                         ## slow_down_sleep_time setting is.
+        logger.debug('Getting chapter text from: %s' % url)
         data = self._fetchUrl(url)
 
         if "Please email this error message in full to <a href='mailto:support@fanfiction.com'>support@fanfiction.com</a>" in data:

fanficdownloader/adapters/adapter_fimfictionnet.py

@@ -21,7 +21,6 @@ logger = logging.getLogger(__name__)
 import re
 import urllib2
 import cookielib as cl
-from datetime import datetime
 import json
 
 from .. import BeautifulSoup as bs
@@ -42,6 +41,10 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
         self._setURL("http://"+self.getSiteDomain()+"/story/"+self.story.getMetadata('storyId')+"/")
         self.is_adult = False
         
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%d %b %Y"
+            
     @staticmethod
     def getSiteDomain():
         return 'www.fimfiction.net'
@@ -93,8 +96,10 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
         if "Warning: mysql_fetch_array(): supplied argument is not a valid MySQL result resource" in data:
             raise exceptions.StoryDoesNotExist(self.url)
 
-        if "/images/missing_story.png" in data:
-            raise exceptions.StoryDoesNotExist(self.url)
+        # Can cause problems if a missing story is referenced in a comment.
+        # Shouldn't be needed anyway.
+        # if "/images/missing_story.png" in data:
+        #     raise exceptions.StoryDoesNotExist(self.url)
         
         if "This story has been marked as having adult content." in data:
             raise exceptions.AdultCheckRequired(self.url)
@@ -185,14 +190,42 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
         hrstr="<hr />"
         descdivstr = '<div class="description">'+descdivstr[descdivstr.index(hrstr)+len(hrstr):]
         self.setDescription(self.url,descdivstr)
-        
+
+        # Can't trust dates from API anymore I'm told.
         # Dates are in Unix time
         # Take the publish date from the first chapter posted
-        rawDatePublished = storyMetadata["chapters"][0]["date_modified"]
-        self.story.setMetadata("datePublished", datetime.fromtimestamp(rawDatePublished))
-        rawDateUpdated = storyMetadata["date_modified"]
-        self.story.setMetadata("dateUpdated", datetime.fromtimestamp(rawDateUpdated))
+        # rawDatePublished = storyMetadata["chapters"][0]["date_modified"]
+        # self.story.setMetadata("datePublished", datetime.fromtimestamp(rawDatePublished))
+        # rawDateUpdated = storyMetadata["date_modified"]
+        # self.story.setMetadata("dateUpdated", datetime.fromtimestamp(rawDateUpdated))
+        
+        oldestChapter = None
+        newestChapter = None
+        self.newestChapterNum = None # save for comparing during update.
+        # Scan all chapters to find the oldest and newest, on
+        # FiMFiction it's possible for authors to insert new chapters
+        # out-of-order or change the dates of earlier ones by editing
+        # them--That WILL break epub update.
+        for index, chapterDate in enumerate(soup.findAll('span', {'class':'date'})):
+            date=re.sub(r"(\d+)(st|nd|rd|th)",r"\1",chapterDate.contents[1].strip())
+            chapterDate = makeDate(date,self.dateformat)
+            if oldestChapter == None or chapterDate < oldestChapter:
+                oldestChapter = chapterDate
+            if newestChapter == None or chapterDate > newestChapter:
+                newestChapter = chapterDate
+                self.newestChapterNum = index
 
+        self.story.setMetadata("dateUpdated", newestChapter)
+        
+        pubdatetag = soup.find('span', {'class':'date_approved'})
+        if pubdatetag is None:
+            self.story.setMetadata("datePublished", oldestChapter)            
+        else:
+            pubdateraw = pubdatetag('span')[1].text
+            datestripped=re.sub(r"(\d+)(st|nd|rd|th)",r"\1",pubdateraw.strip())
+            pubDate = makeDate(datestripped,self.dateformat)
+            self.story.setMetadata("datePublished", pubDate)
+            
         chars = soup.find("div", {"class":"inner_data"})
         # fimfic stopped putting the char name on or around the char
         # icon now for some reason.  Pull it from the image name with
@@ -215,10 +248,22 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
                 if not isinstance(value,basestring):
                     value = unicode(value)
                 self.story.setMetadata(metakey, value)
-        
+
+        rawGroupList = soup.find('ul', {'id':'story_group_list'})
+        if rawGroupList is not None:
+            for groupName in rawGroupList.findAll('a', {'href':re.compile('^/group/')}):
+                self.story.addToList("groups",stripHTML(groupName))
             
+    def hookForUpdates(self,chaptercount):
+        if self.oldchapters and len(self.oldchapters) > self.newestChapterNum:
+            print("Existing epub has %s chapters\nNewest chapter is %s.  Discarding old chapters from there on."%(len(self.oldchapters), self.newestChapterNum+1))
+            self.oldchapters = self.oldchapters[:self.newestChapterNum]
+        return len(self.oldchapters)
+            
+        
     def getChapterText(self, url):
         logger.debug('Getting chapter text from: %s' % url)
+        
         soup = bs.BeautifulSoup(self._fetchUrl(url),selfClosingTags=('br','hr')).find('div', {'class' : 'chapter_content'})
         if soup == None:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)

fanficdownloader/adapters/adapter_indeathnet.py

@@ -46,8 +46,8 @@ class InDeathNetAdapter(BaseSiteAdapter):
         self.password = ""
         self.is_adult=False
         
-		
-		  # get storyId from url--url validation guarantees query correct
+        
+        # get storyId from url--url validation guarantees query correct
         m = re.match(self.getSiteURLPattern(),url)
         if m:
             self.story.setMetadata('storyId',m.group('id'))
@@ -87,6 +87,34 @@ class InDeathNetAdapter(BaseSiteAdapter):
         postdate = makeDate(d.group('day')+' '+ym.group('mon')+' '+ym.group('year'),self.dateformat)
         return postdate
 
+    def getAuthorData(self):
+        
+        mainUrl = self.url.replace("/archive","")
+        
+        try:
+            maindata = self._fetchUrl(mainUrl)
+            
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.meta)
+            else:
+                raise e
+
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        mainsoup = bs.BeautifulSoup(maindata)
+
+        # find first entry
+        e = mainsoup.find('div',{'class':"entry"})
+        
+        # get post author as author
+        d = e.find('div',{'class':"desc"})
+        a = d.find('strong')
+        self.story.setMetadata('author',a.contents[0].string.strip())
+        
+        # Don't seem to be able to get author pages anymore
+        self.story.setMetadata('authorUrl','http://www.indeath.net/')
+        self.story.setMetadata('authorId','0')
+        
     ## Getting the chapter list and the meta data, plus 'is adult' checking.
     def extractChapterUrlsAndMetadata(self):
 
@@ -110,17 +138,13 @@ class InDeathNetAdapter(BaseSiteAdapter):
         h = soup.find('a', id="blog_title")
         t = h.find('span')
         self.story.setMetadata('title',stripHTML(t.contents[0]).strip())
-		
+        
         s = t.find('div')        
         if s != None:
             self.setDescription(url,s)
-		
-        # Find authorid and URL from first link in Recent Entries (don't yet reference 'recent entries' - let's see if that is required)
-        a = soup.find('a', href=re.compile(r"http://www.indeath.net/user/\d+\-[a-z0-9]+/$"))		#http://www.indeath.net/user/9083-cyrex/
-        m = re.search('http://www.indeath.net/user/(?P<id>\d+)\-(?P<name>[a-z0-9]*)/$',a['href'])
-        self.story.setMetadata('authorId',m.group('id'))
-        self.story.setMetadata('authorUrl',a['href'])
-        self.story.setMetadata('author',m.group('name'))
+
+        # Get Author from main blog page since it's not reliably on the archive page
+        self.getAuthorData()
 
         # Find the chapters:
         chapters=soup.findAll('a', title="View entry", href=re.compile(r'http://www.indeath.net/blog/'+self.story.getMetadata('storyId')+"/entry\-(\d+)\-([^/]*)/$"))
@@ -149,21 +173,20 @@ class InDeathNetAdapter(BaseSiteAdapter):
             if len(chapters)==1:
                 self.chapterUrls.append((self.story.getMetadata('title'),chapter['href']))
             else:
-            	ct = stripHTML(chapter)
-            	tnew = re.match("(?i)"+self.story.getMetadata('title')+r" - (?P<newtitle>.*)$",ct)
-            	if tnew:
-            		chaptertitle = tnew.group('newtitle')
-            	else:
-            		chaptertitle = ct
+                ct = stripHTML(chapter)
+                tnew = re.match("(?i)"+self.story.getMetadata('title')+r" - (?P<newtitle>.*)$",ct)
+                if tnew:
+                    chaptertitle = tnew.group('newtitle')
+                else:
+                    chaptertitle = ct
                 self.chapterUrls.append((chaptertitle,chapter['href']))
-	
-		
+
         
         
     # grab the text for an individual chapter.
     def getChapterText(self, url):
         logger.debug('Getting chapter text from: %s' % url)
-		
+        
         #chapter=bs.BeautifulSoup('<div class="story"></div>')
         data = self._fetchUrl(url)
         soup = bs.BeautifulSoup(data,selfClosingTags=('br','hr','span','center'))

fanficdownloader/adapters/adapter_portkeyorg.py

@@ -251,8 +251,10 @@ class PortkeyOrgAdapter(BaseSiteAdapter): # XXX
 
         logger.debug('Getting chapter text from: %s' % url)
         data = self._fetchUrl(url)
-        soup = bs.BeautifulStoneSoup(data,
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+
+        data = data.replace("HTML>","div>")
+        
+        soup = bs.BeautifulSoup(data)
 
         #print("soup:%s"%soup)
         tag = soup.find('td', {'class' : 'story'})

fanficdownloader/adapters/adapter_tthfanficorg.py

@@ -231,10 +231,10 @@ class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter):
         BtVS = True
         BtVSNonX = False
         for cat in verticaltable.findAll('a', href=re.compile(r"^/Category-")):
-            if cat.string not in ['General', 'Non-BtVS/AtS Stories', 'BtVS/AtS Non-Crossover', 'Non-BtVS Crossovers']:
+            if cat.string not in ['General', 'Non-BtVS/AtS Stories', 'Non-BTVS/AtS Stories', 'BtVS/AtS Non-Crossover', 'Non-BtVS Crossovers']:
                 self.story.addToList('category',cat.string)
             else:
-                if 'Non-BtVS' in cat.string:
+                if 'Non-BtVS' in cat.string or 'Non-BTVS' in cat.string:
                     BtVS = False
                 if 'BtVS/AtS Non-Crossover' == cat.string:
                     BtVSNonX = True

fanficdownloader/adapters/adapter_whoficcom.py

@@ -22,6 +22,7 @@ import re
 import urllib2
 
 from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
 from .. import exceptions as exceptions
 
 from base_adapter import BaseSiteAdapter,  makeDate

fanficdownloader/adapters/base_adapter.py

@@ -78,7 +78,7 @@ class BaseSiteAdapter(Configurable):
         
         self.opener = u2.build_opener(u2.HTTPCookieProcessor(),GZipProcessor())
         ## Specific UA because too many sites are blocking the default python UA.
-        self.opener.addheaders = [('User-agent', 'FFDL/1.7')]
+        self.opener.addheaders = [('User-agent', self.getConfig('user_agent'))]
         self.storyDone = False
         self.metadataDone = False
         self.story = Story(configuration)
@@ -247,6 +247,10 @@ class BaseSiteAdapter(Configurable):
             self.metadataDone = True
         return self.story
 
+    def hookForUpdates(self,chaptercount):
+        "Usually not needed."
+        return chaptercount
+
     ###############################
     
     @staticmethod
@@ -379,7 +383,7 @@ fullmon = {"January":"01", "February":"02", "March":"03", "April":"04", "May":"0
            "June":"06","July":"07", "August":"08", "September":"09", "October":"10",
            "November":"11", "December":"12" }
 
-def makeDate(string,format):
+def makeDate(string,dateform):
     # Surprise!  Abstracting this turned out to be more useful than
     # just saving bytes.
 
@@ -388,10 +392,10 @@ def makeDate(string,format):
     # there's non-english content. -- ficbook.net now makes that a
     # lie.  It has to do something even more complicated to get
     # Russian month names correct everywhere.
-    do_abbrev = "%b" in format
+    do_abbrev = "%b" in dateform
         
-    if "%B" in format or do_abbrev:
-        format = format.replace("%B","%m").replace("%b","%m")
+    if "%B" in dateform or do_abbrev:
+        dateform = dateform.replace("%B","%m").replace("%b","%m")
         for (name,num) in fullmon.items():
             if do_abbrev:
                 name = name[:3] # first three for abbrev
@@ -399,5 +403,5 @@ def makeDate(string,format):
                 string = string.replace(name,num)
                 break
             
-    return datetime.datetime.strptime(string,format)
+    return datetime.datetime.strptime(string,dateform)
 

fanficdownloader/htmlcleanup.py

@@ -69,8 +69,9 @@ def removeEntities(text):
     
     if text is None:
         return ""
-    if not (isinstance(text,str) or isinstance(text,unicode)):
-        return str(text)
+    
+    if not isinstance(text,basestring):
+        return unicode(text)
     
     try:
         t = text.decode('utf-8')

fanficdownloader/story.py

@@ -349,6 +349,8 @@ class Story(Configurable):
                 return removeAllEntities(value)
             else:
                 return value
+        else: #if self.getConfig("default_value_"+key):
+            return self.getConfig("default_value_"+key)
         
     def getAllMetadata(self,
                        removeallentities=False,
@@ -485,6 +487,8 @@ class Story(Configurable):
 
         if None in subjectset:
             subjectset.remove(None)
+        if '' in subjectset:
+            subjectset.remove('')
 
         return list(subjectset | set(self.getConfigList("extratags")))
             

index.html

@@ -57,9 +57,18 @@
 	  <h3>Changes:</h3>
 	  <p>
 	    <ul>
-	      <li>Minor optimizations.</li>
+              <li>Fix for indeath.net--Thanks Besnef!</li>
+              <li>Better Non-BtVS/AtS detection for tthfanfic.org</li>
 	    </ul>
 	  </p>
+          
+          <h3>fanfiction.net</h3>
+          <p>
+            Fanfiction.net appears to be blocking access from Google
+            App Engine, which prevents this web service.  There's
+            nothing I can do about it.  At the time of writing, the
+            latest CLI and calibre plugin versions worked.
+          </p>
 	  
 	  <p>
 	    Questions?  Check out our 
@@ -69,7 +78,7 @@
 	    If you have any problems with this application, please
 	    report them in
 	    the <a href="http://groups.google.com/group/fanfic-downloader">FanFictionDownLoader Google Group</a>.  The
-	    <a href="http://4-4-67.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
+	    <a href="http://4-4-74.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
 	  </p>
 	  <div id='error'>
 	    {{ error_message }}

plugin-defaults.ini

@@ -142,6 +142,14 @@ extratags: FanFiction
 ## Add this to genre if there's more than one category.
 #add_genre_when_multi_category: Crossover
 
+## default_value_(entry) can be used to set the value for a metadata
+## entry when no value has been found on the site.  For example, some
+## sites doesn't have a status metadatum.  If uncommented, this will
+## use 'Unknown' for status when no status is found.
+#default_value_status:Unknown
+## Can also be used for other metadata values
+#default_value_category:FanFiction
+
 ## number of seconds to sleep between calls to the story site.  May by
 ## useful if pulling large numbers of stories or if the site is slow.
 #slow_down_sleep_time:0.5
@@ -264,6 +272,8 @@ sort_ships:false
 ## you added calibre_author: keep_in_order_calibre_author:true
 #keep_in_order_author:true
 
+## User-agent
+user_agent:FFDL/1.7
 
 ## Each output format has a section that overrides [defaults]
 [html]
@@ -793,6 +803,10 @@ extracategories:Highlander
 ## Site dedicated to these categories/characters/ships
 extracategories:In Death
 
+## some sites include images that we don't ever want becoming the
+## cover image.  This lets you exclude them.
+cover_exclusion_regexp:/public/style_emoticons/.*
+
 [ksarchive.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:Star Trek
@@ -1100,6 +1114,7 @@ context_label:Context
 type_label:Type of Couple
 
 [www.fanfiction.net]
+user_agent:
 ## fanfiction.net's 'cover' images are really just tiny thumbnails.
 ## Change this to false to use them anyway.
 never_make_cover: true
@@ -1127,7 +1142,12 @@ extracategories:Harry Potter
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+## fictionalley.org doesn't have a status metadatum.  If uncommented,
+## this will be used for status.
+#default_value_status:Unknown
+
 [www.fictionpress.com]
+user_agent:
 ## Clear FanFiction from defaults, fictionpress.com is original fiction.
 extratags:
 
@@ -1149,12 +1169,13 @@ extracategories:My Little Pony: Friendship is Magic
 
 ## Extra metadata that this adapter knows about.  See [dramione.org]
 ## for examples of how to use them.
-extra_valid_entries:likes,dislikes,views,total_views,short_description
+extra_valid_entries:likes,dislikes,views,total_views,short_description,groups
 likes_label:Likes
 dislikes_label:Dislikes
 views_label:Highest Single Chapter Views
 total_views_label:Total Views
 short_description_label:Short Summary
+groups_label:Groups
 
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
@@ -1166,6 +1187,13 @@ short_description_label:Short Summary
 ## when a password is required rather than prompting every time.
 #fail_on_password: false
 
+## fimfiction.net stories allow chapters to be added out of order.  So
+## the newest chapter may not be the last one.  FFDL update doesn't
+## like that.  If do_update_hook is uncommented and set true, the
+## adapter will discard all existing chapters from the newest one on
+## when updating to enforce accurate chapters.
+#do_update_hook:false
+
 [www.harrypotterfanfiction.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter

status.html

@@ -51,7 +51,14 @@
 	      by {{ fic.author }} ({{ fic.format }})
 	  {% endif %}
 	  {% if fic.failure %}
-	      <span id='error'>{{ fic.failure }}</span>
+          <h3>fanfiction.net</h3>
+          <p>
+            FYI, fanfiction.net appears to be blocking access from Google
+            App Engine, which prevents this web service.  There's
+            nothing I can do about it.  At the time of writing, the
+            latest CLI and calibre plugin versions worked.
+          </p>
+	  <span id='error'>{{ fic.failure }}</span>
 	  {% endif %}
 	  {% if  not fic.completed and not fic.failure %}
 	      <p>Not done yet.  This page will periodically poll to see if your story has finished.</p>