Changes for ffnet--longer sleep, config UA, bump versions.

app.yaml

@@ -1,6 +1,6 @@
 # ffd-retief-hrd fanfictiondownloader
 application: fanfictiondownloader
-version: 4-4-70
+version: 4-4-71
 runtime: python27
 api_version: 1
 threadsafe: true

calibre-plugin/__init__.py

@@ -26,7 +26,7 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
     description         = 'UI plugin to download FanFiction stories from various sites.'
     supported_platforms = ['windows', 'osx', 'linux']
     author              = 'Jim Miller'
-    version             = (1, 7, 41)
+    version             = (1, 7, 42)
     minimum_calibre_version = (0, 8, 57)
 
     #: This field defines the GUI plugin class that contains all the code

defaults.ini

@@ -277,6 +277,8 @@ sort_ships:false
 ## you added calibre_author: keep_in_order_calibre_author:true
 #keep_in_order_author:true
 
+## User-agent
+#user_agent:FFDL/1.7
 
 ## Each output format has a section that overrides [defaults]
 [html]

fanficdownloader/adapters/adapter_fanfictionnet.py

@@ -199,7 +199,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         for g in genrelist:
             #logger.debug("g:(%s)"%g)
             if g.strip() not in ffnetgenres:
-                logger.info("g not in ffnetgenres")
+                #logger.info("g not in ffnetgenres")
                 goodgenres=False
         if goodgenres:
             self.story.extendList('genre',genrelist)
@@ -266,11 +266,11 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         return
 
     def getChapterText(self, url):
-        logger.debug('Getting chapter text from: %s' % url)
-        time.sleep(0.5) ## ffnet(and, I assume, fpcom) tends to fail
+        time.sleep(5.0) ## ffnet(and, I assume, fpcom) tends to fail
                         ## more if hit too fast.  This is in
                         ## additional to what ever the
                         ## slow_down_sleep_time setting is.
+        logger.debug('Getting chapter text from: %s' % url)
         data = self._fetchUrl(url)
 
         if "Please email this error message in full to <a href='mailto:support@fanfiction.com'>support@fanfiction.com</a>" in data:

fanficdownloader/adapters/base_adapter.py

@@ -78,7 +78,7 @@ class BaseSiteAdapter(Configurable):
         
         self.opener = u2.build_opener(u2.HTTPCookieProcessor(),GZipProcessor())
         ## Specific UA because too many sites are blocking the default python UA.
-        self.opener.addheaders = [('User-agent', 'FFDL/1.7')]
+        self.opener.addheaders = [('User-agent', self.getConfig('user_agent'))]
         self.storyDone = False
         self.metadataDone = False
         self.story = Story(configuration)

index.html

@@ -53,19 +53,19 @@
 	    <p>Hi, {{ nickname }}! This is FanFictionDownLoader, which makes reading stories from various websites
 	      much easier. </p>
 	  </div>
-	  <!-- put announcements here, h3 is a good title size. -->
+	  <!-- put announcements here, h3 is a good title size.
 	  <h3>Changes:</h3>
 	  <p>
 	    <ul>
               <li>Better doc section override order in ini files.</li>
-              <li>Fix for calibre pre-1.0.0.</li>
               <li>Additional series as site specific data for AO3.</li>
+              <li>Fix for AO3 stories without series.</li>
               <li>Fixes for changes to harrypotterfanfictioncom.</li>
               <li>Add User-agent="FFDL/1.7" for all adapters for fanfiction.net changes.<br>
                 (Remove from specific adapters.)</li>
-              <li>Fix for AO3 stories without series.</li>
 	    </ul>
 	  </p>
+           -->
           <h3>fanfiction.net</h3>
           <p>
             Fanfiction.net appears to be blocking access from Google
@@ -73,7 +73,6 @@
             nothing I can do about it.  At the time of writing, the
             latest CLI and calibre plugin versions worked.
           </p>
-              
 	  
 	  <p>
 	    Questions?  Check out our 

plugin-defaults.ini

@@ -264,6 +264,9 @@ sort_ships:false
 ## you added calibre_author: keep_in_order_calibre_author:true
 #keep_in_order_author:true
 
+## User-agent
+#user_agent:FFDL/1.7
+
 
 ## Each output format has a section that overrides [defaults]
 [html]