Update CLI download zip.

app.yaml

@@ -1,6 +1,6 @@
 # ffd-retief-hrd fanfictiondownloader
 application: fanfictiondownloader
-version: 2-0-06
+version: 2-0-08
 runtime: python27
 api_version: 1
 threadsafe: true

calibre-plugin/__init__.py

@@ -42,7 +42,7 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
     description         = _('UI plugin to download FanFiction stories from various sites.')
     supported_platforms = ['windows', 'osx', 'linux']
     author              = 'Jim Miller'
-    version             = (2, 0, 6)
+    version             = (2, 0, 8)
     minimum_calibre_version = (1, 48, 0)
 
     #: This field defines the GUI plugin class that contains all the code

defaults.ini

@@ -321,7 +321,7 @@ sort_ships:false
 #keep_in_order_author:true
 
 ## User-agent
-user_agent:FFDL/1.7
+user_agent:FFDL/2.0
 
 ## Each output format has a section that overrides [defaults]
 [html]

fanficdownloader/adapters/adapter_archiveofourownorg.py

@@ -167,7 +167,11 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
             
         # use BeautifulSoup HTML parser to make everything easier to find.
         soup = bs.BeautifulSoup(data)
+        for tag in soup.findAll('div',id='admin-banner'):
+            tag.extract()
         metasoup = bs.BeautifulSoup(meta)
+        for tag in metasoup.findAll('div',id='admin-banner'):
+            tag.extract()
 
         # Now go hunting for all the meta data and the chapter list.
         

fanficdownloader/adapters/adapter_ficwadcom.py

@@ -65,7 +65,7 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
         loginUrl = 'http://' + self.getSiteDomain() + '/account/login'
         logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['username']))
-        d = self._postUrl(loginUrl,params)
+        d = self._postUrl(loginUrl,params,usecache=False)
 
         if "Login attempt failed..." in d:
             logger.info("Failed to login to URL %s as %s" % (loginUrl,
@@ -75,6 +75,13 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
         else:
             return True        
     
+    def use_pagecache(self):
+        '''
+        adapters that will work with the page cache need to implement
+        this and change it to True.
+        '''
+        return True
+
     def extractChapterUrlsAndMetadata(self):
 
         # fetch the chapter.  From that we will get almost all the
@@ -96,9 +103,9 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
             else:
                 raise e
             
-        h3 = soup.find('h3')
-        storya = h3.find('a',href=re.compile("^/story/\d+$"))
-        if storya : # if there's a story link in the h3 header, this is a chapter page.
+        divstory = soup.find('div',id='story')
+        storya = divstory.find('a',href=re.compile("^/story/\d+$"))
+        if storya : # if there's a story link in the divstory header, this is a chapter page.
             # normalize story URL on chapter list.
             self.story.setMetadata('storyId',storya['href'].split('/',)[2])
             url = "http://"+self.getSiteDomain()+storya['href']
@@ -113,17 +120,17 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
                     raise e
 
         # if blocked, attempt login.
-        if soup.find("li",{"class":"blocked"}):
+        if soup.find("div",{"class":"blocked"}):
             if self.performLogin(url): # performLogin raises
                                        # FailedToLogin if it fails.
-                soup = bs.BeautifulSoup(self._fetchUrl(url))
+                soup = bs.BeautifulSoup(self._fetchUrl(url,usecache=False))
 
         # title - first h4 tag will be title.
-        titleh4 = soup.find('h4')
+        titleh4 = soup.find('div',{'class':'storylist'}).find('h4')
         self.story.setMetadata('title', stripHTML(titleh4.a))
 
         # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"^/author/\d+"))
+        a = soup.find('span',{'class':'author'}).find('a', href=re.compile(r"^/author/\d+"))
         self.story.setMetadata('authorId',a['href'].split('/')[2])
         self.story.setMetadata('authorUrl','http://'+self.host+a['href'])
         self.story.setMetadata('author',a.string)
@@ -139,7 +146,7 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
 
         # warnings
         # <span class="req"><a href="/help/38" title="Medium Spoilers">[!!] </a> <a href="/help/38" title="Rape/Sexual Violence">[R] </a> <a href="/help/38" title="Violence">[V] </a> <a href="/help/38" title="Child/Underage Sex">[Y] </a></span>
-        spanreq = metap.find("span",{"class":"req"})
+        spanreq = metap.find("span",{"class":"story-warnings"})
         if spanreq: # can be no warnings.
             for a in spanreq.findAll("a"):
                 self.story.addToList('warnings',a['title'])
@@ -165,16 +172,16 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
                 if g:
                     self.story.addToList('characters',g)
         
-        m = re.match(r".*?Published: ([0-9/]+?) -.*?",metastr)
+        m = re.match(r".*?Published: ([0-9-]+?) -.*?",metastr)
         if m:
-            self.story.setMetadata('datePublished',makeDate(m.group(1), "%Y/%m/%d"))
+            self.story.setMetadata('datePublished',makeDate(m.group(1), "%Y-%m-%d"))
 
         # Updated can have more than one space after it. <shrug>
-        m = re.match(r".*?Updated: ([0-9/]+?) +-.*?",metastr) 
+        m = re.match(r".*?Updated: ([0-9-]+?) +-.*?",metastr) 
         if m:
-            self.story.setMetadata('dateUpdated',makeDate(m.group(1), "%Y/%m/%d"))
+            self.story.setMetadata('dateUpdated',makeDate(m.group(1), "%Y-%m-%d"))
 
-        m = re.match(r".*? - ([0-9/]+?) words.*?",metastr)
+        m = re.match(r".*? - ([0-9,]+?) words.*?",metastr)
         if m:
             self.story.setMetadata('numWords',m.group(1))
 
@@ -185,7 +192,7 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
 
         # get the chapter list first this time because that's how we
         # detect the need to login.
-        storylistul = soup.find('ul',{'id':'storylist'})
+        storylistul = soup.find('ul',{'class':'storylist'})
         if not storylistul:
             # no list found, so it's a one-chapter story.
             self.chapterUrls.append((self.story.getMetadata('title'),url))

fanficdownloader/adapters/adapter_fimfictionnet.py

@@ -85,7 +85,7 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
             self.cookiejar.set_cookie(cookie)
         
         try:
-            apiResponse = urllib2.urlopen("http://www.fimfiction.net/api/story.php?story=%s" % (self.story.getMetadata("storyId"))).read()
+            apiResponse = self._fetchUrl("http://www.fimfiction.net/api/story.php?story=%s" % (self.story.getMetadata("storyId")))
             apiData = json.loads(apiResponse)
             
             # Unfortunately, we still need to load the story index
@@ -114,7 +114,7 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
             params['password'] = self.password
             data = self._postUrl(self.url,params)
 
-        if "Enter the password the author set for this story to view it." in data:
+        if not (soup.find('form', {'id' : 'password_form'}) == None):
             if self.getConfig('fail_on_password'):
                 raise exceptions.FailedToDownload("%s requires story password and fail_on_password is true."%self.url)
             else:
@@ -164,14 +164,10 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
         self.story.setMetadata("rating", storyMetadata["content_rating_text"])
 
         ## Warnings aren't included in the API.
-        bottomli = soup.find('li',{'class':'bottom'})
-        if bottomli:
-            bottomspans = bottomli.findAll('span')
-            # the first span in bottom is the rating, obtained above.
-            if bottomspans and len(bottomspans) > 1:
-                for warning in bottomspans[1:]:
-                    self.story.addToList('warnings',warning.string)
-                
+        if soup.find('a',{'class':'story_category story_category_gore'}):
+            self.story.addToList('warnings',"Gore")
+        if soup.find('a',{'class':'story_category story_category_sex'}):
+            self.story.addToList('warnings',"Sex")
         
         for category in storyMetadata["categories"]:
             if storyMetadata["categories"][category]:
@@ -255,13 +251,20 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
                     value = unicode(value)
                 self.story.setMetadata(metakey, value)
 
-        ## Groups and sequels code from FaceDeer
-        allGroupLists = soup.findAll('ul', {'id':'story_group_list'})
-        for groupList in allGroupLists:
-            for groupName in groupList.findAll('a', {'href':re.compile('^/group/')}):
+        #groups
+        if soup.find('button', {'id':'button-view-all-groups'}):
+            groupResponse = self._fetchUrl("http://www.fimfiction.net/ajax/groups/story_groups_list.php?story=%s" % (self.story.getMetadata("storyId")))
+            groupData = json.loads(groupResponse)
+            groupList = bs.BeautifulSoup(groupData["content"])
+        else:
+            groupList = soup.find('ul', {'id':'story-groups-list'})
+
+        if not (groupList == None):
+            for groupName in groupList.findAll('a'):
                 self.story.addToList("groupsUrl", 'http://'+self.host+groupName["href"]) 
                 self.story.addToList("groups",stripHTML(groupName).replace(',', ';'))
 
+        #sequels
         sequelStoryHeader = soup.find('h1', {'class':'header-stories'}, text="Sequels")
         if not sequelStoryHeader == None:
             sequelContainer = sequelStoryHeader.parent.parent

index.html

@@ -60,10 +60,9 @@
 	  </p>
 	  <p>
 	    <ul>
-              <li>New Site: lotrfanfiction.com</li>
-              <li>New Site: csi-forensics.com, thanks scout78.</li>
-              <li>New Site: samandjack.net, thanks scout78.</li>
-              <li>Fixes for sheppardweir.com, thanks scout78.</li>
+              <li>Fixes for sites updates to: fimfiction.net (thanks, facedeer)</li>
+              <li>Fixes for sites updates to: ficwad.com</li>
+              <li>Fixes for sites updates to: archiveofourown.org</li>
               <li>Known issue: Password protected FimFiction.net stories aren't working.  FimF changed API access.</li>
               <li>Known issue: Specific metadata 'eroticatags' for literotica.com doesn't work on all stories.</li>
               <li>Known issue: Metadata collection is not as complete for 'Base eFiction' adapters.</li>
@@ -77,7 +76,7 @@
 	    If you have any problems with this application, please
 	    report them in
 	    the <a href="http://groups.google.com/group/fanfic-downloader">FanFictionDownLoader Google Group</a>.  The
-	    <a href="http://2-0-05.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
+	    <a href="http://2-0-06.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
 	  </p>
 	  <div id='error'>
 	    {{ error_message }}

plugin-defaults.ini

@@ -315,7 +315,7 @@ sort_ships:false
 #keep_in_order_author:true
 
 ## User-agent
-user_agent:FFDL/1.7
+user_agent:FFDL/2.0
 
 ## Each output format has a section that overrides [defaults]
 [html]