Fix fimfic dates, add fimfic site specific 'groups' metadata. Bump versions.

app.yaml

@@ -1,6 +1,6 @@
 # ffd-retief-hrd fanfictiondownloader
 application: fanfictiondownloader
-version: 4-4-71
+version: 4-4-72
 runtime: python27
 api_version: 1
 threadsafe: true

calibre-plugin/__init__.py

@@ -26,7 +26,7 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
     description         = 'UI plugin to download FanFiction stories from various sites.'
     supported_platforms = ['windows', 'osx', 'linux']
     author              = 'Jim Miller'
-    version             = (1, 7, 42)
+    version             = (1, 7, 43)
     minimum_calibre_version = (0, 8, 57)
 
     #: This field defines the GUI plugin class that contains all the code

defaults.ini

@@ -170,6 +170,14 @@ extratags: FanFiction
 ## Add this to genre if there's more than one category.
 #add_genre_when_multi_category: Crossover
 
+## default_value_(entry) can be used to set the value for a metadata
+## entry when no value has been found on the site.  For example, some
+## sites doesn't have a status metadatum.  If uncommented, this will
+## use 'Unknown' for status when no status is found.
+#default_value_status:Unknown
+## Can also be used for other metadata values
+#default_value_category:FanFiction
+
 ## number of seconds to sleep between calls to the story site.  May by
 ## useful if pulling large numbers of stories or if the site is slow.
 #slow_down_sleep_time:0.5
@@ -278,7 +286,7 @@ sort_ships:false
 #keep_in_order_author:true
 
 ## User-agent
-#user_agent:FFDL/1.7
+user_agent:FFDL/1.7
 
 ## Each output format has a section that overrides [defaults]
 [html]
@@ -1117,6 +1125,7 @@ context_label:Context
 type_label:Type of Couple
 
 [www.fanfiction.net]
+user_agent:
 ## fanfiction.net's 'cover' images are really just tiny thumbnails.
 ## Change this to false to use them anyway.
 never_make_cover: true
@@ -1147,7 +1156,12 @@ extracategories:Harry Potter
 ## fictionally.org storyIds are not unique.  Combine with authorId.
 output_filename: ${title}-${siteabbrev}_${authorId}_${storyId}${formatext}
 
+## fictionalley.org doesn't have a status metadatum.  If uncommented,
+## this will be used for status.
+#default_value_status:Unknown
+
 [www.fictionpress.com]
+user_agent:
 ## Clear FanFiction from defaults, fictionpress.com is original fiction.
 extratags:
 
@@ -1169,12 +1183,13 @@ extracategories:My Little Pony: Friendship is Magic
 
 ## Extra metadata that this adapter knows about.  See [dramione.org]
 ## for examples of how to use them.
-extra_valid_entries:likes,dislikes,views,total_views,short_description
+extra_valid_entries:likes,dislikes,views,total_views,short_description,groups
 likes_label:Likes
 dislikes_label:Dislikes
 views_label:Highest Single Chapter Views
 total_views_label:Total Views
 short_description_label:Short Summary
+groups_label:Groups
 
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,

fanficdownloader/adapters/adapter_fimfictionnet.py

@@ -21,7 +21,8 @@ logger = logging.getLogger(__name__)
 import re
 import urllib2
 import cookielib as cl
-from datetime import datetime
+#from datetime import datetime
+import dateutil.parser as dparser
 import json
 
 from .. import BeautifulSoup as bs
@@ -185,14 +186,30 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
         hrstr="<hr />"
         descdivstr = '<div class="description">'+descdivstr[descdivstr.index(hrstr)+len(hrstr):]
         self.setDescription(self.url,descdivstr)
-        
+
+        # Can't trust dates from API anymore I'm told.
         # Dates are in Unix time
         # Take the publish date from the first chapter posted
-        rawDatePublished = storyMetadata["chapters"][0]["date_modified"]
-        self.story.setMetadata("datePublished", datetime.fromtimestamp(rawDatePublished))
-        rawDateUpdated = storyMetadata["date_modified"]
-        self.story.setMetadata("dateUpdated", datetime.fromtimestamp(rawDateUpdated))
-
+        # rawDatePublished = storyMetadata["chapters"][0]["date_modified"]
+        # self.story.setMetadata("datePublished", datetime.fromtimestamp(rawDatePublished))
+        # rawDateUpdated = storyMetadata["date_modified"]
+        # self.story.setMetadata("dateUpdated", datetime.fromtimestamp(rawDateUpdated))
+        
+        oldestChapter = None
+        newestChapter = None
+        # Scan all chapters to find the oldest and newest, on
+        # FiMFiction it's possible for authors to insert new chapters
+        # out-of-order or change the dates of earlier ones by editing
+        # them--That WILL break epub update.
+        for chapterDate in soup.findAll('span', {'class':'date'}):
+            chapterDate = dparser.parse(chapterDate.contents[1].strip())
+            if oldestChapter == None or chapterDate < oldestChapter:
+                oldestChapter = chapterDate
+            if newestChapter == None or chapterDate > newestChapter:
+                newestChapter = chapterDate
+        self.story.setMetadata("datePublished", oldestChapter)
+        self.story.setMetadata("dateUpdated", newestChapter)
+        
         chars = soup.find("div", {"class":"inner_data"})
         # fimfic stopped putting the char name on or around the char
         # icon now for some reason.  Pull it from the image name with
@@ -215,7 +232,11 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
                 if not isinstance(value,basestring):
                     value = unicode(value)
                 self.story.setMetadata(metakey, value)
-        
+
+        rawGroupList = soup.find('ul', {'id':'story_group_list'})
+        if rawGroupList is not None:
+            for groupName in rawGroupList.findAll('a', {'href':re.compile('^/group/')}):
+                self.story.addToList("groups",stripHTML(groupName))
             
     def getChapterText(self, url):
         logger.debug('Getting chapter text from: %s' % url)

fanficdownloader/adapters/base_adapter.py

@@ -379,7 +379,7 @@ fullmon = {"January":"01", "February":"02", "March":"03", "April":"04", "May":"0
            "June":"06","July":"07", "August":"08", "September":"09", "October":"10",
            "November":"11", "December":"12" }
 
-def makeDate(string,format):
+def makeDate(string,dateform):
     # Surprise!  Abstracting this turned out to be more useful than
     # just saving bytes.
 
@@ -388,10 +388,10 @@ def makeDate(string,format):
     # there's non-english content. -- ficbook.net now makes that a
     # lie.  It has to do something even more complicated to get
     # Russian month names correct everywhere.
-    do_abbrev = "%b" in format
+    do_abbrev = "%b" in dateform
         
-    if "%B" in format or do_abbrev:
-        format = format.replace("%B","%m").replace("%b","%m")
+    if "%B" in dateform or do_abbrev:
+        dateform = dateform.replace("%B","%m").replace("%b","%m")
         for (name,num) in fullmon.items():
             if do_abbrev:
                 name = name[:3] # first three for abbrev
@@ -399,5 +399,5 @@ def makeDate(string,format):
                 string = string.replace(name,num)
                 break
             
-    return datetime.datetime.strptime(string,format)
+    return datetime.datetime.strptime(string,dateform)
 

fanficdownloader/story.py

@@ -349,6 +349,8 @@ class Story(Configurable):
                 return removeAllEntities(value)
             else:
                 return value
+        else: #if self.getConfig("default_value_"+key):
+            return self.getConfig("default_value_"+key)
         
     def getAllMetadata(self,
                        removeallentities=False,

index.html

@@ -53,19 +53,15 @@
 	    <p>Hi, {{ nickname }}! This is FanFictionDownLoader, which makes reading stories from various websites
 	      much easier. </p>
 	  </div>
-	  <!-- put announcements here, h3 is a good title size.
+	  <!-- put announcements here, h3 is a good title size. -->
 	  <h3>Changes:</h3>
 	  <p>
 	    <ul>
-              <li>Better doc section override order in ini files.</li>
-              <li>Additional series as site specific data for AO3.</li>
-              <li>Fix for AO3 stories without series.</li>
-              <li>Fixes for changes to harrypotterfanfictioncom.</li>
-              <li>Add User-agent="FFDL/1.7" for all adapters for fanfiction.net changes.<br>
-                (Remove from specific adapters.)</li>
+              <li>Fix dates for fimfiction.net.</li>
+              <li>Additional groups as site specific data for fimfiction.net.</li>
 	    </ul>
 	  </p>
-           -->
+          
           <h3>fanfiction.net</h3>
           <p>
             Fanfiction.net appears to be blocking access from Google

plugin-defaults.ini

@@ -142,6 +142,14 @@ extratags: FanFiction
 ## Add this to genre if there's more than one category.
 #add_genre_when_multi_category: Crossover
 
+## default_value_(entry) can be used to set the value for a metadata
+## entry when no value has been found on the site.  For example, some
+## sites doesn't have a status metadatum.  If uncommented, this will
+## use 'Unknown' for status when no status is found.
+#default_value_status:Unknown
+## Can also be used for other metadata values
+#default_value_category:FanFiction
+
 ## number of seconds to sleep between calls to the story site.  May by
 ## useful if pulling large numbers of stories or if the site is slow.
 #slow_down_sleep_time:0.5
@@ -265,8 +273,7 @@ sort_ships:false
 #keep_in_order_author:true
 
 ## User-agent
-#user_agent:FFDL/1.7
-
+user_agent:FFDL/1.7
 
 ## Each output format has a section that overrides [defaults]
 [html]
@@ -1103,6 +1110,7 @@ context_label:Context
 type_label:Type of Couple
 
 [www.fanfiction.net]
+user_agent:
 ## fanfiction.net's 'cover' images are really just tiny thumbnails.
 ## Change this to false to use them anyway.
 never_make_cover: true
@@ -1130,7 +1138,12 @@ extracategories:Harry Potter
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+## fictionalley.org doesn't have a status metadatum.  If uncommented,
+## this will be used for status.
+#default_value_status:Unknown
+
 [www.fictionpress.com]
+user_agent:
 ## Clear FanFiction from defaults, fictionpress.com is original fiction.
 extratags:
 
@@ -1152,12 +1165,13 @@ extracategories:My Little Pony: Friendship is Magic
 
 ## Extra metadata that this adapter knows about.  See [dramione.org]
 ## for examples of how to use them.
-extra_valid_entries:likes,dislikes,views,total_views,short_description
+extra_valid_entries:likes,dislikes,views,total_views,short_description,groups
 likes_label:Likes
 dislikes_label:Dislikes
 views_label:Highest Single Chapter Views
 total_views_label:Total Views
 short_description_label:Short Summary
+groups_label:Groups
 
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,