Bump versions.

app.yaml

@@ -1,6 +1,6 @@
 # ffd-retief-hrd fanfictiondownloader
 application: fanfictiondownloader
-version: 4-4-71
+version: 4-4-74
 runtime: python27
 api_version: 1
 threadsafe: true

calibre-plugin/__init__.py

@@ -26,7 +26,7 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
     description         = 'UI plugin to download FanFiction stories from various sites.'
     supported_platforms = ['windows', 'osx', 'linux']
     author              = 'Jim Miller'
-    version             = (1, 7, 42)
+    version             = (1, 7, 45)
     minimum_calibre_version = (0, 8, 57)
 
     #: This field defines the GUI plugin class that contains all the code

calibre-plugin/ffdl_plugin.py

@@ -1469,7 +1469,8 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                 db.set_custom(book_id, book['all_metadata'][meta], label, commit=False)
             elif coldef['datatype'] in ('int','float'):
                 num = unicode(book['all_metadata'][meta]).replace(",","")
-                db.set_custom(book_id, num, label=label, commit=False)
+                if num != '':
+                    db.set_custom(book_id, num, label=label, commit=False)
             elif coldef['datatype'] == 'bool' and meta.startswith('status-'):
                 if meta == 'status-C':
                     val = book['all_metadata']['status'] == 'Completed'
@@ -1508,7 +1509,8 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                                 val = unicode(book['all_metadata'][meta]).replace(",","")
                         else:
                             val = book['all_metadata'][meta]
-                        db.set_custom(book_id, val, label=label, commit=False)
+                        if val != '':
+                            db.set_custom(book_id, val, label=label, commit=False)
 
                     if flag == 'a':
                         vallist = []

defaults.ini

@@ -170,6 +170,14 @@ extratags: FanFiction
 ## Add this to genre if there's more than one category.
 #add_genre_when_multi_category: Crossover
 
+## default_value_(entry) can be used to set the value for a metadata
+## entry when no value has been found on the site.  For example, some
+## sites doesn't have a status metadatum.  If uncommented, this will
+## use 'Unknown' for status when no status is found.
+#default_value_status:Unknown
+## Can also be used for other metadata values
+#default_value_category:FanFiction
+
 ## number of seconds to sleep between calls to the story site.  May by
 ## useful if pulling large numbers of stories or if the site is slow.
 #slow_down_sleep_time:0.5
@@ -278,7 +286,7 @@ sort_ships:false
 #keep_in_order_author:true
 
 ## User-agent
-#user_agent:FFDL/1.7
+user_agent:FFDL/1.7
 
 ## Each output format has a section that overrides [defaults]
 [html]
@@ -1117,6 +1125,7 @@ context_label:Context
 type_label:Type of Couple
 
 [www.fanfiction.net]
+user_agent:
 ## fanfiction.net's 'cover' images are really just tiny thumbnails.
 ## Change this to false to use them anyway.
 never_make_cover: true
@@ -1147,7 +1156,12 @@ extracategories:Harry Potter
 ## fictionally.org storyIds are not unique.  Combine with authorId.
 output_filename: ${title}-${siteabbrev}_${authorId}_${storyId}${formatext}
 
+## fictionalley.org doesn't have a status metadatum.  If uncommented,
+## this will be used for status.
+#default_value_status:Unknown
+
 [www.fictionpress.com]
+user_agent:
 ## Clear FanFiction from defaults, fictionpress.com is original fiction.
 extratags:
 
@@ -1169,12 +1183,13 @@ extracategories:My Little Pony: Friendship is Magic
 
 ## Extra metadata that this adapter knows about.  See [dramione.org]
 ## for examples of how to use them.
-extra_valid_entries:likes,dislikes,views,total_views,short_description
+extra_valid_entries:likes,dislikes,views,total_views,short_description,groups
 likes_label:Likes
 dislikes_label:Dislikes
 views_label:Highest Single Chapter Views
 total_views_label:Total Views
 short_description_label:Short Summary
+groups_label:Groups
 
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,

fanficdownloader/adapters/adapter_fimfictionnet.py

@@ -21,7 +21,6 @@ logger = logging.getLogger(__name__)
 import re
 import urllib2
 import cookielib as cl
-from datetime import datetime
 import json
 
 from .. import BeautifulSoup as bs
@@ -42,6 +41,10 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
         self._setURL("http://"+self.getSiteDomain()+"/story/"+self.story.getMetadata('storyId')+"/")
         self.is_adult = False
         
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%d %b %Y"
+            
     @staticmethod
     def getSiteDomain():
         return 'www.fimfiction.net'
@@ -185,14 +188,31 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
         hrstr="<hr />"
         descdivstr = '<div class="description">'+descdivstr[descdivstr.index(hrstr)+len(hrstr):]
         self.setDescription(self.url,descdivstr)
-        
+
+        # Can't trust dates from API anymore I'm told.
         # Dates are in Unix time
         # Take the publish date from the first chapter posted
-        rawDatePublished = storyMetadata["chapters"][0]["date_modified"]
-        self.story.setMetadata("datePublished", datetime.fromtimestamp(rawDatePublished))
-        rawDateUpdated = storyMetadata["date_modified"]
-        self.story.setMetadata("dateUpdated", datetime.fromtimestamp(rawDateUpdated))
-
+        # rawDatePublished = storyMetadata["chapters"][0]["date_modified"]
+        # self.story.setMetadata("datePublished", datetime.fromtimestamp(rawDatePublished))
+        # rawDateUpdated = storyMetadata["date_modified"]
+        # self.story.setMetadata("dateUpdated", datetime.fromtimestamp(rawDateUpdated))
+        
+        oldestChapter = None
+        newestChapter = None
+        # Scan all chapters to find the oldest and newest, on
+        # FiMFiction it's possible for authors to insert new chapters
+        # out-of-order or change the dates of earlier ones by editing
+        # them--That WILL break epub update.
+        for chapterDate in soup.findAll('span', {'class':'date'}):
+            date=re.sub(r"(\d+)(st|nd|rd|th)",r"\1",chapterDate.contents[1].strip())
+            chapterDate = makeDate(date,self.dateformat)
+            if oldestChapter == None or chapterDate < oldestChapter:
+                oldestChapter = chapterDate
+            if newestChapter == None or chapterDate > newestChapter:
+                newestChapter = chapterDate
+        self.story.setMetadata("datePublished", oldestChapter)
+        self.story.setMetadata("dateUpdated", newestChapter)
+        
         chars = soup.find("div", {"class":"inner_data"})
         # fimfic stopped putting the char name on or around the char
         # icon now for some reason.  Pull it from the image name with
@@ -215,7 +235,11 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
                 if not isinstance(value,basestring):
                     value = unicode(value)
                 self.story.setMetadata(metakey, value)
-        
+
+        rawGroupList = soup.find('ul', {'id':'story_group_list'})
+        if rawGroupList is not None:
+            for groupName in rawGroupList.findAll('a', {'href':re.compile('^/group/')}):
+                self.story.addToList("groups",stripHTML(groupName))
             
     def getChapterText(self, url):
         logger.debug('Getting chapter text from: %s' % url)

fanficdownloader/adapters/adapter_portkeyorg.py

@@ -251,8 +251,10 @@ class PortkeyOrgAdapter(BaseSiteAdapter): # XXX
 
         logger.debug('Getting chapter text from: %s' % url)
         data = self._fetchUrl(url)
-        soup = bs.BeautifulStoneSoup(data,
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+
+        data = data.replace("HTML>","div>")
+        
+        soup = bs.BeautifulSoup(data)
 
         #print("soup:%s"%soup)
         tag = soup.find('td', {'class' : 'story'})

fanficdownloader/adapters/adapter_whoficcom.py

@@ -22,6 +22,7 @@ import re
 import urllib2
 
 from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
 from .. import exceptions as exceptions
 
 from base_adapter import BaseSiteAdapter,  makeDate

fanficdownloader/adapters/base_adapter.py

@@ -379,7 +379,7 @@ fullmon = {"January":"01", "February":"02", "March":"03", "April":"04", "May":"0
            "June":"06","July":"07", "August":"08", "September":"09", "October":"10",
            "November":"11", "December":"12" }
 
-def makeDate(string,format):
+def makeDate(string,dateform):
     # Surprise!  Abstracting this turned out to be more useful than
     # just saving bytes.
 
@@ -388,10 +388,10 @@ def makeDate(string,format):
     # there's non-english content. -- ficbook.net now makes that a
     # lie.  It has to do something even more complicated to get
     # Russian month names correct everywhere.
-    do_abbrev = "%b" in format
+    do_abbrev = "%b" in dateform
         
-    if "%B" in format or do_abbrev:
-        format = format.replace("%B","%m").replace("%b","%m")
+    if "%B" in dateform or do_abbrev:
+        dateform = dateform.replace("%B","%m").replace("%b","%m")
         for (name,num) in fullmon.items():
             if do_abbrev:
                 name = name[:3] # first three for abbrev
@@ -399,5 +399,5 @@ def makeDate(string,format):
                 string = string.replace(name,num)
                 break
             
-    return datetime.datetime.strptime(string,format)
+    return datetime.datetime.strptime(string,dateform)
 

fanficdownloader/htmlcleanup.py

@@ -69,8 +69,9 @@ def removeEntities(text):
     
     if text is None:
         return ""
-    if not (isinstance(text,str) or isinstance(text,unicode)):
-        return str(text)
+    
+    if not isinstance(text,basestring):
+        return unicode(text)
     
     try:
         t = text.decode('utf-8')

fanficdownloader/story.py

@@ -349,6 +349,8 @@ class Story(Configurable):
                 return removeAllEntities(value)
             else:
                 return value
+        else: #if self.getConfig("default_value_"+key):
+            return self.getConfig("default_value_"+key)
         
     def getAllMetadata(self,
                        removeallentities=False,
@@ -485,6 +487,8 @@ class Story(Configurable):
 
         if None in subjectset:
             subjectset.remove(None)
+        if '' in subjectset:
+            subjectset.remove('')
 
         return list(subjectset | set(self.getConfigList("extratags")))
             

index.html

@@ -53,23 +53,19 @@
 	    <p>Hi, {{ nickname }}! This is FanFictionDownLoader, which makes reading stories from various websites
 	      much easier. </p>
 	  </div>
-	  <!-- put announcements here, h3 is a good title size.
+	  <!-- put announcements here, h3 is a good title size. -->
 	  <h3>Changes:</h3>
 	  <p>
 	    <ul>
-              <li>Better doc section override order in ini files.</li>
-              <li>Additional series as site specific data for AO3.</li>
-              <li>Fix for AO3 stories without series.</li>
-              <li>Fixes for changes to harrypotterfanfictioncom.</li>
-              <li>Add User-agent="FFDL/1.7" for all adapters for fanfiction.net changes.<br>
-                (Remove from specific adapters.)</li>
+              <li>Fix for whofic.com.</li>
+              <li>Don't include empty string dc:subject tags in epub.</li>
 	    </ul>
 	  </p>
-           -->
+          
           <h3>fanfiction.net</h3>
           <p>
             Fanfiction.net appears to be blocking access from Google
-            App Engine IPs, which prevents this web service.  There's
+            App Engine, which prevents this web service.  There's
             nothing I can do about it.  At the time of writing, the
             latest CLI and calibre plugin versions worked.
           </p>
@@ -82,7 +78,7 @@
 	    If you have any problems with this application, please
 	    report them in
 	    the <a href="http://groups.google.com/group/fanfic-downloader">FanFictionDownLoader Google Group</a>.  The
-	    <a href="http://4-4-67.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
+	    <a href="http://4-4-73.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
 	  </p>
 	  <div id='error'>
 	    {{ error_message }}

plugin-defaults.ini

@@ -142,6 +142,14 @@ extratags: FanFiction
 ## Add this to genre if there's more than one category.
 #add_genre_when_multi_category: Crossover
 
+## default_value_(entry) can be used to set the value for a metadata
+## entry when no value has been found on the site.  For example, some
+## sites doesn't have a status metadatum.  If uncommented, this will
+## use 'Unknown' for status when no status is found.
+#default_value_status:Unknown
+## Can also be used for other metadata values
+#default_value_category:FanFiction
+
 ## number of seconds to sleep between calls to the story site.  May by
 ## useful if pulling large numbers of stories or if the site is slow.
 #slow_down_sleep_time:0.5
@@ -265,8 +273,7 @@ sort_ships:false
 #keep_in_order_author:true
 
 ## User-agent
-#user_agent:FFDL/1.7
-
+user_agent:FFDL/1.7
 
 ## Each output format has a section that overrides [defaults]
 [html]
@@ -1103,6 +1110,7 @@ context_label:Context
 type_label:Type of Couple
 
 [www.fanfiction.net]
+user_agent:
 ## fanfiction.net's 'cover' images are really just tiny thumbnails.
 ## Change this to false to use them anyway.
 never_make_cover: true
@@ -1130,7 +1138,12 @@ extracategories:Harry Potter
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+## fictionalley.org doesn't have a status metadatum.  If uncommented,
+## this will be used for status.
+#default_value_status:Unknown
+
 [www.fictionpress.com]
+user_agent:
 ## Clear FanFiction from defaults, fictionpress.com is original fiction.
 extratags:
 
@@ -1152,12 +1165,13 @@ extracategories:My Little Pony: Friendship is Magic
 
 ## Extra metadata that this adapter knows about.  See [dramione.org]
 ## for examples of how to use them.
-extra_valid_entries:likes,dislikes,views,total_views,short_description
+extra_valid_entries:likes,dislikes,views,total_views,short_description,groups
 likes_label:Likes
 dislikes_label:Dislikes
 views_label:Highest Single Chapter Views
 total_views_label:Total Views
 short_description_label:Short Summary
+groups_label:Groups
 
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,

status.html

@@ -51,7 +51,14 @@
 	      by {{ fic.author }} ({{ fic.format }})
 	  {% endif %}
 	  {% if fic.failure %}
-	      <span id='error'>{{ fic.failure }}</span>
+          <h3>fanfiction.net</h3>
+          <p>
+            FYI, fanfiction.net appears to be blocking access from Google
+            App Engine, which prevents this web service.  There's
+            nothing I can do about it.  At the time of writing, the
+            latest CLI and calibre plugin versions worked.
+          </p>
+	  <span id='error'>{{ fic.failure }}</span>
 	  {% endif %}
 	  {% if  not fic.completed and not fic.failure %}
 	      <p>Not done yet.  This page will periodically poll to see if your story has finished.</p>