Update CLI download zip.

app.yaml

@@ -1,6 +1,6 @@
 # ffd-retief-hrd fanfictiondownloader
 application: fanfictiondownloader
-version: 4-4-98
+version: 4-5-00
 runtime: python27
 api_version: 1
 threadsafe: true

calibre-plugin/__init__.py

@@ -42,7 +42,7 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
     description         = _('UI plugin to download FanFiction stories from various sites.')
     supported_platforms = ['windows', 'osx', 'linux']
     author              = 'Jim Miller'
-    version             = (1, 8, 17)
+    version             = (1, 8, 19)
     minimum_calibre_version = (1, 13, 0)
 
     #: This field defines the GUI plugin class that contains all the code

calibre-plugin/ffdl_plugin.py

@@ -970,7 +970,8 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                 if book_id and mi: # book_id and mi only set if matched by title/author.
                     liburl = self.get_story_url(db,book_id)
                     if book['url'] != liburl and prefs['checkforurlchange'] and \
-                        not (book['url'].replace('https','http') == liburl and 'fanfiction.net' in liburl):
+                        not (book['url'].replace('https','http') == liburl): # several sites have been changing to
+                                                                             # https now.  Don't flag when that's the only change.
                         # special case for ffnet urls change to https.
                         if not question_dialog(self.gui, _('Change Story URL?'),'''
                                                   <h3>%s</h3>

defaults.ini

@@ -219,7 +219,32 @@ connect_timeout:60.0
 # .*-Centered=>
 # characters=>Sam W\.=>Sam Witwicky&&category=>Transformers
 # characters=>Sam W\.=>Sam Winchester&&category=>Supernatural
- 
+
+## Include/Exclude metadata
+##
+## You can use the include/exclude metadata features to either limit
+## the values of particular metadata lists to specific values or to
+## exclude specific values.  Further, you can conditionally apply each
+## line depending on other metadata, use exact strings or regular
+## expressions(regex) to match values, and negate matches.
+##
+## The settings are:
+## include_metadata_pre
+## exclude_metadata_pre
+## include_metadata_post
+## exclude_metadata_post
+##
+## The form of each line is:
+## metakey[,metakey]==exactvalue
+## metakey[,metakey]=~regex
+## metakey[,metakey]==exactvalue&&conditionalkey==exactcondvalue
+## metakey[,metakey]=~regex&&conditionalkey==exactcondvalue
+## metakey[,metakey]==exactvalue&&conditionalkey=~condregex
+##
+## This is fairly complicated, so it's documented on its own wiki
+## page:
+## https://code.google.com/p/fanficdownloader/wiki/InExcludeMetadataFeature
+
 ## Some readers don't show horizontal rule (<hr />) tags correctly.
 ## This replaces them all with a centered '* * *'.  (Note centering
 ## doesn't work on some devices either.)
@@ -1314,6 +1339,11 @@ extra_valid_entries:reviews,favs,follows
 ## when updating to enforce accurate chapters.
 #do_update_hook:false
 
+## fimfiction.net is reported to misinterprete some BBCode with
+## blockquotes incorrectly.  This fixes those instances and defaults
+## to on, but can be switched off if it is found to cause problems.
+fix_fimf_blockquotes:true
+
 ## Site dedicated to these categories/characters/ships
 extracategories:My Little Pony: Friendship is Magic
 

downloader.py

@@ -292,7 +292,7 @@ def main(argv,
        else:
            # regular download
            if options.metaonly:
-               print adapter.getStoryMetadataOnly()
+               print adapter.getStoryMetadataOnly().getAllMetadata()
            
            output_filename=writeStory(configuration,adapter,options.format,options.metaonly)
        

fanficdownloader/adapters/adapter_fimfictionnet.py

@@ -59,7 +59,7 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
         return "http://www.fimfiction.net/story/1234/story-title-here http://www.fimfiction.net/story/1234/ http://www.fimfiction.com/story/1234/1/ http://mobile.fimfiction.net/story/1234/1/story-title-here/chapter-title-here"
 
     def getSiteURLPattern(self):
-        return r"http://(www|mobile)\.fimfiction\.(net|com)/story/\d+/?.*"
+        return r"https?://(www|mobile)\.fimfiction\.(net|com)/story/\d+/?.*"
         
     def extractChapterUrlsAndMetadata(self):
         
@@ -85,7 +85,7 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
             
             # Unfortunately, we still need to load the story index
             # page to parse the characters.  And chapters, now, too.
-            data = self._fetchUrl(self.url)
+            data = self.do_fix_blockquotes(self._fetchUrl(self.url))
             soup = bs.BeautifulSoup(data)
         except urllib2.HTTPError, e:
             if e.code == 404:
@@ -283,12 +283,21 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
             print("Existing epub has %s chapters\nNewest chapter is %s.  Discarding old chapters from there on."%(len(self.oldchapters), self.newestChapterNum+1))
             self.oldchapters = self.oldchapters[:self.newestChapterNum]
         return len(self.oldchapters)
-            
+
+    def do_fix_blockquotes(self,data):
+        if self.getConfig('fix_fimf_blockquotes'):
+            # <p class="double"><blockquote>
+            # </blockquote></p>
+            # include > in re groups so there's always something in the group.
+            data = re.sub(r'<p([^>]*>\s*)<blockquote([^>]*>)',r'<blockquote\2<p\1',data)
+            data = re.sub(r'</blockquote(>\s*)</p>',r'</p\1</blockquote>',data)
+        return data
         
     def getChapterText(self, url):
         logger.debug('Getting chapter text from: %s' % url)
-        
-        soup = bs.BeautifulSoup(self._fetchUrl(url),selfClosingTags=('br','hr')).find('div', {'class' : 'chapter_content'})
+
+        data = self.do_fix_blockquotes(self._fetchUrl(url))
+        soup = bs.BeautifulSoup(data,selfClosingTags=('br','hr')).find('div', {'class' : 'chapter_content'})
         if soup == None:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
         return self.utf8FromSoup(url,soup)

fanficdownloader/adapters/adapter_onedirectionfanfictioncom.py

@@ -192,7 +192,7 @@ class OneDirectionFanfictionComAdapter(BaseSiteAdapter):
             if 'Summary' in label:
                 ## Everything until the next span class='label'
                 svalue = ""
-                while not defaultGetattr(value,'class') == 'label':
+                while value and not defaultGetattr(value,'class') == 'label':
                     svalue += str(value)
                     value = value.nextSibling
                 self.setDescription(url,svalue)

fanficdownloader/adapters/adapter_squidgeorgpeja.py

@@ -62,7 +62,7 @@ class SquidgeOrgPejaAdapter(BaseSiteAdapter):
         
         
         # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/peja/cgi-bin/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        self._setURL('https://' + self.getSiteDomain() + '/peja/cgi-bin/viewstory.php?sid='+self.story.getMetadata('storyId'))
         
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','wwomb')
@@ -83,10 +83,10 @@ class SquidgeOrgPejaAdapter(BaseSiteAdapter):
     
     @classmethod
     def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/peja/cgi-bin/viewstory.php?sid=1234"
+        return "https://"+self.getSiteDomain()+"/peja/cgi-bin/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/")+"~?"+re.escape("peja/cgi-bin/viewstory.php?sid=")+r"\d+$"        
+        return r"https?"+re.escape("://"+self.getSiteDomain()+"/")+r"~?"+re.escape("peja/cgi-bin/viewstory.php?sid=")+r"\d+$"        
 
     ## Getting the chapter list and the meta data, plus 'is adult' checking.
     def extractChapterUrlsAndMetadata(self):
@@ -116,7 +116,7 @@ class SquidgeOrgPejaAdapter(BaseSiteAdapter):
         # Find authorid and URL from... author url.
         author = soup.find('div', {'id':"pagetitle"}).find('a')
         self.story.setMetadata('authorId',author['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/peja/cgi-bin/'+author['href'])
+        self.story.setMetadata('authorUrl','https://'+self.host+'/peja/cgi-bin/'+author['href'])
         self.story.setMetadata('author',author.string)
 		
         authorSoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
@@ -131,7 +131,7 @@ class SquidgeOrgPejaAdapter(BaseSiteAdapter):
         chapterselect=soup.find('select',{'name':'chapter'})
         if chapterselect:
             for ch in chapterselect.findAll('option'):
-                self.chapterUrls.append((stripHTML(ch),'http://'+self.host+'/peja/cgi-bin/viewstory.php?sid='+self.story.getMetadata('storyId')+'&chapter='+ch['value']))
+                self.chapterUrls.append((stripHTML(ch),'https://'+self.host+'/peja/cgi-bin/viewstory.php?sid='+self.story.getMetadata('storyId')+'&chapter='+ch['value']))
         else:
             self.chapterUrls.append((title,url))
 		
@@ -207,7 +207,7 @@ class SquidgeOrgPejaAdapter(BaseSiteAdapter):
             # http://www.squidge.org/peja/cgi-bin/series.php?seriesid=254
             a = titleblock.find('a', href=re.compile(r"series.php\?seriesid=\d+"))
             series_name = a.string
-            series_url = 'http://'+self.host+'/peja/cgi-bin/'+a['href']
+            series_url = 'https://'+self.host+'/peja/cgi-bin/'+a['href']
     
             # use BeautifulSoup HTML parser to make everything easier to find.
             seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))

fanficdownloader/adapters/base_adapter.py

@@ -188,9 +188,9 @@ class BaseSiteAdapter(Configurable):
             try:
                 return self._decode(self._fetchUrlRaw(url,parameters))
             except u2.HTTPError, he:
+                excpt=he
                 if he.code == 404:
                     logger.warn("Caught an exception reading URL: %s  Exception %s."%(unicode(url),unicode(he)))
-                    excpt=he
                     break # break out on 404
             except Exception, e:
                 excpt=e

fanficdownloader/story.py

@@ -221,6 +221,65 @@ langs = {
     "Devanagari":"hi",
     }
 
+class InExMatch:
+    keys = []
+    regex = None
+    match = None
+    negate = False
+
+    def  __init__(self,line):
+        if "=~" in line:
+            (self.keys,self.match) = line.split("=~")
+            self.match = self.match.replace('\s',' ')
+            self.regex = re.compile(self.match)
+        elif "!~" in line:
+            (self.keys,self.match) = line.split("!~")
+            self.match = self.match.replace('\s',' ')
+            self.regex = re.compile(self.match)
+            self.negate = True
+        elif "==" in line:
+            (self.keys,self.match) = line.split("==")
+            self.match = self.match.replace('\s',' ')
+        elif "!=" in line:
+            (self.keys,self.match) = line.split("!=")
+            self.match = self.match.replace('\s',' ')
+            self.negate = True
+        self.keys = map( lambda x: x.strip(), self.keys.split(",") )
+
+    # For conditional, only one key
+    def is_key(self,key):
+        return key == self.keys[0]
+
+    # For conditional, only one key
+    def key(self):
+        return self.keys[0]
+
+    def in_keys(self,key):
+        return key in self.keys
+
+    def is_match(self,value):
+        retval = False
+        if self.regex:
+            if self.regex.search(value):
+                retval = True
+            #print(">>>>>>>>>>>>>%s=~%s r: %s,%s=%s"%(self.match,value,self.negate,retval,self.negate != retval))
+        else:
+            retval = self.match == value
+            #print(">>>>>>>>>>>>>%s==%s r: %s,%s=%s"%(self.match,value,self.negate,retval, self.negate != retval))
+            
+        return self.negate != retval
+
+    def __str__(self):
+        if self.negate:
+            f='!'
+        else:
+            f='='
+        if self.regex:
+            s='~'
+        else:
+            s='='
+        return u'InExMatch(%s %s%s %s)'%(self.keys,f,s,self.match)
+
 class Story(Configurable):
 
     def __init__(self, configuration):
@@ -231,6 +290,7 @@ class Story(Configurable):
         except:
             self.metadata = {'version':'4.4'}
         self.replacements = []
+        self.in_ex_cludes = {}
         self.chapters = [] # chapters will be tuples of (title,html)
         self.imgurls = []
         self.imgtuples = []
@@ -252,6 +312,15 @@ class Story(Configurable):
 
         self.setReplace(self.getConfig('replace_metadata'))
 
+        in_ex_clude_list = ['include_metadata_pre','exclude_metadata_pre',
+                            'include_metadata_post','exclude_metadata_post']
+        for ie in in_ex_clude_list:
+            ies = self.getConfig(ie)
+            # print("%s %s"%(ie,ies))
+            if ies:
+                iel = []
+                self.in_ex_cludes[ie] = self.set_in_ex_clude(ies)
+
     def setMetadata(self, key, value, condremoveentities=True):
         ## still keeps < < and &
         if condremoveentities:
@@ -269,6 +338,56 @@ class Story(Configurable):
             self.addToList('lastupdate',value.strftime("Last Update Year/Month: %Y/%m"))
             self.addToList('lastupdate',value.strftime("Last Update: %Y/%m/%d"))
 
+        
+    ## metakey[,metakey]=~pattern
+    ## metakey[,metakey]==string
+    ## *for* part lines.  Effect only when trailing conditional key=~regexp matches
+    ## metakey[,metakey]=~pattern[&&metakey=~regexp]
+    ## metakey[,metakey]==string[&&metakey=~regexp]
+    ## metakey[,metakey]=~pattern[&&metakey==string]
+    ## metakey[,metakey]==string[&&metakey==string]
+    def set_in_ex_clude(self,setting):
+        dest = []
+        # print("set_in_ex_clude:"+setting)
+        for line in setting.splitlines():
+            if line:
+                (match,condmatch)=(None,None)
+                if "&&" in line:
+                    (line,conditional) = line.split("&&")
+                    condmatch = InExMatch(conditional)
+                match = InExMatch(line)
+                dest.append([match,condmatch])
+        return dest
+              
+    def do_in_ex_clude(self,which,value,key):
+        if value and which in self.in_ex_cludes:
+            include = 'include' in which
+            keyfound = False
+            found = False
+            for (match,condmatch) in self.in_ex_cludes[which]:
+                keyfndnow = False
+                if match.in_keys(key):
+                    # key in keys and either no conditional, or conditional matched
+                    if condmatch == None or condmatch.is_key(key):
+                        keyfndnow = True
+                    else:
+                        condval = self.getMetadata(condmatch.key())
+                        keyfndnow = condmatch.is_match(condval)
+                    keyfound |= keyfndnow
+                        # print("match:%s %s\ncondmatch:%s %s\n\tkeyfound:%s\n\tfound:%s"%(
+                        #         match,value,condmatch,condval,keyfound,found))
+                    if keyfndnow:
+                        found = isinstance(value,basestring) and match.is_match(value)
+                    if found:
+                        # print("match:%s %s\n\tkeyfndnow:%s\n\tfound:%s"%(
+                        #         match,value,keyfndnow,found))
+                        if not include:
+                            value = None
+                        break
+            if include and keyfound and not found:
+                value = None
+        return value
+        
 
     ## Two or three part lines.  Two part effect everything.
     ## Three part effect only those key(s) lists.
@@ -301,6 +420,10 @@ class Story(Configurable):
                 self.replacements.append([metakeys,regexp,replacement,condkey,condregexp])
 
     def doReplacements(self,value,key):
+        value = self.do_in_ex_clude('include_metadata_pre',value,key)
+        value = self.do_in_ex_clude('exclude_metadata_pre',value,key)
+        # if key=='characters' and value=='Bilbo':
+        #     value = ''
         for (metakeys,regexp,replacement,condkey,condregexp) in self.replacements:
             if (metakeys == None or key in metakeys) \
                     and isinstance(value,basestring) \
@@ -312,6 +435,8 @@ class Story(Configurable):
 
                 if doreplace:
                     value = regexp.sub(replacement,value)
+        value = self.do_in_ex_clude('include_metadata_post',value,key)
+        value = self.do_in_ex_clude('exclude_metadata_post',value,key)
         return value
 
     def getMetadataRaw(self,key):

index.html

@@ -64,7 +64,10 @@
 	  <h3>Changes:</h3>
 	  <p>
 	    <ul>
-              <li>Fix for a problem with literotica.com multi-page chapters on Kobo readers.</li>
+              <li>Allow https URLs for fimfiction.net.</li>
+              <li>Both allow https URLs for squidge.org/peja and change canonical URLs for squidge.org/peja to https.</li>
+              <li>Fix for some stories' summaries on onedirectionfanfiction.com.</li>
+              <li>Add include/exclude metadata feature.</li>
 	    </ul>
 	  </p>
 	  <p>
@@ -75,7 +78,7 @@
 	    If you have any problems with this application, please
 	    report them in
 	    the <a href="http://groups.google.com/group/fanfic-downloader">FanFictionDownLoader Google Group</a>.  The
-	    <a href="http://4-4-97.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
+	    <a href="http://4-4-99.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
 	  </p>
 	  <div id='error'>
 	    {{ error_message }}

plugin-defaults.ini

@@ -186,7 +186,32 @@ connect_timeout:60.0
 # .*-Centered=>
 # characters=>Sam W\.=>Sam Witwicky&&category=>Transformers
 # characters=>Sam W\.=>Sam Winchester&&category=>Supernatural
- 
+
+## Include/Exclude metadata
+##
+## You can use the include/exclude metadata features to either limit
+## the values of particular metadata lists to specific values or to
+## exclude specific values.  Further, you can conditionally apply each
+## line depending on other metadata, use exact strings or regular
+## expressions(regex) to match values, and negate matches.
+##
+## The settings are:
+## include_metadata_pre
+## exclude_metadata_pre
+## include_metadata_post
+## exclude_metadata_post
+##
+## The form of each line is:
+## metakey[,metakey]==exactvalue
+## metakey[,metakey]=~regex
+## metakey[,metakey]==exactvalue&&conditionalkey==exactcondvalue
+## metakey[,metakey]=~regex&&conditionalkey==exactcondvalue
+## metakey[,metakey]==exactvalue&&conditionalkey=~condregex
+##
+## This is fairly complicated, so it's documented on its own wiki
+## page:
+## https://code.google.com/p/fanficdownloader/wiki/InExcludeMetadataFeature
+
 ## Some readers don't show horizontal rule (<hr />) tags correctly.
 ## This replaces them all with a centered '* * *'.  (Note centering
 ## doesn't work on some devices either.)
@@ -1307,6 +1332,11 @@ extra_valid_entries:reviews,favs,follows
 ## when updating to enforce accurate chapters.
 #do_update_hook:false
 
+## fimfiction.net is reported to misinterprete some BBCode with
+## blockquotes incorrectly.  This fixes those instances and defaults
+## to on, but can be switched off if it is found to cause problems.
+fix_fimf_blockquotes:true
+
 ## Site dedicated to these categories/characters/ships
 extracategories:My Little Pony: Friendship is Magic