Integrate new adapter www.checkmated.com.

app.yaml

@@ -1,6 +1,6 @@
 # ffd-retief-hrd fanfictiondownloader
 application: fanfictiondownloader
-version: 4-4-9
+version: 4-4-10
 runtime: python27
 api_version: 1
 threadsafe: true

calibre-plugin/__init__.py

@@ -27,7 +27,7 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
     description         = 'UI plugin to download FanFiction stories from various sites.'
     supported_platforms = ['windows', 'osx', 'linux']
     author              = 'Jim Miller'
-    version             = (1, 5, 20)
+    version             = (1, 5, 22)
     minimum_calibre_version = (0, 8, 30)
 
     #: This field defines the GUI plugin class that contains all the code

calibre-plugin/config.py

@@ -47,6 +47,7 @@ all_prefs.defaults['collision'] = OVERWRITE
 all_prefs.defaults['deleteotherforms'] = False
 all_prefs.defaults['adddialogstaysontop'] = False
 all_prefs.defaults['includeimages'] = False
+all_prefs.defaults['lookforurlinhtml'] = False
 
 all_prefs.defaults['send_lists'] = ''
 all_prefs.defaults['read_lists'] = ''
@@ -54,6 +55,7 @@ all_prefs.defaults['addtolists'] = False
 all_prefs.defaults['addtoreadlists'] = False
 all_prefs.defaults['addtolistsonread'] = False
 
+all_prefs.defaults['gcnewonly'] = False
 all_prefs.defaults['gc_site_settings'] = {}
 all_prefs.defaults['allow_gc_from_ini'] = True
 
@@ -72,6 +74,8 @@ copylist = ['personal.ini',
             'deleteotherforms',
             'adddialogstaysontop',
             'includeimages',
+            'lookforurlinhtml',
+            'gcnewonly',
             'gc_site_settings',
             'allow_gc_from_ini']
 
@@ -176,6 +180,7 @@ class ConfigWidget(QWidget):
         prefs['deleteotherforms'] = self.basic_tab.deleteotherforms.isChecked()
         prefs['adddialogstaysontop'] = self.basic_tab.adddialogstaysontop.isChecked()
         prefs['includeimages'] = self.basic_tab.includeimages.isChecked()
+        prefs['lookforurlinhtml'] = self.basic_tab.lookforurlinhtml.isChecked()
 
         if self.readinglist_tab:
             # lists
@@ -196,6 +201,7 @@ class ConfigWidget(QWidget):
             prefs['personal.ini'] = get_resources('plugin-example.ini')
 
         # Generate Covers tab
+        prefs['gcnewonly'] = self.generatecover_tab.gcnewonly.isChecked()
         gc_site_settings = {}
         for (site,combo) in self.generatecover_tab.gc_dropdowns.iteritems():
             val = unicode(combo.itemData(combo.currentIndex()).toString())
@@ -309,6 +315,11 @@ class BasicTab(QWidget):
         self.includeimages.setChecked(prefs['includeimages'])
         self.l.addWidget(self.includeimages)
 
+        self.lookforurlinhtml = QCheckBox("Search EPUB text for Story URL?",self)
+        self.lookforurlinhtml.setToolTip("Look for first valid story URL inside EPUB text if not found in metadata.\nSomewhat risky, could find wrong URL depending on EPUB content.\nAlso finds and corrects bad ffnet URLs from ficsaver.com files.")
+        self.lookforurlinhtml.setChecked(prefs['lookforurlinhtml'])
+        self.l.addWidget(self.lookforurlinhtml)
+
         self.l.insertStretch(-1)
         
     def set_collisions(self):
@@ -511,6 +522,11 @@ class GenerateCoverTab(QWidget):
             horz.addWidget(dropdown)
             self.sl.addLayout(horz)
         
+        self.gcnewonly = QCheckBox("Run Generate Cover Only on New Books",self)
+        self.gcnewonly.setToolTip("Default is to run GC any time the calibre metadata is updated.")
+        self.gcnewonly.setChecked(prefs['gcnewonly'])
+        self.l.addWidget(self.gcnewonly)
+
         self.allow_gc_from_ini = QCheckBox('Allow generate_cover_settings from personal.ini to override.',self)
         self.allow_gc_from_ini.setToolTip("The INI parameter generate_cover_settings allows you to choose a GC setting based on metadata rather than site,\nbut it's much more complex.  generate_cover_settings is ignored when this is off.")
         self.allow_gc_from_ini.setChecked(prefs['allow_gc_from_ini'])

calibre-plugin/ffdl_plugin.py

@@ -36,7 +36,7 @@ from calibre_plugins.fanfictiondownloader_plugin.common_utils import (set_plugin
 
 from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapters, writers, exceptions
 from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.htmlcleanup import stripHTML
-from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.epubutils import get_dcsource, get_dcsource_chaptercount
+from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.epubutils import get_dcsource, get_dcsource_chaptercount, get_story_url_from_html
 
 from calibre_plugins.fanfictiondownloader_plugin.config import (prefs, permitted_values)
 from calibre_plugins.fanfictiondownloader_plugin.dialogs import (
@@ -815,8 +815,8 @@ make_firstimage_cover:true
 
         db.commit()
         
-        if 'Generate Cover' in self.gui.iactions:
-
+        print("book['added']:%s"%book['added'])
+        if 'Generate Cover' in self.gui.iactions and (book['added'] or not prefs['gcnewonly']):
             gc_plugin = self.gui.iactions['Generate Cover']
             setting_name = None
             if prefs['allow_gc_from_ini']:
@@ -1018,8 +1018,12 @@ make_firstimage_cover:true
                 if 'url' in identifiers:
                     #print("url from epub:"+identifiers['url'].replace('|',':'))
                     return identifiers['url'].replace('|',':')
-                # look for dc:source
-                return get_dcsource(existingepub)
+                # look for dc:source first, then scan HTML if 
+                link = get_dcsource(existingepub)
+                if link:
+                    return link
+                elif prefs['lookforurlinhtml']:
+                    return get_story_url_from_html(existingepub,self._is_good_downloader_url)
         return None
 
     def _is_good_downloader_url(self,url):

defaults.ini

@@ -382,6 +382,14 @@ extratags: FanFiction,Testing,HTML
 ## personal.ini, not defaults.ini.
 #is_adult:true
 
+[www.checkmated.com]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
 [www.fanfiction.net]
 
 [www.ficbook.net]

fanficdownloader/adapters/__init__.py

@@ -55,6 +55,7 @@ import adapter_archiveskyehawkecom
 import adapter_squidgeorgpeja
 import adapter_libraryofmoriacom
 import adapter_wraithbaitcom
+import adapter_checkmatedcom
 
 ## This bit of complexity allows adapters to be added by just adding
 ## importing.  It eliminates the long if/else clauses we used to need

fanficdownloader/adapters/adapter_checkmatedcom.py

@@ -0,0 +1,239 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+
+def getClass():
+    return CheckmatedComAdapter
+
+
+class CheckmatedComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        self._setURL('http://' + self.getSiteDomain() + '/story.php?story='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','chm')
+
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","Harry Potter")
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%b %d, %Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.checkmated.com'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/story.php?story=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/story.php?story=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites. This story is in The Bedchamber
+    def needToLoginCheck(self, data):
+        if 'This story is in The Bedchamber' in data \
+                or 'That username is not in our database' in data \
+                or "That password is not correct, please try again" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['name'] = self.username
+            params['pass'] = self.password
+        else:
+            params['name'] = self.getConfig("username")
+            params['pass'] = self.getConfig("password")
+        params['login'] = 'yes'
+        params['submit'] = 'login'	
+	
+        loginUrl = 'http://' + self.getSiteDomain()+'/login.php'
+        d = self._fetchUrl(loginUrl,params)
+        e = self._fetchUrl(url)
+
+        if "Welcome back," not in d : #Member Account
+            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['name']))
+            raise exceptions.FailedToLogin(url,params['name'])
+            return False
+        elif "This story is in The Bedchamber" in e:
+            logging.info("Your account does not have sufficient priviliges to read this story.")
+            return False
+        else:
+            return True
+
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+
+        # The actual text that is used to announce you need to be an
+        # adult varies from site to site.  Again, print data before
+        # the title search to troubleshoot.
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.findAll('span', {'class' : 'storytitle'})
+        self.story.setMetadata('title',a[0].string)
+        
+        # Find authorid and URL from... author url.
+        a = a[1].find('a', href=re.compile(r"authors.php\?name\=\w+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+			
+        a = soup.find('select', {'name' : 'chapter'})
+        if a == None:
+            self.chapterUrls.append((self.story.getMetadata('title'),url))
+        else:
+            for chapter in a.findAll('option'):
+                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/story.php?story='+self.story.getMetadata('storyId')+'&chapter='+chapter['value']))
+			
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+				
+        # website does not keep track of word count, and there is no convenient way to calculate it
+
+        summary = soup.find('fieldset')
+        summary.find('legend').extract()
+        summary.name='div'
+        self.setDescription(url,summary)
+
+		
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        table = soup.findAll('div', {'class' : 'text'})[1]
+        for labels in table.findAll('tr'):
+            value = labels.findAll('td')[1]
+            label = labels.findAll('td')[0]
+
+			
+            if 'Rating' in stripHTML(label):
+                self.story.setMetadata('rating', stripHTML(value))
+
+            if 'Ship' in stripHTML(label):
+                for char in value.string.split('/'):
+                    if char != 'none':
+                        self.story.addToList('characters',char)		
+
+            if 'Status' in stripHTML(label):
+                if value.find('img', {'src' : 'img/incomplete.gif'}) == None:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in stripHTML(label):
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in stripHTML(label):
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+			
+        a = self._fetchUrl(self.story.getMetadata('authorUrl')+'&cat=stories')	
+        for story in bs.BeautifulSoup(a).findAll('table', {'class' : 'storyinfo'}):
+            a = story.find('a', href=re.compile(r"review.php\?s\="+self.story.getMetadata('storyId')+'&act=view'))
+            if a != None:
+                for labels in story.findAll('tr'):
+                    value = labels.findAll('td')[1]
+                    label = labels.findAll('td')[0]
+                    if 'genre' in stripHTML(label):
+                        for genre in value.findAll('img'):
+                            self.story.addToList('genre',genre['title'])
+        
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'resizeableText'})
+        div.find('div', {'class' : 'storyTools'}).extract()
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/epubutils.py

@@ -94,3 +94,40 @@ def get_path_part(n):
     if( len(relpath) > 0 ):
         relpath=relpath+"/"
     return relpath
+
+def get_story_url_from_html(inputio,_is_good_url=None):
+
+    #print("get_story_url_from_html called")
+    epub = ZipFile(inputio, 'r')
+
+    ## Find the .opf file.
+    container = epub.read("META-INF/container.xml")
+    containerdom = parseString(container)
+    rootfilenodelist = containerdom.getElementsByTagName("rootfile")
+    rootfilename = rootfilenodelist[0].getAttribute("full-path")
+
+    contentdom = parseString(epub.read(rootfilename))
+    #firstmetadom = contentdom.getElementsByTagName("metadata")[0]
+
+    ## Save the path to the .opf file--hrefs inside it are relative to it.
+    relpath = get_path_part(rootfilename)
+            
+    # spin through the manifest--only place there are item tags.
+    for item in contentdom.getElementsByTagName("item"):
+        # First, count the 'chapter' files.  FFDL uses file0000.xhtml,
+        # but can also update epubs downloaded from Twisting the
+        # Hellmouth, which uses chapter0.html.
+        #print("---- item:%s"%item)
+        if( item.getAttribute("media-type") == "application/xhtml+xml" ):
+            filehref=relpath+item.getAttribute("href")
+            soup = bs.BeautifulSoup(epub.read(filehref).decode("utf-8"))
+            for link in soup.findAll('a',href=re.compile(r'^http.*')):
+                ahref=link['href']
+                #print("href:(%s)"%ahref)
+                # hack for bad ficsaver ffnet URLs.
+                m = re.match(r"^http://www.fanfiction.net/s(?P<id>\d+)//$",ahref)
+                if m != None:
+                    ahref="http://www.fanfiction.net/s/%s/1/"%m.group('id')
+                if _is_good_url == None or _is_good_url(ahref):
+                    return ahref
+    return None

index.html

@@ -54,18 +54,9 @@
 	      much easier. </p>
 	  </div>
 	  <!-- put announcements here, h3 is a good title size. -->
-	  <h3>New Sites</h3>
+	  <h3>New Site</h3>
 	  <p>
-	    We now support www.wraithbait.com and
-	    www.libraryofmoria.com, thanks to Ida for adding these!
-	  </p>
-	  <p>
-	    Support for the Wonderful World of MakeBelieve(WWOMB)
-	    archive at
-	    <a href="http://www.squidge.org/peja/cgi-bin/index.php">http://www.squidge.org/peja/cgi-bin/index.php</a>
-	    has also been added.  This does not support other sections of
-	    www.squidge.org, or the other files under www.squidge.org/peja that
-	    aren't in the eFiction instance.
+	    We now support www.checkmated.com, thanks to Ida for adding this!
 	  </p>
 	  <p>
 	    Questions?  Check out our 
@@ -75,7 +66,7 @@
 	    If you have any problems with this application, please
 	    report them in
 	    the <a href="http://groups.google.com/group/fanfic-downloader">FanFictionDownLoader Google Group</a>.  The
-	    <a href="http://4-4-8.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
+	    <a href="http://4-4-9.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
 	  </p>
 	  <div id='error'>
 	    {{ error_message }}
@@ -316,6 +307,11 @@
 	    <br /><a href="http://www.squidge.org/peja/cgi-bin/viewstory.php?sid=1234">http://www.squidge.org/peja/cgi-bin/viewstory.php?sid=1234</a>.<br />
 	    This is <i>only</i> for squidge.org/peja, not other parts of squidge.org.
 	  </dd>
+	  <dt>www.checkmated.com</dt>
+	  <dd>
+	    Use the URL of the story's first chapter, such as
+	    <br /><a href="http://www.checkmated.com/story.php?story=10898">http://www.checkmated.com/story.php?story=10898</a>.
+	  </dd>
 	</dl>
 	<p>
 	  A few additional things to know, which will make your life substantially easier:

plugin-defaults.ini

@@ -368,6 +368,14 @@ extratags: FanFiction,Testing,HTML
 ## personal.ini, not defaults.ini.
 #is_adult:true
 
+[www.checkmated.com]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
 [www.fanfiction.net]
 
 [www.ficbook.net]