Package Ida's new archive.skyehawke.com adapter for release.

Add DB error to mw.

app.yaml

@@ -1,6 +1,6 @@
 # ffd-retief-hrd fanfictiondownloader
 application: fanfictiondownloader
-version: 4-4-2
+version: 4-4-5
 runtime: python27
 api_version: 1
 threadsafe: true

calibre-plugin/__init__.py

@@ -27,7 +27,7 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
     description         = 'UI plugin to download FanFiction stories from various sites.'
     supported_platforms = ['windows', 'osx', 'linux']
     author              = 'Jim Miller'
-    version             = (1, 5, 8)
+    version             = (1, 5, 13)
     minimum_calibre_version = (0, 8, 30)
 
     #: This field defines the GUI plugin class that contains all the code

calibre-plugin/about.txt

@@ -18,3 +18,11 @@ group</a> for the downloader.  That covers the web application and CLI, too.
 
 The source for this plugin is available at it's 
 <a href="http://code.google.com/p/fanficdownloader">project home</a>.
+<hr />
+
+<p>
+See the <a href="http://code.google.com/p/fanficdownloader/wiki/FanFictionDownloaderSupportedsites">list of supported sites</a>.
+</p>
+<p>
+Read the <a href="http://code.google.com/p/fanficdownloader/wiki/FanFictionDownloaderFAQs">FAQs</a>.
+</p>

calibre-plugin/config.py

@@ -126,6 +126,10 @@ class ConfigWidget(QWidget):
         self.l = QVBoxLayout()
         self.setLayout(self.l)
 
+        label = QLabel('<a href="http://code.google.com/p/fanficdownloader/wiki/FanFictionDownloaderSupportedsites">List of Supported Sites</a> -- <a href="http://code.google.com/p/fanficdownloader/wiki/FanFictionDownloaderFAQs">FAQs</a>')
+        label.setOpenExternalLinks(True)
+        self.l.addWidget(label)
+
         tab_widget = QTabWidget(self)
         self.l.addWidget(tab_widget)
 

calibre-plugin/ffdl_plugin.py

@@ -506,8 +506,11 @@ make_firstimage_cover:true
             if len(identicalbooks) > 1:
                 raise NotGoingToDownload("More than one identical book--can't tell which to update/overwrite.","minusminus.png")
 
+            ## changed: add new book when CALIBREONLY if none found.
             if collision == CALIBREONLY and not identicalbooks:
-                raise NotGoingToDownload("Not updating Calibre Metadata, no existing book to update.","search_delete_saved.png")
+                collision = ADDNEW
+                options['collision'] = ADDNEW
+            #     raise NotGoingToDownload("Not updating Calibre Metadata, no existing book to update.","search_delete_saved.png")
 
             if len(identicalbooks)>0:
                 book_id = identicalbooks.pop()
@@ -515,7 +518,7 @@ make_firstimage_cover:true
                 book['icon'] = 'edit-redo.png'
 
         if book_id != None and collision != ADDNEW:
-            if options['collision'] in (CALIBREONLY):
+            if collision in (CALIBREONLY):
                 book['comment'] = 'Metadata collected.'
                 # don't need temp file created below.
                 return
@@ -640,12 +643,14 @@ make_firstimage_cover:true
     def _update_books_completed(self, book_list, options={}):
         
         add_list = filter(lambda x : x['good'] and x['added'], book_list)
+        add_ids = [ x['calibre_id'] for x in add_list ]
         update_list = filter(lambda x : x['good'] and not x['added'], book_list)
         update_ids = [ x['calibre_id'] for x in update_list ]
         
         if len(add_list):
             ## even shows up added to searchs.  Nice.
             self.gui.library_view.model().books_added(len(add_list))
+            self.gui.library_view.model().refresh_ids(add_ids)
 
         if update_ids:
             self.gui.library_view.model().refresh_ids(update_ids)
@@ -763,7 +768,13 @@ make_firstimage_cover:true
             if epubmi.cover_data[1] is not None:
                 db.set_cover(book_id, epubmi.cover_data[1])
             #mi.cover = epubmi.cover_data[1]
-                
+
+        # set author link if found.  All current adapters have authorUrl.
+        if 'authorUrl' in book['all_metadata']:
+            autid=db.get_author_id(book['author'])
+            db.set_link_field_for_author(autid, unicode(book['all_metadata']['authorUrl']),
+                                         commit=False, notify=False)
+            
         db.set_metadata(book_id,mi)
 
         # do configured column updates here.

defaults.ini

@@ -112,9 +112,9 @@ zip_output: false
 ## Can include directories.  .zip will be added if not in name somewhere
 zip_filename: ${title}-${siteabbrev}_${storyId}${formatext}.zip
 
-## Normally, try to make the output file name 'safe' by removing
-## invalid filename chars.  Applies to both output_filename &
-## zip_filename.  
+## Normally, try to make the filenames 'safe' by removing invalid
+## filename chars.  Applies to default_cover_image, output_filename &
+## zip_filename.
 allow_unsafe_filename: false
 
 ## entries to make epub subjects and calibre tags
@@ -259,9 +259,18 @@ output_css:
 ## Note that if you enable make_firstimage_cover in [epub], but want
 ## to use default_cover_image for a specific site, use the site:format
 ## section, for example: [www.ficwad.com:epub]
+## default_cover_image is a python string Template string with
+## ${title}, ${author} etc, same as titlepage_entries. Unless
+## allow_unsafe_filename is true, invalid filename chars will be
+## removed from metadata fields
 #default_cover_image:file:///C:/Users/username/Desktop/nook/images/icon.png
+#default_cover_image:file:///C:/Users/username/Desktop/nook/images/${title}/icon.png
 #default_cover_image:http://www.somesite.com/someimage.gif
 
+## some sites include images that we don't ever want becoming the
+## cover image.  This lets you exclude them.
+#cover_exclusion_regexp:/stories/999/images/.*?_trophy.png
+
 ## Resize images down to width, height, preserving aspect ratio.
 ## Nook size, with margin.
 image_max_size: 580, 725
@@ -311,9 +320,16 @@ extratags: FanFiction,Testing,HTML
 #is_adult:true
 
 [fanfiction.mugglenet.com]
-## Some sites do not require a login, but do require the user to
-## confirm they are adult for adult content.  In commandline version,
-## this should go in your personal.ini, not defaults.ini.
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
 #is_adult:true
 
 [fanfiction.portkey.org]
@@ -330,6 +346,12 @@ extratags: FanFiction,Testing,HTML
 #username:YourName
 #password:yourpassword
 
+[nfacommunity.com]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
 ## Some sites also require the user to confirm they are adult for
 ## adult content.  In commandline version, this should go in your
 ## personal.ini, not defaults.ini.
@@ -396,8 +418,6 @@ extratags:
 ## when a password is required rather than prompting every time.
 #fail_on_password: false
 
-[www.gayauthors.org]
-
 [www.harrypotterfanfiction.com]
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
@@ -412,6 +432,16 @@ extratags:
 
 [www.mediaminer.org]
 
+[www.midnightwhispers.ca]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+## some sites include images that we don't ever want becoming the
+## cover image.  This lets you exclude them.
+cover_exclusion_regexp:/stories/999/images/.*?_trophy.png
+
 [www.potionsandsnitches.net]
 
 [www.siye.co.uk]

fanficdownloader/adapters/__init__.py

@@ -44,11 +44,14 @@ import adapter_whoficcom
 import adapter_siyecouk
 import adapter_archiveofourownorg
 import adapter_ficbooknet
-import adapter_gayauthorsorg
 import adapter_portkeyorg
 import adapter_mugglenetcom
 import adapter_hpfandomnet
 import adapter_thequidditchpitchorg
+import adapter_nfacommunitycom
+import adapter_midnightwhispersca
+import adapter_ksarchivecom
+import adapter_archiveskyehawkecom
 
 ## This bit of complexity allows adapters to be added by just adding
 ## importing.  It eliminates the long if/else clauses we used to need

fanficdownloader/adapters/adapter_archiveskyehawkecom.py

@@ -0,0 +1,190 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+
+def getClass():
+    return ArchiveSkyeHawkeComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class ArchiveSkyeHawkeComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/story.php?no='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','ash')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%Y-%m-%d"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'archive.skyehawke.com'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/story.php?no=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/story.php?no=")+r"\d+$"
+
+
+        
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        url = self.url
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        data = self._fetchUrl(url)
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('div', {'class':"story border"}).find('span',{'class':'left'})
+        title=a.text.split('"')[1]
+        self.story.setMetadata('title',title)
+        
+        # Find authorid and URL from... author url.
+        author = a.find('a')
+        self.story.setMetadata('authorId',author['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+author['href'])
+        self.story.setMetadata('author',author.string)
+		
+        authorSoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
+		
+        chapter=soup.find('select',{'name':'chapter'}).findAll('option')
+	
+        for i in range(1,len(chapter)):
+            ch=chapter[i]
+            self.chapterUrls.append((stripHTML(ch),ch['value']))
+		
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+		
+        box=soup.find('div', {'class': "container borderridge"})
+        sum=box.find('span').text
+        self.setDescription(url,sum)
+
+        boxes=soup.findAll('div', {'class': "container bordersolid"})
+        for box in boxes:
+            if box.find('b') != None and box.find('b').text == "History and Story Information":
+			
+                for b in box.findAll('b'):
+                    if "words" in b.nextSibling:
+                        self.story.setMetadata('numWords', b.text)
+                    if "archived" in b.previousSibling:
+                        self.story.setMetadata('datePublished', makeDate(stripHTML(b.text), self.dateformat))
+                    if "updated" in b.previousSibling:
+                        self.story.setMetadata('dateUpdated', makeDate(stripHTML(b.text), self.dateformat))
+                    if "fandom" in b.nextSibling:
+                        self.story.addToList('category', b.text)
+						
+                for br in box.findAll('br'):
+                    br.replaceWith('split')
+                genre=box.text.split("Genre:")[1].split("split")[0]
+                if not "Unspecified" in genre:
+                    self.story.addToList('genre',genre)
+			
+			
+            if box.find('span') != None and box.find('span').text == "WARNING":
+			
+                rating=box.findAll('span')[1]
+                rating.find('br').replaceWith('split')
+                rating=rating.text.replace("This story is rated",'').split('split')[0]
+                self.story.setMetadata('rating',rating)
+                logging.debug(self.story.getMetadata('rating'))
+			
+                warnings=box.find('ol')
+                if warnings != None:
+                    warnings=warnings.text.replace(']', '').replace('[', '').split('  ')
+                    for warning in warnings:
+                        self.story.addToList('warnings',warning)
+	
+	
+        for asoup in authorSoup.findAll('div', {'class':"story bordersolid"}):
+            if asoup.find('a')['href'] == 'story.php?no='+self.story.getMetadata('storyId'):
+                if '[ Completed ]' in asoup.text:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+                chars=asoup.findNext('div').text.split('Characters')[1].split(']')[0]
+                for char in chars.split(','):
+                    if not "None" in char:
+                        self.story.addToList('characters',char)
+                break
+	
+	
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div',{'class':"chapter bordersolid"}).findNext('div').findNext('div')
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_gayauthorsorg.py

@@ -1,205 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2011 Fanficdownloader team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import datetime
-import logging
-import re
-import urllib2
-from urllib import unquote
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter, makeDate
-
-def getClass():
-    return GayAuthorsAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class GayAuthorsAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["utf8",
-                       "Windows-1252"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-       
-		
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[3])
-        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-
-        # unqoute, change '_' and ' ' to '-', downcase, and remove non-[a-z0-9-]
-        authid = unquote(self.parsedUrl.path.split('/',)[2])
-        authid = authid.lower().replace('_','-').replace(' ','-')
-        authid = re.sub(r"[^a-z0-9-]","",authid)
-        
-        self.story.setMetadata('authorId',authid)
-        logging.debug("authorId: (%s)"%self.story.getMetadata('authorId'))
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/story/'+self.story.getMetadata('authorId') + '/' + self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','ga')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%d %b %Y"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'www.gayauthors.org'
-
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/story/author/storytitle"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/story/")+r".*?/\w+.*?$"
-
-    
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-
-        url = self.url
-        logging.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        msoup = soup.find('div', {'class' : 'story'})
-        if msoup == None:
-            msoup = soup.find('div', {'class' : 'story ispinned'})
-        csoup = soup.find('div', {'id' : 'story_chapters'})
-        
-        ## Title
-        a = msoup.find('span', {'class' : 'title'})
-        title=a.find('span', {'itemprop' : 'name'})
-        self.story.setMetadata('title',title.text)
-        try:
-            # Find Series name from series URL.
-            series = a.find('span',{'class':"description"})
-            series_name = series.find('a')
-            series_name.extract()
-            series_index = int(series.text.split(' ')[1])
-            self.setSeries(series_name.text, series_index)
-            
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-        
-        # Find authorid and URL from... author url.
-        a = msoup.find('a', href=re.compile(r'/author/'+self.story.getMetadata('authorId')))
-        self.story.setMetadata('authorUrl',a['href'])
-        self.story.setMetadata('author',a.text)
-		
-
-        # Find the chapters:
-        spans=csoup.findAll('span', {'class' : 'desc chapter-info'})
-        for span in spans:
-            span.extract()
-        for chapter in csoup.findAll('a'):
-            # just in case there's tags, like <i> in chapter titles.
-            a=chapter['href'].split(self.story.getMetadata('author'))
-            a=a[0]+self.story.getMetadata('authorId')+a[1]
-            self.chapterUrls.append((stripHTML(chapter),a))
-
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-		
-        cats = msoup.findAll('a', href=re.compile(r'/browse/list/page__filtertype_0__category\w+$'))
-        for cat in cats:
-            self.story.addToList('category',cat.text)
-		
-        genres = msoup.findAll('a', href=re.compile(r'/browse/list/page__filtertype_1__genre\w+$'))
-        for genre in genres:
-            self.story.addToList('genre',genre.text)
-        genres = msoup.findAll('a', href=re.compile(r'/browse/list/page__filtertype_2__tag\w+$'))
-        for genre in genres:
-            self.story.addToList('genre',genre.text)
-			
-        status = msoup.find('a', href=re.compile(r'/browse/list/page__filtertype_3__status\w+$'))
-        self.story.setMetadata('status',status.text)
-		
-        rating = msoup.find('a', href=re.compile(r'/browse/list/page__filtertype_4__rating\w+$'))
-        self.story.setMetadata('rating',rating.text)
-		
-        summary = msoup.find('span', {'itemprop' : 'description'})
-        self.setDescription(self.url,summary.text)
-        #self.story.setMetadata('description',summary.text)
-	
-
-        stats = msoup.find('dl',{'class':'info'})
-        dt = stats.findAll('dt')
-        dd = stats.findAll('dd')
-        for x in range(0,len(dt)):
-            label = dt[x].text
-            value = dd[x].text
-
-            if 'Words:' in label:
-                self.story.setMetadata('numWords', value)
-
-            if 'Published:' in label:
-                date=stripHTML(value.split(' - ')[0])
-                if ',' in date:
-                    date=datetime.date.today().strftime(self.dateformat)
-                self.story.setMetadata('datePublished', makeDate(date, self.dateformat))
-				
-            if 'Updated:' in label:
-                date=stripHTML(value.split(' - ')[0])
-                if ',' in date:
-                    date=datetime.date.today().strftime(self.dateformat)
-                self.story.setMetadata('dateUpdated', makeDate(date, self.dateformat))
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logging.debug('Getting chapter text from: %s' % url)
-
-        data = self._fetchUrl(url)
-        data = data[data.index("<div id='chapter-content'>"):]
-        soup = bs.BeautifulSoup(data) # this one's happier with Soup, not StoneSoup for some reason.
-        
-        div = soup.find('div', {'id' : 'chapter-content'})
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_ksarchivecom.py

@@ -0,0 +1,306 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+# Search for XXX comments--that's where things are most likely to need changing.
+
+# This function is called by the downloader in all adapter_*.py files
+# in this dir to register the adapter class.  So it needs to be
+# updated to reflect the class below it.  That, plus getSiteDomain()
+# take care of 'Registering'.
+def getClass():
+    return KSArchiveComAdapter # XXX
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class KSArchiveComAdapter(BaseSiteAdapter): # XXX
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','ksa') # XXX
+
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","Star Trek") # XXX
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%b/%d/%Y" # XXX
+            
+    @classmethod
+    def getAcceptDomains(cls):
+        return ['www.ksarchive.com','ksarchive.com']
+
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'ksarchive.com' # XXX
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return "http://(www.)?"+re.escape(self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+
+            # Furthermore, there's a couple sites now with more than
+            # one warning level for different ratings.  And they're
+            # fussy about it.  midnightwhispers has three: 10, 3 & 5.
+            # we'll try 5 first.
+            addurl = "&ageconsent=ok&warning=2" # XXX
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        # The actual text that is used to announce you need to be an
+        # adult varies from site to site.  Again, print data before
+        # the title search to troubleshoot.
+            
+        # Since the warning text can change by warning level, let's
+        # look for the warning pass url.  ksarchive uses
+        # &warning= -- actually, so do other sites.  Must be an
+        # eFiction book.
+            
+        # viewstory.php?sid=1882&warning=4
+        # viewstory.php?sid=1654&ageconsent=ok&warning=5
+        #print data
+        #m = re.search(r"'viewstory.php\?sid=1882(&warning=4)'",data)
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid & error in url.
+                addurl = addurl.replace("&","&")
+                url = self.url+'&index=1'+addurl
+                logging.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',stripHTML(a)) # title's inside a <b> tag.
+
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                catstext = [cat.string for cat in cats]
+                for cat in catstext:
+                    if cat.string.strip() in ('Poetry','Essays'):
+                        self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                self.story.addToList('characters','Kirk')
+                self.story.addToList('characters','Spock')
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                charstext = [char.string for char in chars]
+                for char in charstext:
+                    self.story.addToList('characters',char.string)
+
+            ## Not all sites use Genre, but there's no harm to
+            ## leaving it in.  Check to make sure the type_id number
+            ## is correct, though--it's site specific.
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) # XXX
+                genrestext = [genre.string for genre in genres]
+                self.genre = ', '.join(genrestext)
+                for genre in genrestext:
+                    self.story.addToList('genre',genre.string)
+
+            ## In addition to Genre (which is very site specific) KSA
+            ## has 'Story Type', which is much more what most sites
+            ## call genre.
+            if 'Story Type' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=5')) # XXX
+                genrestext = [genre.string for genre in genres]
+                self.genre = ', '.join(genrestext)
+                for genre in genrestext:
+                    self.story.addToList('genre',genre.string)
+
+            ## Not all sites use Warnings, but there's no harm to
+            ## leaving it in.  Check to make sure the type_id number
+            ## is correct, though--it's site specific.
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
+                warningstext = [warning.string for warning in warnings]
+                self.warning = ', '.join(warningstext)
+                for warning in warningstext:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        data = self._fetchUrl(url)
+        soup = bs.BeautifulStoneSoup(data,
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            if "A fatal MySQL error was encountered" in data:
+                raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Database error on the site reported!" % url)
+            else:
+                raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_midnightwhispersca.py

@@ -0,0 +1,289 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+# Search for XXX comments--that's where things are most likely to need changing.
+
+# This function is called by the downloader in all adapter_*.py files
+# in this dir to register the adapter class.  So it needs to be
+# updated to reflect the class below it.  That, plus getSiteDomain()
+# take care of 'Registering'.
+def getClass():
+    return MidnightwhispersCaAdapter # XXX
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class MidnightwhispersCaAdapter(BaseSiteAdapter): # XXX
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','mw') # XXX
+
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","Queer as Folk") # XXX
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%B %d, %Y" # XXX
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.midnightwhispers.ca' # XXX
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+
+            # Furthermore, there's a couple sites now with more than
+            # one warning level for different ratings.  And they're
+            # fussy about it.  midnightwhispers has three: 10, 3 & 5.
+            # we'll try 5 first.
+            addurl = "&ageconsent=ok&warning=5" # XXX
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        # The actual text that is used to announce you need to be an
+        # adult varies from site to site.  Again, print data before
+        # the title search to troubleshoot.
+            
+        # Since the warning text can change by warning level, let's
+        # look for the warning pass url.  nfacommunity uses
+        # &warning= -- actually, so do other sites.  Must be an
+        # eFiction book.
+            
+        # viewstory.php?sid=1882&warning=4
+        # viewstory.php?sid=1654&ageconsent=ok&warning=5
+        #print data
+        #m = re.search(r"'viewstory.php\?sid=1882(&warning=4)'",data)
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid & error in url.
+                addurl = addurl.replace("&","&")
+                url = self.url+'&index=1'+addurl
+                logging.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',stripHTML(a)) # title's inside a <b> tag.
+
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                catstext = [cat.string for cat in cats]
+                for cat in catstext:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                charstext = [char.string for char in chars]
+                for char in charstext:
+                    self.story.addToList('characters',char.string)
+
+            ## Not all sites use Genre, but there's no harm to
+            ## leaving it in.  Check to make sure the type_id number
+            ## is correct, though--it's site specific.
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
+                genrestext = [genre.string for genre in genres]
+                self.genre = ', '.join(genrestext)
+                for genre in genrestext:
+                    self.story.addToList('genre',genre.string)
+
+            ## Not all sites use Warnings, but there's no harm to
+            ## leaving it in.  Check to make sure the type_id number
+            ## is correct, though--it's site specific.
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
+                warningstext = [warning.string for warning in warnings]
+                self.warning = ', '.join(warningstext)
+                for warning in warningstext:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        data = self._fetchUrl(url)
+        soup = bs.BeautifulStoneSoup(data,
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            if "A fatal MySQL error was encountered" in data:
+                raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Database error on the site reported!" % url)
+            else:
+                raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_mugglenetcom.py

@@ -79,6 +79,41 @@ class MuggleNetComAdapter(BaseSiteAdapter): # XXX
     def getSiteURLPattern(self):
         return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
 
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if "class='errortext'>Registered Users Only" in data \
+                or 'There is no such account on our website' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['cookiecheck'] = '1'
+        params['submit'] = 'Submit'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login&sid='+self.story.getMetadata('storyId')
+        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "Member Account" not in d : #Member Account
+            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
     ## Getting the chapter list and the meta data, plus 'is adult' checking.
     def extractChapterUrlsAndMetadata(self):
 
@@ -87,7 +122,8 @@ class MuggleNetComAdapter(BaseSiteAdapter): # XXX
             # If the title search below fails, there's a good chance
             # you need a different number.  print data at that point
             # and see what the 'click here to continue' url says.
-            addurl = "&warning=5" # XXX
+            # http://fanfiction.mugglenet.com/viewstory.php?sid=91079&ageconsent=ok&warning=3
+            addurl = "&ageconsent=ok&warning=3" # XXX  &warning=5
         else:
             addurl=""
 
@@ -104,13 +140,47 @@ class MuggleNetComAdapter(BaseSiteAdapter): # XXX
             else:
                 raise e
 
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+            #print("\nurl:%s\ndata:\n%s\n"%(url,data))
+
         # The actual text that is used to announce you need to be an
         # adult varies from site to site.  Again, print data before
         # the title search to troubleshoot.
-        if "This story may contain some sexuality, violence and or profanity not suitable for younger readers." in data: # XXX 
-            raise exceptions.AdultCheckRequired(self.url)
+            
+        # Since the warning text can change by warning level, let's
+        # look for the warning pass url.  nfacommunity uses
+        # &warning= -- actually, so do other sites.  Must be an
+        # eFiction book.
+            
+        # viewstory.php?sid=1882&warning=4
+        # viewstory.php?sid=1654&ageconsent=ok&warning=5
+        #print data
+        #m = re.search(r"'viewstory.php\?sid=1882(&warning=4)'",data)
+        m = re.search(r"'viewstory.php\?sid=%s((?:&ageconsent=ok)?&warning=\d+)'"%self.story.getMetadata('storyId'),data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid & error in url.
+                addurl = addurl.replace("&","&")
+                url = self.url+'&index=1'+addurl
+                logging.debug("URL 2nd try: "+url)
 
-        # Dunno if this site uses this or not.
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
         if "Access denied. This story has not been validated by the adminstrators of this site." in data:
             raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
             
@@ -137,6 +207,16 @@ class MuggleNetComAdapter(BaseSiteAdapter): # XXX
 
         self.story.setMetadata('numChapters',len(self.chapterUrls))
 
+
+        # Not good enough-- content can contain a ('), which ends the content prematurely.
+        # metadesc = soup.find('meta',{'name':'description'})
+        # print("removeAllEntities(metadesc['content']):\n%s\n"%removeAllEntities(metadesc['content']))
+        start='<span class="label">Summary: </span>'
+        end='<span class="label">Rated:</span>'
+        summarydata = data[data.index(start)+len(start):data.index(end)]
+        #print("summarydata:\n%s\n"%summarydata)
+        self.setDescription(url,bs.BeautifulSoup(summarydata))
+        
         # eFiction sites don't help us out a lot with their meta data
         # formating, so it's a little ugly.
 
@@ -153,13 +233,14 @@ class MuggleNetComAdapter(BaseSiteAdapter): # XXX
             value = labelspan.nextSibling
             label = labelspan.string
 
-            if 'Summary' in label:
-                ## Everything until the next span class='label'
-                svalue = ""
-                while not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
-                    value = value.nextSibling
-                self.setDescription(url,svalue)
+            # not good enough--poorly formated summary html will break it.
+            # if 'Summary' in label:
+            #     ## Everything until the next span class='label'
+            #     svalue = ""
+            #     while not defaultGetattr(value,'class') == 'label':
+            #         svalue += str(value)
+            #         value = value.nextSibling
+            #     self.setDescription(url,svalue)
                 #self.story.setMetadata('description',stripHTML(svalue))
 
             if 'Rated' in label:

fanficdownloader/adapters/adapter_nfacommunitycom.py

@@ -0,0 +1,289 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+# Search for XXX comments--that's where things are most likely to need changing.
+
+# This function is called by the downloader in all adapter_*.py files
+# in this dir to register the adapter class.  So it needs to be
+# updated to reflect the class below it.  That, plus getSiteDomain()
+# take care of 'Registering'.
+def getClass():
+    return NfaCommunityComAdapter # XXX
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class NfaCommunityComAdapter(BaseSiteAdapter): # XXX
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','nfa') # XXX
+
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","NCIS") # XXX
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%m/%d/%Y" # XXX
+            
+    @classmethod
+    def getAcceptDomains(cls):
+        return ['www.nfacommunity.com','nfacommunity.com']
+
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'nfacommunity.com' # XXX
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return "http://(www.)?"+re.escape(self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+
+            # Furthermore, there's a couple sites now with more than
+            # one warning level for different ratings.  And they're
+            # fussy about it.  nfacommunity has two: 4 & 5.
+            # we'll try 5 first.
+            addurl = "&ageconsent=ok&warning=5" # XXX
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        # The actual text that is used to announce you need to be an
+        # adult varies from site to site.  Again, print data before
+        # the title search to troubleshoot.
+            
+        # Since the warning text can change by warning level, let's
+        # look for the warning pass url.  nfacommunity uses
+        # &warning= -- actually, so do other sites.  Must be an
+        # eFiction book.
+            
+        # viewstory.php?sid=1882&warning=4
+        # viewstory.php?sid=1654&ageconsent=ok&warning=5
+        #print data
+        #m = re.search(r"'viewstory.php\?sid=1882(&warning=4)'",data)
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid & error in url.
+                addurl = addurl.replace("&","&")
+                url = self.url+'&index=1'+addurl
+                logging.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                catstext = [cat.string for cat in cats]
+                for cat in catstext:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                charstext = [char.string for char in chars]
+                for char in charstext:
+                    self.story.addToList('characters',char.string)
+
+            ## Not all sites use Genre, but there's no harm to
+            ## leaving it in.  Check to make sure the type_id number
+            ## is correct, though--it's site specific.
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
+                genrestext = [genre.string for genre in genres]
+                self.genre = ', '.join(genrestext)
+                for genre in genrestext:
+                    self.story.addToList('genre',genre.string)
+
+            ## Not all sites use Warnings, but there's no harm to
+            ## leaving it in.  Check to make sure the type_id number
+            ## is correct, though--it's site specific.
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
+                warningstext = [warning.string for warning in warnings]
+                self.warning = ', '.join(warningstext)
+                for warning in warningstext:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_portkeyorg.py

@@ -253,14 +253,18 @@ class PortkeyOrgAdapter(BaseSiteAdapter): # XXX
     def getChapterText(self, url):
 
         logging.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+        data = self._fetchUrl(url)
+        soup = bs.BeautifulStoneSoup(data,
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
 
         #print("soup:%s"%soup)
-        td = soup.find('td', {'class' : 'story'})
+        tag = soup.find('td', {'class' : 'story'})
+        if tag == None and "<center><b>Chapter does not exist!</b></center>" in data:
+            print("Chapter is missing at: %s"%url)
+            return  self.utf8FromSoup(url,bs.BeautifulStoneSoup("<div><p><center><b>Chapter does not exist!</b></center></p><p>Chapter is missing at: <a href='%s'>%s</a></p></div>"%(url,url)))
+        tag.name='div' # force to be a div to avoid problems with nook.
 
-        centers = td.findAll('center')
+        centers = tag.findAll('center')
         # first two and last two center tags are some script, 'report
         # story', 'report story' and an ad.
         centers[0].extract()
@@ -268,7 +272,7 @@ class PortkeyOrgAdapter(BaseSiteAdapter): # XXX
         centers[-1].extract()
         centers[-2].extract()
 
-        if None == td:
+        if None == tag:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
     
-        return self.utf8FromSoup(url,td)
+        return self.utf8FromSoup(url,tag)

fanficdownloader/adapters/adapter_twiwritenet.py

@@ -38,6 +38,7 @@ class TwiwriteNetSiteAdapter(BaseSiteAdapter):
                                # iso-8859-1 (and some that claim to be
                                # utf8) are really windows-1252.
         self.story.addToList("category","Twilight")
+        self.is_adult = False
         self.username = "NoneGiven" # if left empty, twiwrite.net doesn't return any message at all.
         self.password = ""
         
@@ -99,7 +100,16 @@ class TwiwriteNetSiteAdapter(BaseSiteAdapter):
 
     def extractChapterUrlsAndMetadata(self):
 
-        url = self.url+'&index=1'
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=1" # XXX
+        else:
+            addurl=""
+            
+        url = self.url+'&index=1'+addurl
         logging.debug("URL: "+url)
 
         try:
@@ -118,6 +128,9 @@ class TwiwriteNetSiteAdapter(BaseSiteAdapter):
         if "Access denied. This story has not been validated by the adminstrators of this site." in data:
             raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
 
+        if "Contains Explicit Content for mature adults only! May contain graphic violence, mature sexual situations, and explicit language. Read with caution." in data:
+            raise exceptions.AdultCheckRequired(self.url)
+        
         # problems with some stories, but only in calibre.  I suspect
         # issues with different SGML parsers in python.  This is a
         # nasty hack, but it works.
@@ -139,7 +152,7 @@ class TwiwriteNetSiteAdapter(BaseSiteAdapter):
         # Find the chapters:
         for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
             # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']))
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
 
         self.story.setMetadata('numChapters',len(self.chapterUrls))
 

fanficdownloader/adapters/base_adapter.py

@@ -215,7 +215,9 @@ class BaseSiteAdapter(Configurable):
                     self.getConfig('default_cover_image'):
                 self.story.addImgUrl(self,
                                      None,
-                                     self.getConfig('default_cover_image'),
+                                     #self.getConfig('default_cover_image'),
+                                     self.story.formatFileName(self.getConfig('default_cover_image'),
+                                                               self.getConfig('allow_unsafe_filename')),
                                      self._fetchUrlRaw,
                                      cover=True)
         return self.story
@@ -287,7 +289,8 @@ class BaseSiteAdapter(Configurable):
                 # some pre-existing epubs have img tags that had src stripped off.
                 if img.has_key('src'):
                     img['longdesc']=img['src']
-                    img['src']=self.story.addImgUrl(self,url,img['src'],fetch)
+                    img['src']=self.story.addImgUrl(self,url,img['src'],fetch,
+                                                    coverexclusion=self.getConfig('cover_exclusion_regexp'))
 
         for attr in soup._getAttrMap().keys():
             if attr not in acceptable_attributes:
@@ -328,10 +331,6 @@ class BaseSiteAdapter(Configurable):
         # This is primarily for epub updates.
         return re.sub(r"</?body>\r?\n?","",retval)
 
-fullmon = {"January":"01", "February":"02", "March":"03", "April":"04", "May":"05",
-           "June":"06","July":"07", "August":"08", "September":"09", "October":"10",
-           "November":"11", "December":"12" }
-
 def cachedfetch(realfetch,cache,url):
     if url in cache:
         print("cache hit")
@@ -339,6 +338,9 @@ def cachedfetch(realfetch,cache,url):
     else:
         return realfetch(url)
     
+fullmon = {"January":"01", "February":"02", "March":"03", "April":"04", "May":"05",
+           "June":"06","July":"07", "August":"08", "September":"09", "October":"10",
+           "November":"11", "December":"12" }
 
 def makeDate(string,format):
     # Surprise!  Abstracting this turned out to be more useful than

fanficdownloader/htmlcleanup.py

@@ -23,11 +23,16 @@ def _unirepl(match):
         radix=16
     else:
         radix=10
-    value = int(match.group(2), radix )
-    return unichr(value)
+    value = int(match.group(2), radix)
+    return "%s%s"%(unichr(value),match.group(3))
 
 def _replaceNumberEntities(data):
-    p = re.compile(r'&#(x?)([0-9a-fA-F]+);')
+    # The same brokenish entity parsing in SGMLParser that inserts ';'
+    # after non-entities will also insert ';' incorrectly after number
+    # entities, including part of the next word if it's a-z.
+    # "Don't&#8212ever&#8212do&#8212that&#8212again," becomes
+    # "Don't&#8212e;ver&#8212d;o—that&#8212a;gain,"
+    p = re.compile(r'&#(x?)([0-9a-fA-F]{,4})([0-9a-fA-F]*);')
     return p.sub(_unirepl, data)
 
 def _replaceNotEntities(data):
@@ -50,9 +55,6 @@ def removeAllEntities(text):
     return removeEntities(text).replace('&lt;', '<').replace('&gt;', '>').replace('&amp;', '&')
 
 def removeEntities(text):
-
-    # replace numeric versions of [&<>] with named versions,
-    # then replace named versions with actual characters,
     
     if text is None:
         return ""
@@ -67,6 +69,8 @@ def removeEntities(text):
         except UnicodeEncodeError, e:
             t = text
     text = t 
+    # replace numeric versions of [&<>] with named versions,
+    # then replace named versions with actual characters,
     text = re.sub(r'&#0*38;','&amp;',text)
     text = re.sub(r'&#0*60;','&lt;',text)
     text = re.sub(r'&#0*62;','&gt;',text)

fanficdownloader/story.py

@@ -17,6 +17,7 @@
 
 import os, re
 import urlparse
+import string
 from math import floor
 
 from htmlcleanup import conditionalRemoveEntities, removeAllEntities
@@ -74,6 +75,9 @@ except:
                 export = True
 
             if normalize_format_name(img.format) != "jpg":
+                if img.mode == "P":
+                    # convert pallete gifs to RGB so jpg save doesn't fail.
+                    img = img.convert("RGB")
                 export = True
 
             if export:
@@ -212,7 +216,7 @@ class Story:
         for (p,v) in self.replacements:
             if (isinstance(value,str) or isinstance(value,unicode)) and re.match(p,value):
                 value = re.sub(p,v,value)                
-        return value;
+        return value
         
     def getMetadata(self, key, removeallentities=False):
         value = None
@@ -275,9 +279,23 @@ class Story:
         "Chapters will be tuples of (title,html)"
         return self.chapters
 
+    def formatFileName(self,template,allowunsafefilename=True):
+        values = origvalues = self.getAllMetadata()
+        # fall back default:
+        if not template:
+            template="${title}-${siteabbrev}_${storyId}${formatext}"
+
+        if not allowunsafefilename:
+            values={}
+            pattern = re.compile(r"[^a-zA-Z0-9_\. \[\]\(\)&'-]+")
+            for k in origvalues.keys():
+                values[k]=re.sub(pattern,'_', removeAllEntities(self.getMetadata(k)))
+
+        return string.Template(template).substitute(values).encode('utf8')
+
     # pass fetch in from adapter in case we need the cookies collected
     # as well as it's a base_story class method.
-    def addImgUrl(self,configurable,parenturl,url,fetch,cover=False):
+    def addImgUrl(self,configurable,parenturl,url,fetch,cover=False,coverexclusion=None):
         
         url = url.strip() # ran across an image with a space in the
                           # src. Browser handled it, so we'd better, too.
@@ -298,11 +316,17 @@ class Story:
                      url,
                      '','',''))
             else:
+                toppath=""
+                if parsedUrl.path.endswith("/"):
+                    toppath = parsedUrl.path
+                else:
+                    toppath = parsedUrl.path[:parsedUrl.path.rindex('/')]
                 imgurl = urlparse.urlunparse(
                     (parsedUrl.scheme,
                      parsedUrl.netloc,
-                     parsedUrl.path + url,
+                     toppath + '/' + url,
                      '','',''))
+                #print("\n===========\nparsedUrl.path:%s\ntoppath:%s\nimgurl:%s\n\n"%(parsedUrl.path,toppath,imgurl))
 
         prefix='ffdl'
         if imgurl not in self.imgurls:
@@ -330,9 +354,14 @@ class Story:
             else:
                 self.imgurls.append(imgurl)
                 # First image, copy not link because calibre will replace with it's cover.
-                if len(self.imgurls)==1 and \
+                # Only if: No cover already AND
+                #          make_firstimage_cover AND
+                #          NOT never_make_cover AND
+                #          either no coverexclusion OR coverexclusion doesn't match
+                if self.cover == None and \
                         configurable.getConfig('make_firstimage_cover') and \
-                        not configurable.getConfig('never_make_cover'):
+                        not configurable.getConfig('never_make_cover') and \
+                        (not coverexclusion or not re.search(coverexclusion,imgurl)):
                     newsrc = "images/cover.%s"%ext
                     self.cover=newsrc
                     self.imgtuples.append({'newsrc':newsrc,'mime':mime,'data':data})

fanficdownloader/writers/base_writer.py

@@ -18,7 +18,6 @@
 import re
 import os.path
 import datetime
-import string
 import StringIO
 import zipfile
 from zipfile import ZipFile, ZIP_DEFLATED
@@ -123,24 +122,10 @@ class BaseStoryWriter(Configurable):
             return self.getBaseFileName()
 
     def getBaseFileName(self):
-        return self.formatFileName(self.getConfig('output_filename'))
+        return self.story.formatFileName(self.getConfig('output_filename'),self.getConfig('allow_unsafe_filename'))
     
     def getZipFileName(self):
-        return self.formatFileName(self.getConfig('zip_filename'))
-
-    def formatFileName(self,template):
-        values = origvalues = self.story.getAllMetadata()
-        # fall back default:
-        if not template:
-            template="${title}-${siteabbrev}_${storyId}${formatext}"
-
-        if not self.getConfig('allow_unsafe_filename'):
-            values={}
-            pattern = re.compile(r"[^a-zA-Z0-9_\. \[\]\(\)&'-]+")
-            for k in origvalues.keys():
-                values[k]=re.sub(pattern,'_', removeAllEntities(self.story.getMetadata(k)))
-
-        return string.Template(template).substitute(values).encode('utf8')
+        return self.story.formatFileName(self.getConfig('zip_filename'),self.getConfig('allow_unsafe_filename'))
 
     def _write(self, out, text):
         out.write(text.encode('utf8'))

index.html

@@ -54,35 +54,19 @@
 	      much easier. </p>
 	  </div>
 	  <!-- put announcements here, h3 is a good title size. -->
-	  <h3>Outages</h3>
+	  <h3>New Site Added</h3>
 	  <p>
-	    This web service has been having almost daily outages
-	    recently due to problems with running out of quota for the
-	    application.  We are working to improve the situation.
-	  </p>
-	  <h3>Update - New Sites</h3>
-	  <p>
-	    Four new sites have been added:
-	    <ul>
-	      <li>fanfiction.mugglenet.com</li>
-	      <li>fanfiction.portkey.org</li>
-	      <li>thequidditchpitch.org</li>
-	      <li>www.hpfandom.net</li>
-	    </ul>
-	    These are the four largest/most active of the sites that have been requested lately.
+	    Support for archive.skyehawke.com has been added.  Thanks to Ida Leter for implementing this.
 	  </p>
 	  <p>
-	    Another site that's been requested a few times is
-	    adultfanfiction.net.  None of the current developers are
-	    interested in writing code to support it because of the
-	    ominous legal warnings and requirements to access the
-	    site.
-	  </p>
+	    Questions?  Check out our 
+	    <a href="http://code.google.com/p/fanficdownloader/wiki/FanFictionDownloaderFAQs">FAQs</a>.
+	    </p>
 	  <p>
 	    If you have any problems with this application, please
 	    report them in
 	    the <a href="http://groups.google.com/group/fanfic-downloader">FanFictionDownLoader Google Group</a>.  The
-	    <a href="http://4-4-0.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
+	    <a href="http://4-4-4.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
 	  </p>
 	  <div id='error'>
 	    {{ error_message }}
@@ -146,6 +130,13 @@
 	  </p>
       </div>
       <div id='helpbox'>
+	<h3>Supported sites:</h3>
+	<p>
+	  There's a 
+ 	  <a href="http://code.google.com/p/fanficdownloader/wiki/FanFictionDownloaderSupportedsites">Supported
+ 	  Sites</a> page in our wiki.  If you have a site you'd like
+ 	  to see supported, please check there first.
+	</p>
 	<dl>
 	  <dt>fictionalley.org</dt>
 	  <dd>
@@ -257,12 +248,6 @@
 	    <br /><a href="http://ficbook.net/readfic/93626">http://ficbook.net/readfic/93626</a>.
 	    <br /><a href="http://ficbook.net/readfic/93626/246417#part_content">http://ficbook.net/readfic/93626/246417#part_content</a>.
 	  </dd>
-	  <dt>gayauthors.org</dt>
-	  <dd>
-	    Use the URL of the story, or one of it's chapters, such as
-	    <br /><a href="http://www.gayauthors.org/story/mark-arbour/stvincent">http://www.gayauthors.org/story/mark-arbour/stvincent</a>.
-	    <br /><a href="http://www.gayauthors.org/story/Mark%20Arbour/stvincent/7">http://www.gayauthors.org/story/Mark Arbour/stvincent/7</a>.
-	  </dd>
 	  <dt>fanfiction.mugglenet.com</dt>
 	  <dd>
 	    Use the URL of the story's chapter list, such as
@@ -283,17 +268,38 @@
 	    Use the URL of the story's chapter list, such as
 	    <br /><a href="http://fanfiction.portkey.org/story/123">http://fanfiction.portkey.org/story/123</a>.
 	  </dd>
+	  <dt>nfacommunity.com</dt>
+	  <dd>
+	    Use the URL of the story's chapter list, such as
+	    <br /><a href="http://nfacommunity.com/viewstory.php?sid=1654">http://nfacommunity.com/viewstory.php?sid=1654</a>.
+	  </dd>
+	  <dt>www.midnightwhispers.ca</dt>
+	  <dd>
+	    Use the URL of the story's chapter list, such as
+	    <br /><a href="http://www.midnightwhispers.ca/viewstory.php?sid=1124">http://www.midnightwhispers.ca/viewstory.php?sid=1124</a>.
+	  </dd>
+	  <dt>ksarchive.com</dt>
+	  <dd>
+	    Use the URL of the story's chapter list, such as
+	    <br /><a href="http://ksarchive.com/viewstory.php?sid=1124">http://ksarchive.com/viewstory.php?sid=1124</a>.
+	  </dd>
+	  <dt>archive.skyehawke.com</dt>
+	  <dd>
+	    Use the URL of the story's summary, such as
+	    <br /><a href="http://archive.skyehawke.com/story.php?no=17466">http://archive.skyehawke.com/story.php?no=17466</a>.
+	  </dd>
 	</dl>
 	<p>
 	  A few additional things to know, which will make your life substantially easier:
 	</p>
 	<ol>
 	  <li>
-	    First thing to know: I do not use your Google login and password. In fact, all I know about it is your ID &ndash; password
+	    First thing to know: We do not use your Google login and password. In fact, all we know about it is your ID &ndash; password
 	    is being verified by Google and is absolutely, totally unknown to anyone but you.
 	  </li>
 	  <li>
-	    Small <a href="http://www.sigizmund.com/reading-fanfiction-off-line-in-stanza-and-oth">post written by me</a>
+	    
+	    Small <a href="http://www.sigizmund.com/reading-fanfiction-off-line-in-stanza-and-oth">post written by Roman</a>
 	    &mdash; how to read fiction in Stanza or any other ebook reader.
 	  </li>
 	  <li>
@@ -309,7 +315,7 @@
 	  </li>
 	  <li>
 	    If you think that something that should work in fact doesn't, post a message to
-	    our <a href="http://groups.google.com/group/fanfic-downloader">Google Group</a>. I also encourage you to join it so
+	    our <a href="http://groups.google.com/group/fanfic-downloader">Google Group</a>. we also encourage you to join it so
 	    you will find out about latest updates and fixes as soon as possible
 	  </li>
 	</ol>

plugin-defaults.ini

@@ -231,9 +231,18 @@ output_css:
 ## Note that if you enable make_firstimage_cover in [epub], but want
 ## to use default_cover_image for a specific site, use the site:format
 ## section, for example: [www.ficwad.com:epub]
+## default_cover_image is a python string Template string with
+## ${title}, ${author} etc, same as titlepage_entries. Unless
+## allow_unsafe_filename is true, invalid filename chars will be
+## removed from metadata fields
 #default_cover_image:file:///C:/Users/username/Desktop/nook/images/icon.png
+#default_cover_image:file:///C:/Users/username/Desktop/nook/images/${title}/icon.png
 #default_cover_image:http://www.somesite.com/someimage.gif
 
+## some sites include images that we don't ever want becoming the
+## cover image.  This lets you exclude them.
+#cover_exclusion_regexp:/stories/999/images/.*?_trophy.png
+
 ## Resize images down to width, height, preserving aspect ratio.
 ## Nook size, with margin.
 image_max_size: 580, 725
@@ -283,9 +292,16 @@ extratags: FanFiction,Testing,HTML
 #is_adult:true
 
 [fanfiction.mugglenet.com]
-## Some sites do not require a login, but do require the user to
-## confirm they are adult for adult content.  In commandline version,
-## this should go in your personal.ini, not defaults.ini.
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
 #is_adult:true
 
 [fanfiction.portkey.org]
@@ -302,6 +318,12 @@ extratags: FanFiction,Testing,HTML
 #username:YourName
 #password:yourpassword
 
+[nfacommunity.com]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
 ## Some sites also require the user to confirm they are adult for
 ## adult content.  In commandline version, this should go in your
 ## personal.ini, not defaults.ini.
@@ -365,8 +387,6 @@ extratags:
 ## when a password is required rather than prompting every time.
 #fail_on_password: false
 
-[www.gayauthors.org]
-
 [www.harrypotterfanfiction.com]
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
@@ -381,6 +401,16 @@ extratags:
 
 [www.mediaminer.org]
 
+[www.midnightwhispers.ca]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+## some sites include images that we don't ever want becoming the
+## cover image.  This lets you exclude them.
+cover_exclusion_regexp:/stories/999/images/.*?_trophy.png
+
 [www.potionsandsnitches.net]
 
 [www.siye.co.uk]

readme.txt

@@ -10,5 +10,10 @@ python downloader.py http://www.fanfiction.net/s/5192986/1/A_Fox_in_Tokyo
 
 Do 'python downloader.py -h' for more options.
 
-This tool uses Python 2.5.2, but should work with newer versions of Python.
+This tool uses Python 2.7, but should work with newer versions of Python.
 
+For more information, see:
+
+http://code.google.com/p/fanficdownloader/wiki/FanFictionDownloaderSupportedsites
+
+http://code.google.com/p/fanficdownloader/wiki/FanFictionDownloaderFAQs