Integrate new site, add adobe-text-layout to epub CSS for no nook hyphenation.

www.thepetulantpoetess.com www.scarvesandcoffee.net
http://www.dracoandginny.com

app.yaml

@@ -1,6 +1,6 @@
 # ffd-retief-hrd fanfictiondownloader
 application: fanfictiondownloader
-version: 4-4-21
+version: 4-4-23
 runtime: python27
 api_version: 1
 threadsafe: true

calibre-plugin/__init__.py

@@ -27,7 +27,7 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
     description         = 'UI plugin to download FanFiction stories from various sites.'
     supported_platforms = ['windows', 'osx', 'linux']
     author              = 'Jim Miller'
-    version             = (1, 6, 2)
+    version             = (1, 6, 6)
     minimum_calibre_version = (0, 8, 57)
 
     #: This field defines the GUI plugin class that contains all the code

calibre-plugin/config.py

@@ -628,27 +628,27 @@ class CountPagesTab(QWidget):
         self.l.addSpacing(5)
         
         # 'PageCount', 'WordCount', 'FleschReading', 'FleschGrade', 'GunningFog'
-        self.pagecount = QCheckBox('PageCount',self)
+        self.pagecount = QCheckBox('Page Count',self)
         self.pagecount.setToolTip('Which column and algorithm to use are configured in Count Pages.')
         self.pagecount.setChecked('PageCount' in prefs['countpagesstats'])
         self.l.addWidget(self.pagecount)
             
-        self.wordcount = QCheckBox('WordCount',self)
+        self.wordcount = QCheckBox('Word Count',self)
         self.wordcount.setToolTip('Which column and algorithm to use are configured in Count Words.\nWill overwrite word count from FFDL metadata if set to update the same custom column.')
         self.wordcount.setChecked('WordCount' in prefs['countpagesstats'])
         self.l.addWidget(self.wordcount)
 
-        self.fleschreading = QCheckBox('FleschReading',self)
+        self.fleschreading = QCheckBox('Flesch Reading Ease',self)
         self.fleschreading.setToolTip('Which column and algorithm to use are configured in Count Pages.')
         self.fleschreading.setChecked('FleschReading' in prefs['countpagesstats'])
         self.l.addWidget(self.fleschreading)
         
-        self.fleschgrade = QCheckBox('Fleschgrade',self)
+        self.fleschgrade = QCheckBox('Flesch-Kincaid Grade Level',self)
         self.fleschgrade.setToolTip('Which column and algorithm to use are configured in Count Pages.')
-        self.fleschgrade.setChecked('Fleschgrade' in prefs['countpagesstats'])
+        self.fleschgrade.setChecked('FleschGrade' in prefs['countpagesstats'])
         self.l.addWidget(self.fleschgrade)
         
-        self.gunningfog = QCheckBox('GunningFog',self)
+        self.gunningfog = QCheckBox('Gunning Fog Index',self)
         self.gunningfog.setToolTip('Which column and algorithm to use are configured in Count Pages.')
         self.gunningfog.setChecked('GunningFog' in prefs['countpagesstats'])
         self.l.addWidget(self.gunningfog)

calibre-plugin/dialogs.py

@@ -172,15 +172,17 @@ class FakeLineEdit():
     def text(self):
         pass
     
-class CollectURLDialog(QDialog):
+class CollectURLDialog(SizePersistedDialog):
     '''
     Collect single url for get urls.
     '''
-    def __init__(self, gui, title):
-        QDialog.__init__(self, gui)
+    def __init__(self, gui, title, url_text): 
+        SizePersistedDialog.__init__(self, gui, 'FanFictionDownLoader plugin:get story urls')
         self.gui = gui
         self.status=False
 
+        self.setMinimumWidth(300)
+        
         self.l = QGridLayout()
         self.setLayout(self.l)
 
@@ -189,6 +191,7 @@ class CollectURLDialog(QDialog):
         
         self.l.addWidget(QLabel("URL:"),1,0)
         self.url = QLineEdit(self)
+        self.url.setText(url_text)
         self.l.addWidget(self.url,1,1)
    
         self.ok_button = QPushButton('OK', self)
@@ -199,15 +202,16 @@ class CollectURLDialog(QDialog):
         self.cancel_button.clicked.connect(self.cancel)
         self.l.addWidget(self.cancel_button,2,1)
 
-        self.resize(self.sizeHint())
+        # restore saved size.
+        self.resize_dialog()
 
     def ok(self):
         self.status=True
-        self.hide()
+        self.accept()
 
     def cancel(self):
         self.status=False
-        self.hide()
+        self.reject()
 
 class UserPassDialog(QDialog):
     '''

calibre-plugin/ffdl_plugin.py

@@ -220,10 +220,12 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
     def create_menu_item_ex(self, parent_menu, menu_text, image=None, tooltip=None,
                            shortcut=None, triggered=None, is_checked=None, shortcut_name=None,
                            unique_name=None):
+        #print("create_menu_item_ex before %s"%unique_name)
         ac = create_menu_action_unique(self, parent_menu, menu_text, image, tooltip,
                                        shortcut, triggered, is_checked, shortcut_name, unique_name)
         self.actions_unique_map[ac.calibre_shortcut_unique_name] = ac.calibre_shortcut_unique_name
         self.menu_actions.append(ac)
+        #print("create_menu_item_ex after %s"%unique_name)
         return ac
 
     def plugin_button(self):
@@ -238,13 +240,23 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
             self._update_reading_lists(self.gui.library_view.get_selected_ids(),add)
 
     def get_urls_from_page(self):
-        d = CollectURLDialog(self.gui,"Get Story URLs from Web Page")
+
+        if prefs['urlsfromclip']:
+            try:
+                urltxt = self.get_urls_clip(storyurls=False)[0]
+            except:
+                urltxt = ""
+        
+        d = CollectURLDialog(self.gui,"Get Story URLs from Web Page",urltxt)
         d.exec_()
         if not d.status:
             return
-        print("URL:%s"%d.url.text())
+        print("get_urls_from_page URL:%s"%d.url.text())
 
-        url_list = get_urls_from_page("%s"%d.url.text())
+        ffdlconfig = SafeConfigParser()
+        ffdlconfig.readfp(StringIO(get_resources("plugin-defaults.ini")))
+        ffdlconfig.readfp(StringIO(prefs['personal.ini']))
+        url_list = get_urls_from_page("%s"%d.url.text(),ffdlconfig)
 
         if url_list:
             d = ViewLog(_("List of URLs"),"\n".join(url_list),parent=self.gui)
@@ -361,13 +373,14 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
             options['version'] = self.version
             print(self.version)
             self.start_downloads( options, update_books )
-        
-    def get_urls_clip(self):
+            
+    def get_urls_clip(self,storyurls=True):
         url_list = []
         if prefs['urlsfromclip']:
             for url in unicode(QApplication.instance().clipboard().text()).split():
-                if( self._is_good_downloader_url(url) ):
+                if not storyurls or self._is_good_downloader_url(url):
                     url_list.append(url)
+                    
         return url_list
 
     def apply_settings(self):
@@ -766,20 +779,20 @@ make_firstimage_cover:true
 <p>Proceed with updating your library?</p>
 '''%(len(good_list),len(bad_list))
 
-        htmllog='<html><body><table border="1"><tr><th>Status</th><th>Title</th><th>Author</th><th>URL</th><th>Comment</th></tr>'
+        htmllog='<html><body><table border="1"><tr><th>Status</th><th>Title</th><th>Author</th><th>Comment</th><th>URL</th></tr>'
         for book in good_list:
             if 'status' in book:
                 status = book['status']
             else:
                 status = 'Good'
-            htmllog = htmllog + '<tr><td>' + '</td><td>'.join([status,book['title'],", ".join(book['author']),book['url'],book['comment']]) + '</td></tr>'
+            htmllog = htmllog + '<tr><td>' + '</td><td>'.join([status,book['title'],", ".join(book['author']),book['comment'],book['url']]) + '</td></tr>'
         
         for book in bad_list:
             if 'status' in book:
                 status = book['status']
             else:
                 status = 'Bad'
-            htmllog = htmllog + '<tr><td>' + '</td><td>'.join([status,book['title'],", ".join(book['author']),book['url'],book['comment']]) + '</td></tr>'
+            htmllog = htmllog + '<tr><td>' + '</td><td>'.join([status,book['title'],", ".join(book['author']),book['comment'],book['url']]) + '</td></tr>'
         
         htmllog = htmllog + '</table></body></html>'
         

calibre-plugin/jobs.py

@@ -154,11 +154,13 @@ def do_download_for_worker(book,options):
             # update now handled by pre-populating the old images and
             # chapters in the adapter rather than merging epubs.
             urlchaptercount = int(story.getMetadata('numChapters'))
-            (url,chaptercount,
+            (url,
+             chaptercount,
              adapter.oldchapters,
              adapter.oldimgs,
              adapter.oldcover,
-             adapter.calibrebookmark) = get_update_data(book['epub_for_update'])
+             adapter.calibrebookmark,
+             adapter.logfile) = get_update_data(book['epub_for_update'])
 
             print("Do update - epub(%d) vs url(%d)" % (chaptercount, urlchaptercount))
             print("write to %s"%outfile)

defaults.ini

@@ -197,9 +197,24 @@ windows_eol: true
 zip_output: false
 
 ## epub carries the TOC in metadata.
-## mobi generated from epub will have a TOC at the end.
+## mobi generated from epub by calibre will have a TOC at the end.
 include_tocpage: false
 
+## include a Update Log page before the story text.  If included, the
+## log will be updated each time the epub is an all the metadata
+## fields that have changed since the last update (typically
+## dateUpdated,numChapters,numWords at a minimum) will be shown.
+## Great for tracking when chapters came out and when the description,
+## etc changed.
+include_logpage: false
+
+## items to include in the log page Empty metadata entries, or those
+## that haven't changed since the last update, will *not* appear, even
+## if in the list.  You can include extra text or HTML that will be
+## included as-is in each log entry. Eg: logpage_entries: ...,<br />,
+## summary,<br />,...
+logpage_entries: dateCreated,datePublished,dateUpdated,numChapters,numWords,status,title,author,description,category,genre,rating,warnings
+
 ## epub->mobi conversions typically don't like tables.
 titlepage_use_table: false
 
@@ -214,10 +229,12 @@ background_color: ffffff
 ## Allow customization of CSS.  Make sure to keep at least one space
 ## at the start of each line and to escape % to %%.  Also need
 ## background_color to be in the same section, if included in CSS.
+## 'adobe-text-layout: optimizeSpeed;' prevents hyphenation on newer Nooks
 output_css: 
  body { background-color: #%(background_color)s;
         text-align: justify;
-        margin: 2%%; }
+        margin: 2%%;
+	adobe-text-layout: optimizeSpeed; }
  pre { font-size: x-small; }
  sml { font-size: small; }
  h1 { text-align: center; }
@@ -546,6 +563,19 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+[www.dracoandginny.com]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
 [www.thealphagate.com]
 
 [www.checkmated.com]
@@ -670,6 +700,18 @@ cover_exclusion_regexp:/stories/999/images/.*?_trophy.png
 
 [www.prisonbreakfic.net]
 
+[www.scarvesandcoffee.net]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+[www.sinful-desire.org]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
 [www.siye.co.uk]
 
 [www.squidge.org/peja]
@@ -685,6 +727,14 @@ titlepage_entries: series,category,genre,language,status,datePublished,dateUpdat
 
 [www.storiesofarda.com]
 
+[www.thepetulantpoetess.com]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
 [www.thewriterscoffeeshop.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -759,6 +809,8 @@ collect_series: false
 ## personal.ini, not defaults.ini.
 #is_adult:true
 
+[www.wolverineandrogue.com]
+
 [www.wraithbait.com]
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,

downloader.py

@@ -128,7 +128,7 @@ def main():
            config.set("overrides",var,val)
 
    if options.list:
-       retlist = get_urls_from_page(args[0])
+       retlist = get_urls_from_page(args[0], config)
        print "\n".join(retlist)
                
        return
@@ -193,11 +193,13 @@ def main():
                    # update now handled by pre-populating the old
                    # images and chapters in the adapter rather than
                    # merging epubs.
-                   (url,chaptercount,
+                   (url,
+                    chaptercount,
                     adapter.oldchapters,
                     adapter.oldimgs,
                     adapter.oldcover,
-                    adapter.calibrebookmark) = get_update_data(args[0])
+                    adapter.calibrebookmark,
+                    adapter.logfile) = get_update_data(args[0])
 
                    writeStory(config,adapter,"epub")
                    

fanficdownloader/adapters/__init__.py

@@ -92,6 +92,11 @@ import adapter_wizardtalesnet
 import adapter_nhamagicalworldsus
 import adapter_hlfictionnet
 import adapter_grangerenchantedcom
+import adapter_dracoandginnycom
+import adapter_scarvesandcoffeenet
+import adapter_thepetulantpoetesscom
+import adapter_wolverineandroguecom
+import adapter_sinfuldesireorg
 
 ## This bit of complexity allows adapters to be added by just adding
 ## importing.  It eliminates the long if/else clauses we used to need
@@ -127,7 +132,7 @@ def getAdapter(config,url,fileform=None):
     if( domain != parsedUrl.netloc ):
         fixedurl = fixedurl.replace(parsedUrl.netloc,domain)
 
-    logging.debug("site:"+domain)
+    logging.debug("trying url:"+url)
     cls = getClassFor(domain)
     if not cls and domain.startswith("www."):
         domain = domain.replace("www.","")

fanficdownloader/adapters/adapter_archiveofourownorg.py

@@ -50,8 +50,17 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
         self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
         logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/works/'+self.story.getMetadata('storyId'))
+        # get storyId from url--url validation guarantees query correct
+        m = re.match(self.getSiteURLPattern(),url)
+        if m:
+            self.story.setMetadata('storyId',m.group('id'))
+            logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+            # normalized story URL.
+            self._setURL('http://' + self.getSiteDomain() + '/works/'+self.story.getMetadata('storyId'))
+        else:
+            raise exceptions.InvalidStoryURL(url,
+                                             self.getSiteDomain(),
+                                             self.getSiteExampleURLs())
         
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','ao3')
@@ -70,10 +79,11 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
         return ['www.archiveofourown.org','archiveofourown.org']
 
     def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/works/123456"
+        return "http://"+self.getSiteDomain()+"/works/123456 http://"+self.getSiteDomain()+"/collections/Some_Archive/works/123456"
 
     def getSiteURLPattern(self):
-        return re.escape("http://")+"(www\.)?"+re.escape(self.getSiteDomain()+"/works/")+r"\d+(/chapters/\d+)?/?$"
+        # http://archiveofourown.org/collections/Smallville_Slash_Archive/works/159770
+        return re.escape("http://")+"(www.)?"+re.escape(self.getSiteDomain())+r"(/collections/[^/]+)?/works/(?P<id>\d+)(/chapters/\d+)?/?$"
         
     ## Login
     def needToLoginCheck(self, data):

fanficdownloader/adapters/adapter_dracoandginnycom.py

@@ -0,0 +1,302 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return DracoAndGinnyComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class DracoAndGinnyComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','dcagn')
+
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","Harry Potter")
+        self.story.addToList("characters","Ginny Weasley")
+        self.story.addToList("characters","Draco Malfoy")
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%b %d, %Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.dracoandginny.com'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Registered Users Only' in data \
+                or 'There is no such account on our website' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['cookiecheck'] = '1'
+        params['submit'] = 'Submit'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
+        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "Member Account" not in d : #Member Account
+            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=2"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid & error in url.
+                addurl = addurl.replace("&","&")
+                url = self.url+'&index=1'+addurl
+                logging.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        content=soup.find('div',{'class' : 'listbox'})
+        
+        self.setDescription(url,content.find('blockquote'))
+        
+        for genre in content.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')):
+            self.story.addToList('genre',genre.string)
+
+        for warning in content.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')):
+            self.story.addToList('warnings',warning.string)
+        
+        labels = content.findAll('b')
+        
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value.split(' |')[0])
+                
+            if 'Rating' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value.nextSibling).split(' |')[0], self.dateformat))
+            
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            i=1
+            for a in storyas:
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        break
+                    i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'class' : 'listbox'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_nhamagicalworldsus.py

@@ -122,7 +122,7 @@ class NHAMagicalWorldsUsAdapter(BaseSiteAdapter):
 		
 
         # Find the chapters:
-        chapters=soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+'&chapter=\d$'))
+        chapters=soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+'&chapter=\d+$'))
         if len(chapters) == 0:
             self.chapterUrls.append((self.story.getMetadata('title'),url))
         else:

fanficdownloader/adapters/adapter_scarvesandcoffeenet.py

@@ -0,0 +1,251 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return ScarvesAndCoffeeNetAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class ScarvesAndCoffeeNetAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','scacf')
+
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","Glee")
+        self.story.addToList("characters","Kurt Hummel")
+        self.story.addToList("characters","Blaine Anderson")
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%m/%d/%Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.scarvesandcoffee.net'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Registered Users Only' in data \
+                or 'There is no such account on our website' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=20"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid & error in url.
+                addurl = addurl.replace("&","&")
+                url = self.url+'&index=1'+addurl
+                logging.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('genre',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            i=1
+            for a in storyas:
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        break
+                    i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_sinfuldesireorg.py

@@ -0,0 +1,253 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return SinfulDesireOrgAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class SinfulDesireOrgAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/archive/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','snds')
+
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","Supernatural")
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%m/%d/%Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.sinful-desire.org'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/archive/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/archive/viewstory.php?sid=")+r"\d+$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&warning=5"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid & error in url.
+                addurl = addurl.replace("&","&")
+                url = self.url+'&index=1'+addurl
+                logging.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/archive/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/archive/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=3'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/archive/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            i=1
+            for a in storyas:
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        break
+                    i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_test1.py

@@ -140,25 +140,25 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
                             ('Chapter 3, Over Cinnabar',self.url+"&chapter=4"),
                             ('Chapter 4',self.url+"&chapter=5"),
                             ('Chapter 5',self.url+"&chapter=6"),
-                            ('Chapter 6',self.url+"&chapter=6"),
-                            # ('Chapter 7',self.url+"&chapter=6"),
-                            # ('Chapter 8',self.url+"&chapter=6"),
-                            # ('Chapter 9',self.url+"&chapter=6"),
-                            # ('Chapter 0',self.url+"&chapter=6"),
-                            # ('Chapter a',self.url+"&chapter=6"),
-                            # ('Chapter b',self.url+"&chapter=6"),
-                            # ('Chapter c',self.url+"&chapter=6"),
-                            # ('Chapter d',self.url+"&chapter=6"),
-                            # ('Chapter e',self.url+"&chapter=6"),
-                            # ('Chapter f',self.url+"&chapter=6"),
-                            # ('Chapter g',self.url+"&chapter=6"),
-                            # ('Chapter h',self.url+"&chapter=6"),
-                            # ('Chapter i',self.url+"&chapter=6"),
-                            # ('Chapter j',self.url+"&chapter=6"),
-                            # ('Chapter k',self.url+"&chapter=6"),
-                            # ('Chapter l',self.url+"&chapter=6"),
-                            # ('Chapter m',self.url+"&chapter=6"),
-                            # ('Chapter n',self.url+"&chapter=6"),
+                            #('Chapter 6',self.url+"&chapter=7"),
+                            #('Chapter 7',self.url+"&chapter=8"),
+                            #('Chapter 8',self.url+"&chapter=9"),
+                            #('Chapter 9',self.url+"&chapter=0"),
+                            #('Chapter 0',self.url+"&chapter=a"),
+                            #('Chapter a',self.url+"&chapter=b"),
+                            #('Chapter b',self.url+"&chapter=c"),
+                            #('Chapter c',self.url+"&chapter=d"),
+                            #('Chapter d',self.url+"&chapter=e"),
+                            #('Chapter e',self.url+"&chapter=f"),
+                            #('Chapter f',self.url+"&chapter=g"),
+                            #('Chapter g',self.url+"&chapter=h"),
+                            #('Chapter h',self.url+"&chapter=i"),
+                            #('Chapter i',self.url+"&chapter=j"),
+                            #('Chapter j',self.url+"&chapter=k"),
+                            #('Chapter k',self.url+"&chapter=l"),
+                            #('Chapter l',self.url+"&chapter=m"),
+                            #('Chapter m',self.url+"&chapter=n"),
+                            #('Chapter n',self.url+"&chapter=o"),
                             ]
         self.story.setMetadata('numChapters',len(self.chapterUrls))
                             

fanficdownloader/adapters/adapter_thepetulantpoetesscom.py

@@ -0,0 +1,239 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return ThePetulantPoetessComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class ThePetulantPoetessComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId') +'&i=1')
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','tpp')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%Y/%m/%d"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.thepetulantpoetess.com'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'You must be a member to read this story.' in data \
+                or "The Ministry of Magic does not have a record of that password. " in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['cookiecheck'] = '1'
+        params['submit'] = 'Submit'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
+        logging.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "My Account Page" not in d : #Member Account
+            logging.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        chapters=soup.find('select', {'name' : 'sid'})
+        if chapters == None:
+            self.chapterUrls.append((self.story.getMetadata('title'),url))
+        else:
+            for chapter in chapters.findAll('option', value=re.compile(r"viewstory.php\?sid=\d+&i=1")):
+                # just in case there's tags, like <i> in chapter titles.
+                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['value']))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+        
+        # make sure that the story id is from the first chapter
+        self.story.setMetadata('storyId',self.chapterUrls[0][1].split('=')[1].split('&')[0])
+        
+        #locate the story on author's page
+        index = 1
+        found = 0
+        while found == 0:
+            asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')+"&page="+str(index)))
+        
+            for info in asoup.findAll('td', {'class' : 'highlightcolor1'}):
+                a = info.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+                if a != None:
+                    self.story.setMetadata('title',a.text)                
+                    found = 1
+                    break
+            index=index+1
+            
+        # extract metadata
+        b=info.find('b')
+        b.find('a').extract()
+        self.story.setMetadata('rating', b.text.split('[')[1].split(']')[0])
+        
+        info = info.findNext('td', {'colspan' : '2'}) 
+        for label in info.findAll('b'):
+            value = label.nextSibling
+            
+            if "Category" in label.text:
+                for cat in info.findAll('a'):
+                    self.story.addToList('category',cat.string)
+            
+            if "Characters" in label.text:
+                for char in value.split(', '):
+                    self.story.addToList('characters',char)
+                    
+            if "Genres" in label.text:
+                for genre in value.split(', '):
+                    if "General" not in genre:
+                        self.story.addToList('genre',genre)
+                    
+            if "Warnings" in label.text:
+                for warning in value.split(', '):
+                    if "none" not in warning.lower():
+                        self.story.addToList('warnings',warning)
+                        
+                        
+        info = info.findNext('td', {'class' : 'tblborder'}) 
+        info.find('b').extract()
+        self.setDescription(url,info)
+        
+        info = info.findNext('td', {'class' : 'highlightcolor2'}) 
+        for label in info.findAll('b'):
+            value = label.nextSibling
+            
+            if "Completed" in label.text:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+                    
+            if 'Published' in label.text:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label.text:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+                
+            if 'Word Count' in label.text:
+                self.story.setMetadata('numWords', value)
+            
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.findAll('table')[2].findAll('td')[1]
+        for a in div.findAll('div'):
+            a.extract()
+        div.name='div'
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_wolverineandroguecom.py

@@ -0,0 +1,223 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2012 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return WolverineAndRogueComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class WolverineAndRogueComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/wrfa/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','wrfa')
+
+        # If all stories from the site fall into the same category,
+        # the site itself isn't likely to label them as such, so we
+        # do.
+        self.story.addToList("category","X-Men Movie")
+        self.story.addToList("characters","Wolverine")
+        self.story.addToList("characters","Rogue")
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%m/%d/%Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.wolverineandrogue.com'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/wrfa/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/wrfa/viewstory.php?sid=")+r"\d+$"
+
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'
+        logging.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        pt = soup.find('div', {'id' : 'pagetitle'})
+        a = pt.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.text)
+        
+        # Find authorid and URL from... author url.
+        a = pt.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/wrfa/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        rating=pt.text.split('(')[1].split(')')[0]
+        self.story.setMetadata('rating', rating)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/wrfa/'+chapter['href']))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        content=soup.find('div',{'class' : 'content'})
+        labels = soup.findAll('span',{'class':'label'})
+        
+        value = labels[0].previousSibling
+        svalue = ""
+        while value != None:
+            val = value
+            value = value.previousSibling
+        while "Categories" not in str(val):
+            svalue += str(val)
+            val = val.nextSibling
+        self.setDescription(url,svalue)
+        
+        
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value.split(' -')[0])
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Complete' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(value.split(' -')[0], self.dateformat))
+            
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/wrfa/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            i=1
+            for a in storyas:
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        break
+                    i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logging.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_yourfanfictioncom.py

@@ -141,7 +141,6 @@ class YourFanfictionComAdapter(BaseSiteAdapter):
         if "Access denied. This story has not been validated by the adminstrators of this site." in data:
             raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
             
-        # use BeautifulSoup HTML parser to make everything easier to find.
         # because for some reason, this works while simple 'print data' errors on ascii conversion.
         # loopdata = data
         # chklen=5000
@@ -151,6 +150,7 @@ class YourFanfictionComAdapter(BaseSiteAdapter):
         #     logging.info("loopdata: %s" % loopdata[:chklen])
         #     loopdata = loopdata[chklen:]
             
+        # use BeautifulSoup HTML parser to make everything easier to find.
         soup = bs.BeautifulSoup(data)
 
         # Now go hunting for all the meta data and the chapter list.
@@ -191,9 +191,11 @@ class YourFanfictionComAdapter(BaseSiteAdapter):
             if 'Summary' in label:
                 ## Everything until the next span class='label'
                 svalue = ""
-                while not defaultGetattr(value,'class') == 'label':
+                while value and not defaultGetattr(value,'class') == 'label':
                     svalue += str(value)
                     value = value.nextSibling
+                # sometimes poorly formated desc (<p> w/o </p>) leads
+                # to all labels being included.
                 self.setDescription(url,svalue)
                 #self.story.setMetadata('description',stripHTML(svalue))
 

fanficdownloader/adapters/base_adapter.py

@@ -89,6 +89,7 @@ class BaseSiteAdapter(Configurable):
         self.oldimgs = None
         self.oldcover = None # (data of existing cover html, data of existing cover image)
         self.calibrebookmark = None
+        self.logfile = None
         ## order of preference for decoding.
         self.decode = ["utf8",
                        "Windows-1252"] # 1252 is a superset of
@@ -229,6 +230,8 @@ class BaseSiteAdapter(Configurable):
             # cheesy way to carry calibre bookmark file forward across update.
             if self.calibrebookmark:
                 self.story.calibrebookmark = self.calibrebookmark
+            if self.logfile:
+                self.story.logfile = self.logfile
                 
         return self.story
 

fanficdownloader/epubutils.py

@@ -42,6 +42,7 @@ def get_update_data(inputio,
             
     oldcover = None
     calibrebookmark = None
+    logfile = None
     # Looking for pre-existing cover.
     for item in contentdom.getElementsByTagName("reference"):
         if item.getAttribute("type") == "cover":
@@ -96,6 +97,11 @@ def get_update_data(inputio,
             if( item.getAttribute("media-type") == "application/xhtml+xml" ):
                 href=relpath+item.getAttribute("href")
                 #print("---- item href:%s path part: %s"%(href,get_path_part(href)))
+                if re.match(r'.*/log_page\.x?html',href):
+                    try:
+                        logfile = epub.read(href).decode("utf-8")
+                    except:
+                        pass # corner case I bumped into while testing.
                 if re.match(r'.*/(file|chapter)\d+\.x?html',href):
                     if getsoups:
                         soup = bs.BeautifulSoup(epub.read(href).decode("utf-8"))
@@ -136,7 +142,7 @@ def get_update_data(inputio,
                     
     for k in images.keys():
         print("\tlongdesc:%s\n\tData len:%s\n"%(k,len(images[k])))
-    return (source,filecount,soups,images,oldcover,calibrebookmark)
+    return (source,filecount,soups,images,oldcover,calibrebookmark,logfile)
 
 def get_path_part(n):
     relpath = os.path.dirname(n)

fanficdownloader/geturls.py

@@ -25,14 +25,38 @@ from gziphttp import GZipProcessor
 
 import adapters
 
-def get_urls_from_page(url):
-    
-    opener = u2.build_opener(u2.HTTPCookieProcessor(),GZipProcessor())
-    soup = BeautifulSoup(opener.open(url).read())
-    
+def get_urls_from_page(url,config=None):
+
     normalized = set() # normalized url
     retlist = [] # orig urls.
-    config = ConfigParser.SafeConfigParser()
+    if not config:
+        config = ConfigParser.SafeConfigParser()
+
+    data = None
+    
+    # special stuff to log into archiveofourown.org, if possible.
+    # Unlike most that show the links to 'adult' stories, but protect
+    # them, AO3 doesn't even show them if not logged in.  Only works
+    # with saved user/pass--not going to prompt for list.
+    if 'archiveofourown.org' in url:
+        ao3adapter = adapters.getAdapter(config,"http://www.archiveofourown.org/works/0","EPUB")
+        if ao3adapter.getConfig("username"):
+            if ao3adapter.getConfig("is_adult"):
+                addurl = "?view_adult=true"
+            else:
+                addurl=""
+            # just to get an authenticity_token.
+            data = ao3adapter._fetchUrl(url+addurl)
+            # login the session.
+            ao3adapter.performLogin(url,data)
+            # get the list page with logged in session.
+            data = ao3adapter._fetchUrl(url)
+
+    if not data:
+        opener = u2.build_opener(u2.HTTPCookieProcessor(),GZipProcessor())
+        data = opener.open(url).read()
+    
+    soup = BeautifulSoup(data)
     
     for a in soup.findAll('a'):
         if a.has_key('href'):

fanficdownloader/story.py

@@ -201,6 +201,7 @@ class Story:
         self.cover=None # *href* of new cover image--need to create html.
         self.oldcover=None # (oldcoverhtmlhref,oldcoverhtmltype,oldcoverhtmldata,oldcoverimghref,oldcoverimgtype,oldcoverimgdata)
         self.calibrebookmark=None # cheesy way to carry calibre bookmark file forward across update.
+        self.logfile=None # cheesy way to carry log file forward across update.
 
     def setMetadata(self, key, value):
         ## still keeps < < and &
@@ -230,6 +231,8 @@ class Story:
             if value:
                 if key == "numWords":
                     value = commaGroups(value)
+                if key == "numChapters":
+                    value = commaGroups("%d"%value)
                 if key == "dateCreated":
                     value = value.strftime("%Y-%m-%d %H:%M:%S")
                 if key == "datePublished" or key == "dateUpdated":
@@ -279,7 +282,7 @@ class Story:
     # just for less clutter in adapters.
     def extendList(self,listname,l):
         for v in l:
-            self.addToList(listname,v)
+            self.addToList(listname,v.strip())
     
     def addToList(self,listname,value):
         if value==None:

fanficdownloader/writers/writer_epub.py

@@ -140,6 +140,101 @@ ${value}<br />
 </html>
 ''')
 
+        self.EPUB_LOG_PAGE_START = string.Template('''<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head>
+<title>Update Log</title>
+<link href="stylesheet.css" type="text/css" charset="UTF-8" rel="stylesheet"/>
+</head>
+<body>
+<h3>Update Log</h3>
+''')
+
+        self.EPUB_LOG_ENTRY = string.Template('''
+<b>${label}:</b> <span id="${id}">${value}</span>
+''')
+                          
+        self.EPUB_LOG_PAGE_END = string.Template('''
+</body>
+</html>
+''')
+
+    def writeLogPage(self, out):
+        """
+       XXX 
+
+        
+        Write the log page, but only include entries that there's
+        metadata for.  START, ENTRY and END are expected to already by
+        string.Template().  START and END are expected to use the same
+        names as Story.metadata, but ENTRY should use id, label and value.
+        """
+        if self.getConfig("include_logpage"):
+
+            # if there's a self.story.logfile, there's an existing log
+            # to add to.
+            if self.story.logfile:
+                print("existing logfile found, appending")
+                print("existing data:%s"%self._getLastLogData(self.story.logfile))
+                replace_string = "</body>" # "</h3>"
+                self._write(out,self.story.logfile.replace(replace_string,self._makeLogEntry(self._getLastLogData(self.story.logfile))+replace_string))
+            else:
+                # otherwise, write a new one.
+                self._write(out,self.EPUB_LOG_PAGE_START.substitute(self.story.getAllMetadata()))
+                self._write(out,self._makeLogEntry())
+                self._write(out,self.EPUB_LOG_PAGE_END.substitute(self.story.getAllMetadata()))
+
+    # self parsing instead of Soup because it should be simple and not
+    # worth the overhead.
+    def _getLastLogData(self,logfile):
+        """
+        Make a dict() of the most recent(last) log entry for each piece of metadata.
+        Switch rindex to index to search from top instead of bottom.
+        """
+        values = {}
+        for entry in self.getConfigList("logpage_entries"):
+            if entry in self.validEntries:
+                try:
+                    # <span id="dateUpdated">1975-04-15</span>
+                    span = '<span id="%s">'%entry
+                    idx = logfile.rindex(span)+len(span)
+                    values[entry] = logfile[idx:logfile.index('</span>',idx)]
+                except Exception, e:
+                    #print("e:%s"%e)
+                    pass
+
+        return values
+        
+    def _makeLogEntry(self, oldvalues={}):
+        retval = "<p class='log_entry'>"
+
+        for entry in self.getConfigList("logpage_entries"):
+            if entry in self.validEntries:
+                val = self.story.getMetadata(entry)
+                if val and ( entry not in oldvalues or val != oldvalues[entry] ):
+                    if self.hasConfig(entry+"_label"):
+                        label=self.getConfig(entry+"_label")
+                    else:
+                        print("Using fallback label for %s_label"%entry)
+                        label=self.titleLabels[entry]
+
+                    retval = retval + self.EPUB_LOG_ENTRY.substitute({'id':entry,
+                                                                      'label':label,
+                                                                      'value':val})
+            else:
+                # could be useful for introducing extra text, but
+                # mostly it makes it easy to tell when you get the
+                # keyword wrong.
+                retval = retval + entry
+                
+        retval = retval + "</p><hr />"
+        
+        if self.getConfig('replace_hr'):
+            retval = retval.replace("<hr />","<div class='center'>* * *</div>")
+            
+        return retval
+                
     def writeStoryImpl(self, out):
 
         ## Python 2.5 ZipFile is rather more primative than later
@@ -349,6 +444,11 @@ div { margin: 0pt; padding: 0pt; }
         if len(self.story.getChapters()) > 1 and self.getConfig("include_tocpage") and not self.metaonly :
             items.append(("toc_page","OEBPS/toc_page.xhtml","application/xhtml+xml","Table of Contents"))
             itemrefs.append("toc_page")
+
+        if self.getConfig("include_logpage"):
+            items.append(("log_page","OEBPS/log_page.xhtml","application/xhtml+xml","Update Log"))
+            itemrefs.append("log_page")
+            
         for index, (title,html) in enumerate(self.story.getChapters()):
             if html:
                 i=index+1
@@ -480,6 +580,13 @@ div { margin: 0pt; padding: 0pt; }
             outputepub.writestr("OEBPS/toc_page.xhtml",tocpageIO.getvalue())
         tocpageIO.close()
 
+        # write log page.
+        logpageIO = StringIO.StringIO()
+        self.writeLogPage(logpageIO)
+        if logpageIO.getvalue(): # will be false if no log page.
+            outputepub.writestr("OEBPS/log_page.xhtml",logpageIO.getvalue())
+        logpageIO.close()
+        
         for index, (title,html) in enumerate(self.story.getChapters()):
             if html:
                 logging.debug('Writing chapter text for: %s' % title)

index.html

@@ -56,10 +56,12 @@
 	  <!-- put announcements here, h3 is a good title size. -->
 	  <h3>New Sites</h3>
 	  <p>
-	    
-	    New sites grangerenchanted.com, hlfiction.net and nha.magical-worlds.us.
-	    <br />Thanks, Ida!
-	    
+	    Now supporing www.dracoandginny.com,
+	    www.scarvesandcoffee.net, www.thepetulantpoetess.com,
+	    www.wolverineandrogue.com and www.sinful-desire.com.  Thanks, Ida!
+	  </p>
+	  <p>
+	    FFDL now supports over 70 different fanfiction sites.
 	  </p>
 	  <p>
 	    Questions?  Check out our 
@@ -69,7 +71,7 @@
 	    If you have any problems with this application, please
 	    report them in
 	    the <a href="http://groups.google.com/group/fanfic-downloader">FanFictionDownLoader Google Group</a>.  The
-	    <a href="http://4-4-20.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
+	    <a href="http://4-4-22.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
 	  </p>
 	  <div id='error'>
 	    {{ error_message }}
@@ -506,6 +508,31 @@
 	    Use the URL of the story's chapter list, such as
 	    <br /><a href="http://nha.magical-worlds.us/viewstory.php?sid=1234">http://nha.magical-worlds.us/viewstory.php?sid=1234</a>
 	  </dd>
+	  <dt>www.dracoandginny.com</dt>
+	  <dd>
+	    Use the URL of the story's chapter list, such as
+	    <br /><a href="http://www.dracoandginny.com/viewstory.php?sid=1234">http://www.dracoandginny.com/viewstory.php?sid=1234</a>
+	  </dd>
+	  <dt>www.scarvesandcoffee.net</dt>
+	  <dd>
+	    Use the URL of the story's chapter list, such as
+	    <br /><a href="http://www.scarvesandcoffee.net/viewstory.php?sid=1234">http://www.scarvesandcoffee.net/viewstory.php?sid=1234</a>
+	  </dd>
+	  <dt>www.thepetulantpoetess.com</dt>
+	  <dd>
+	    Use the URL of the story's chapter list, such as
+	    <br /><a href="http://www.thepetulantpoetess.com/viewstory.php?sid=1234">http://www.thepetulantpoetess.com/viewstory.php?sid=1234</a>
+	  </dd>
+	  <dt>www.wolverineandrogue.com</dt>
+	  <dd>
+	    Use the URL of the story's chapter list, such as
+	    <br /><a href="http://www.wolverineandrogue.com/wrfa/viewstory.php?sid=1234">http://www.wolverineandrogue.com/wrfa/viewstory.php?sid=1234</a>
+	  </dd>
+	  <dt>www.sinful-desire.org</dt>
+	  <dd>
+	    Use the URL of the story's chapter list, such as
+	    <br /><a href="http://www.sinful-desire.org/archive/viewstory.php?sid=1234">http://www.sinful-desire.org/archive/viewstory.php?sid=1234</a>
+	  </dd>
 	</dl>
 	<p>
 	  A few additional things to know, which will make your life substantially easier:

plugin-defaults.ini

@@ -19,8 +19,7 @@
 ## overridden at several levels
 
 ## Some sites also require the user to confirm they are adult for
-## adult content.  In commandline version, this should go in your
-## personal.ini, not defaults.ini.
+## adult content.  Uncomment by removing '#' in front of is_adult.
 #is_adult:true
 
 ## All available titlepage_entries and the label used for them:
@@ -183,9 +182,24 @@ windows_eol: true
 [epub]
 
 ## epub carries the TOC in metadata.
-## mobi generated from epub will have a TOC at the end.
+## mobi generated from epub by calibre will have a TOC at the end.
 include_tocpage: false
 
+## include a Update Log page before the story text.  If included, the
+## log will be updated each time the epub is an all the metadata
+## fields that have changed since the last update (typically
+## dateUpdated,numChapters,numWords at a minimum) will be shown.
+## Great for tracking when chapters came out and when the description,
+## etc changed.
+include_logpage: false
+
+## items to include in the log page Empty metadata entries, or those
+## that haven't changed since the last update, will *not* appear, even
+## if in the list.  You can include extra text or HTML that will be
+## included as-is in each log entry. Eg: logpage_entries: ...,<br />,
+## summary,<br />,...
+logpage_entries: dateCreated,datePublished,dateUpdated,numChapters,numWords,status,title,author,description,category,genre,rating,warnings
+
 ## epub->mobi conversions typically don't like tables.
 titlepage_use_table: false
 
@@ -200,10 +214,12 @@ background_color: ffffff
 ## Allow customization of CSS.  Make sure to keep at least one space
 ## at the start of each line and to escape % to %%.  Also need
 ## background_color to be in the same section, if included in CSS.
+## 'adobe-text-layout: optimizeSpeed;' prevents hyphenation on newer Nooks
 output_css: 
  body { background-color: #%(background_color)s;
         text-align: justify;
-        margin: 2%%; }
+        margin: 2%%;
+	adobe-text-layout: optimizeSpeed; }
  pre { font-size: x-small; }
  sml { font-size: small; }
  h1 { text-align: center; }
@@ -532,6 +548,19 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+[www.dracoandginny.com]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
 [www.thealphagate.com]
 
 [www.checkmated.com]
@@ -653,6 +682,18 @@ cover_exclusion_regexp:/stories/999/images/.*?_trophy.png
 
 [www.prisonbreakfic.net]
 
+[www.scarvesandcoffee.net]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+[www.sinful-desire.org]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
 [www.siye.co.uk]
 
 [www.squidge.org/peja]
@@ -668,6 +709,14 @@ titlepage_entries: series,category,genre,language,status,datePublished,dateUpdat
 
 [www.storiesofarda.com]
 
+[www.thepetulantpoetess.com]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
 [www.thewriterscoffeeshop.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -742,6 +791,8 @@ collect_series: false
 ## personal.ini, not defaults.ini.
 #is_adult:true
 
+[www.wolverineandrogue.com]
+
 [www.wraithbait.com]
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,

plugin-example.ini

@@ -3,8 +3,7 @@
 
 [defaults]
 ## Some sites also require the user to confirm they are adult for
-## adult content.  In commandline version, this should go in your
-## personal.ini, not defaults.ini.
+## adult content.
 #is_adult:true
 
 ## include images from img tags in the body and summary of