Fix login failure for tth, add anthology=true metadata, bump version.

http://archive.skyehawke.com/story.php?no=1234
http://www.skyehawke.com/archive/story.php?no=1234
http://skyehawke.com/archive/story.php?no=1234

app.yaml

@@ -1,6 +1,6 @@
 # ffd-retief-hrd fanfictiondownloader
 application: fanfictiondownloader
-version: 4-4-40
+version: 4-4-46
 runtime: python27
 api_version: 1
 threadsafe: true

calibre-plugin/__init__.py

@@ -27,7 +27,7 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
     description         = 'UI plugin to download FanFiction stories from various sites.'
     supported_platforms = ['windows', 'osx', 'linux']
     author              = 'Jim Miller'
-    version             = (1, 7, 6)
+    version             = (1, 7, 12)
     minimum_calibre_version = (0, 8, 57)
 
     #: This field defines the GUI plugin class that contains all the code

calibre-plugin/config.py

@@ -911,7 +911,7 @@ titleLabels = {
     'ships':'Relationships',
     'datePublished':'Published',
     'dateUpdated':'Updated',
-    'dateCreated':'Packaged',
+    'dateCreated':'Created',
     'rating':'Rating',
     'warnings':'Warnings',
     'numChapters':'Chapters',
@@ -922,7 +922,7 @@ titleLabels = {
     'extratags':'Extra Tags',
     'title':'Title',
     'storyUrl':'Story URL',
-    'description':'Summary',
+    'description':'Description',
     'author':'Author',
     'authorUrl':'Author URL',
     'formatname':'File Format',

calibre-plugin/dialogs.py

@@ -30,13 +30,13 @@ from calibre_plugins.fanfictiondownloader_plugin.common_utils \
     import (ReadOnlyTableWidgetItem, ReadOnlyTextIconWidgetItem, SizePersistedDialog,
             ImageTitleLayout, get_icon)
 
-SKIP='Skip'
-ADDNEW='Add New Book'
-UPDATE='Update EPUB if New Chapters'
-UPDATEALWAYS='Update EPUB Always'
-OVERWRITE='Overwrite if Newer'
-OVERWRITEALWAYS='Overwrite Always'
-CALIBREONLY='Update Calibre Metadata Only'
+SKIP=u'Skip'
+ADDNEW=u'Add New Book'
+UPDATE=u'Update EPUB if New Chapters'
+UPDATEALWAYS=u'Update EPUB Always'
+OVERWRITE=u'Overwrite if Newer'
+OVERWRITEALWAYS=u'Overwrite Always'
+CALIBREONLY=u'Update Calibre Metadata Only'
 collision_order=[SKIP,
                  ADDNEW,
                  UPDATE,
@@ -44,6 +44,10 @@ collision_order=[SKIP,
                  OVERWRITE,
                  OVERWRITEALWAYS,
                  CALIBREONLY,]
+
+anthology_collision_order=[UPDATE,
+                           UPDATEALWAYS,
+                           OVERWRITEALWAYS]
         
 # This is a more than slightly kludgey way to get
 # EditWithComplete to *not* alpha-order the reasons, but leave
@@ -84,10 +88,23 @@ class DroppableQTextEdit(QTextEdit):
                             
 class AddNewDialog(SizePersistedDialog):
 
-    def __init__(self, gui, prefs, icon, url_list_text):
+    def __init__(self, gui, prefs, icon, url_list_text, merge=False, newmerge=False):
         SizePersistedDialog.__init__(self, gui, 'FanFictionDownLoader plugin:add new dialog')
         self.gui = gui
+        self.merge = merge
+        self.newmerge = newmerge
 
+        if merge:
+            labeltext = 'Story URL(s) for anthology, one per line:'
+            tooltiptext = 'URLs for stories to include in the anthology, one per line.\nWill take URLs from clipboard, but only valid URLs.'
+            collisiontext = 'If Story Already Exists in Anthology?'
+            collisiontooltip = "What to do if there's already an existing story with the same URL in the anthology."
+        else:
+            labeltext = 'Story URL(s), one per line:'
+            tooltiptext = 'URLs for stories, one per line.\nWill take URLs from clipboard, but only valid URLs.\nAdd [1,5] after the URL to limit the download to chapters 1-5.'
+            collisiontext = 'If Story Already Exists?'
+            collisiontooltip = "What to do if there's already an existing story with the same URL or title and author."
+            
         if prefs['adddialogstaysontop']:
             QDialog.setWindowFlags ( self, Qt.Dialog|Qt.WindowStaysOnTopHint )
         
@@ -98,55 +115,58 @@ class AddNewDialog(SizePersistedDialog):
         self.setWindowTitle('FanFictionDownLoader')
         self.setWindowIcon(icon)
 
-        self.l.addWidget(QLabel('Story URL(s), one per line:'))
+        self.l.addWidget(QLabel(labeltext))
         self.url = DroppableQTextEdit(self)
-        self.url.setToolTip('URLs for stories, one per line.\nWill take URLs from clipboard, but only valid URLs.\nAdd [1,5] after the URL to limit the download to chapters 1-5.')
+        self.url.setToolTip(tooltiptext)
         self.url.setLineWrapMode(QTextEdit.NoWrap)
         self.url.setText(url_list_text)
         self.l.addWidget(self.url)
-        
-        horz = QHBoxLayout()
-        label = QLabel('Output &Format:')
-        horz.addWidget(label)
-        self.fileform = QComboBox(self)
-        self.fileform.addItem('epub')
-        self.fileform.addItem('mobi')
-        self.fileform.addItem('html')
-        self.fileform.addItem('txt')
-        self.fileform.setCurrentIndex(self.fileform.findText(prefs['fileform']))
-        self.fileform.setToolTip('Choose output format to create.  May set default from plugin configuration.')
-        self.fileform.activated.connect(self.set_collisions)
-        
-        label.setBuddy(self.fileform)
-        horz.addWidget(self.fileform)
-        self.l.addLayout(horz)
 
-        horz = QHBoxLayout()
-        label = QLabel('If Story Already Exists?')
-        horz.addWidget(label)
-        self.collision = QComboBox(self)
-        self.collision.setToolTip("What to do if there's already an existing story with the same URL or title and author.")
-        # add collision options
-        self.set_collisions()
-        i = self.collision.findText(prefs['collision'])
-        if i > -1:
-            self.collision.setCurrentIndex(i)
-        label.setBuddy(self.collision)
-        horz.addWidget(self.collision)
-        self.l.addLayout(horz)
+        if not merge:
+            horz = QHBoxLayout()
+            label = QLabel('Output &Format:')
+            horz.addWidget(label)
+            self.fileform = QComboBox(self)
+            self.fileform.addItem('epub')
+            self.fileform.addItem('mobi')
+            self.fileform.addItem('html')
+            self.fileform.addItem('txt')
+            self.fileform.setCurrentIndex(self.fileform.findText(prefs['fileform']))
+            self.fileform.setToolTip('Choose output format to create.  May set default from plugin configuration.')
+            self.fileform.activated.connect(self.set_collisions)
+            
+            label.setBuddy(self.fileform)
+            horz.addWidget(self.fileform)
+            self.l.addLayout(horz)
 
-        horz = QHBoxLayout()
-        self.updatemeta = QCheckBox('Update Calibre &Metadata?',self)
-        self.updatemeta.setToolTip("Update metadata for existing stories in Calibre from web site?\n(Columns set to 'New Only' in the column tabs will only be set for new books.)")
-        self.updatemeta.setChecked(prefs['updatemeta'])
-        horz.addWidget(self.updatemeta)
-
-        self.updateepubcover = QCheckBox('Update EPUB Cover?',self)
-        self.updateepubcover.setToolTip('Update book cover image from site or defaults (if found) <i>inside</i> the EPUB when EPUB is updated.')
-        self.updateepubcover.setChecked(prefs['updateepubcover'])
-        horz.addWidget(self.updateepubcover)
-        
-        self.l.addLayout(horz)
+        if not newmerge:
+            horz = QHBoxLayout()
+            label = QLabel(collisiontext)
+            horz.addWidget(label)
+            self.collision = QComboBox(self)
+            self.collision.setToolTip(collisiontooltip)
+            # add collision options
+            self.set_collisions()
+            i = self.collision.findText(prefs['collision'])
+            if i > -1:
+                self.collision.setCurrentIndex(i)
+            label.setBuddy(self.collision)
+            horz.addWidget(self.collision)
+            self.l.addLayout(horz)
+    
+            horz = QHBoxLayout()
+            self.updatemeta = QCheckBox('Update Calibre &Metadata?',self)
+            self.updatemeta.setToolTip("Update metadata for existing stories in Calibre from web site?\n(Columns set to 'New Only' in the column tabs will only be set for new books.)")
+            self.updatemeta.setChecked(prefs['updatemeta'])
+            horz.addWidget(self.updatemeta)
+    
+            if not merge: # hide if anthology merge.
+                self.updateepubcover = QCheckBox('Update EPUB Cover?',self)
+                self.updateepubcover.setToolTip('Update book cover image from site or defaults (if found) <i>inside</i> the EPUB when EPUB is updated.')
+                self.updateepubcover.setChecked(prefs['updateepubcover'])
+                horz.addWidget(self.updateepubcover)
+            
+            self.l.addLayout(horz)
 
         button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
         button_box.accepted.connect(self.accept)
@@ -163,20 +183,39 @@ class AddNewDialog(SizePersistedDialog):
     def set_collisions(self):
         prev=self.collision.currentText()
         self.collision.clear()
-        for o in collision_order:
-            if self.fileform.currentText() == 'epub' or o not in [UPDATE,UPDATEALWAYS]:
+        if self.merge:
+            order = anthology_collision_order
+        else:
+            order = collision_order
+        for o in order:
+            if self.merge or self.fileform.currentText() == 'epub' or o not in [UPDATE,UPDATEALWAYS]:
                 self.collision.addItem(o)
         i = self.collision.findText(prev)
         if i > -1:
             self.collision.setCurrentIndex(i)
         
     def get_ffdl_options(self):
-        return {
-            'fileform': unicode(self.fileform.currentText()),
-            'collision': unicode(self.collision.currentText()),
-            'updatemeta': self.updatemeta.isChecked(),
-            'updateepubcover': self.updateepubcover.isChecked(),
-            }
+        if self.merge:
+            if self.newmerge:
+                updatemeta=True
+                collision=ADDNEW
+            else:
+                updatemeta=self.updatemeta.isChecked()
+                collision=unicode(self.collision.currentText())
+                
+            return {
+                'fileform': 'epub',
+                'collision': collision,
+                'updatemeta': updatemeta,
+                'updateepubcover': True,
+                }
+        else:
+            return {
+                'fileform': unicode(self.fileform.currentText()),
+                'collision': unicode(self.collision.currentText()),
+                'updatemeta': self.updatemeta.isChecked(),
+                'updateepubcover': self.updateepubcover.isChecked(),
+                }
 
     def get_urlstext(self):
         return unicode(self.url.toPlainText())
@@ -193,10 +232,11 @@ class CollectURLDialog(SizePersistedDialog):
     '''
     Collect single url for get urls.
     '''
-    def __init__(self, gui, title, url_text): 
+    def __init__(self, gui, title, url_text, epubmerge_plugin=None): 
         SizePersistedDialog.__init__(self, gui, 'FanFictionDownLoader plugin:get story urls')
         self.gui = gui
         self.status=False
+        self.anthology=False
 
         self.setMinimumWidth(300)
         
@@ -204,28 +244,40 @@ class CollectURLDialog(SizePersistedDialog):
         self.setLayout(self.l)
 
         self.setWindowTitle(title)
-        self.l.addWidget(QLabel(title),0,0,1,2)
+        self.l.addWidget(QLabel(title),0,0,1,3)
         
         self.l.addWidget(QLabel("URL:"),1,0)
         self.url = QLineEdit(self)
         self.url.setText(url_text)
-        self.l.addWidget(self.url,1,1)
+        self.l.addWidget(self.url,1,1,1,2)
    
-        self.ok_button = QPushButton('OK', self)
-        self.ok_button.clicked.connect(self.ok)
-        self.l.addWidget(self.ok_button,2,0)
+        self.indiv_button = QPushButton('For Individual Books', self)
+        self.indiv_button.setToolTip('Get URLs and go to dialog for individual story downloads.')
+        self.indiv_button.clicked.connect(self.indiv)
+        self.l.addWidget(self.indiv_button,2,0)
+
+        self.merge_button = QPushButton('For Anthology Epub', self)
+        self.merge_button.setToolTip('Get URLs and go to dialog for Anthology download.\nRequires EpubMerge 1.3.1+ plugin.')
+        self.merge_button.clicked.connect(self.merge)
+        self.l.addWidget(self.merge_button,2,1)
+        self.merge_button.setEnabled(epubmerge_plugin!=None)
 
         self.cancel_button = QPushButton('Cancel', self)
         self.cancel_button.clicked.connect(self.cancel)
-        self.l.addWidget(self.cancel_button,2,1)
+        self.l.addWidget(self.cancel_button,2,2)
 
         # restore saved size.
         self.resize_dialog()
 
-    def ok(self):
+    def indiv(self):
         self.status=True
         self.accept()
 
+    def merge(self):
+        self.status=True
+        self.anthology=True
+        self.accept()
+
     def cancel(self):
         self.status=False
         self.reject()

calibre-plugin/ffdl_util.py

@@ -0,0 +1,43 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2013, Jim Miller'
+__docformat__ = 'restructuredtext en'
+
+from StringIO import StringIO
+
+from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapters, exceptions
+from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.configurable import Configuration
+from calibre_plugins.fanfictiondownloader_plugin.config import (prefs)
+
+def get_ffdl_personalini():
+    if prefs['includeimages']:
+        # this is a cheat to make it easier for users.
+        return '''[epub]
+include_images:true
+keep_summary_html:true
+make_firstimage_cover:true
+''' + prefs['personal.ini']
+    else:
+        return prefs['personal.ini']
+
+def get_ffdl_config(url,fileform="epub",personalini=None):
+    if not personalini:
+        personalini = get_ffdl_personalini()
+    site='unknown'
+    try:
+        site = adapters.getConfigSectionFor(url)
+    except Exception as e:
+        print("Failed trying to get ini config for url(%s): %s, using section [%s] instead"%(url,e,site))
+    configuration = Configuration(site,fileform)
+    configuration.readfp(StringIO(get_resources("plugin-defaults.ini")))
+    configuration.readfp(StringIO(personalini))
+
+    return configuration
+
+def get_ffdl_adapter(url,fileform="epub",personalini=None):
+    return adapters.getAdapter(get_ffdl_config(url,fileform,personalini),url)
+    

calibre-plugin/jobs.py

@@ -18,9 +18,9 @@ from calibre.utils.ipc.job import ParallelJob
 from calibre_plugins.fanfictiondownloader_plugin.dialogs import (NotGoingToDownload,
     OVERWRITE, OVERWRITEALWAYS, UPDATE, UPDATEALWAYS, ADDNEW, SKIP, CALIBREONLY)
 from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapters, writers, exceptions
-from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.configurable import Configuration
 from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.epubutils import get_update_data
 
+from calibre_plugins.fanfictiondownloader_plugin.ffdl_util import (get_ffdl_adapter, get_ffdl_config)
 # ------------------------------------------------------------------------------
 #
 #              Functions to perform downloads using worker jobs
@@ -114,9 +114,9 @@ def do_download_for_worker(book,options):
     try:
         book['comment'] = 'Download started...'
 
-        configuration = Configuration(adapters.getConfigSectionFor(book['url']),options['fileform'])
-        configuration.readfp(StringIO(get_resources("plugin-defaults.ini")))
-        configuration.readfp(StringIO(options['personal.ini']))
+        configuration = get_ffdl_config(book['url'],
+                                        options['fileform'],
+                                        options['personal.ini'])
         
         if not options['updateepubcover'] and 'epub_for_update' in book and options['collision'] in (UPDATE, UPDATEALWAYS):
             configuration.set("overrides","never_make_cover","true")
@@ -171,6 +171,17 @@ def do_download_for_worker(book,options):
              adapter.calibrebookmark,
              adapter.logfile) = get_update_data(book['epub_for_update'])
 
+            # dup handling from ffdl_plugin needed for anthology updates.
+            if options['collision'] == UPDATE:
+                if chaptercount == urlchaptercount:
+                    book['comment']="Already contains %d chapters.  Reuse as is."%chaptercount
+                    book['outfile'] = book['epub_for_update'] # for anthology merge ops.
+                    return book
+
+            # dup handling from ffdl_plugin needed for anthology updates.
+            if chaptercount > urlchaptercount:
+                raise NotGoingToDownload("Existing epub contains %d chapters, web site only has %d. Use Overwrite to force update." % (chaptercount,urlchaptercount),'dialog_error.png')
+                            
             print("Do update - epub(%d) vs url(%d)" % (chaptercount, urlchaptercount))
             print("write to %s"%outfile)
 

defaults.ini

@@ -320,12 +320,13 @@ background_color: ffffff
 ## Allow customization of CSS.  Make sure to keep at least one space
 ## at the start of each line and to escape % to %%.  Also need
 ## background_color to be in the same section, if included in CSS.
-## 'adobe-text-layout: optimizeSpeed;' prevents hyphenation on newer Nooks
+## 'adobe-hyphenate: none;' prevents hyphenation on newer Nooks
+## STR(wG) (1.2.1+ for sure)
 output_css: 
  body { background-color: #%(background_color)s;
         text-align: justify;
         margin: 2%%;
-	adobe-text-layout: optimizeSpeed; }
+	adobe-hyphenate: none; }
  pre { font-size: x-small; }
  sml { font-size: small; }
  h1 { text-align: center; }
@@ -1030,7 +1031,7 @@ extratags:
 ## for examples of how to use them.
 extra_valid_entries:reviews,favs,follows
 
-[www.ficwad.com]
+[ficwad.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -1070,6 +1071,10 @@ extracategories:Harry Potter
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+[www.henneth-annun.net]
+## Site dedicated to these categories/characters/ships
+extracategories:The Hobbit
+
 [www.hpfandom.net]
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter
@@ -1157,6 +1162,15 @@ extracategories:Harry Potter
 ## Site dedicated to these categories/characters/ships
 extracategories:Prison Break
 
+[www.psychfic.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Psych
+
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
 [www.qaf-fic.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:Queer as Folk

downloader.py

@@ -69,13 +69,13 @@ def main():
                      help="Retrieve metadata and stop.  Or, if --update-epub, update metadata title page only.",)
    parser.add_option("-u", "--update-epub",
                      action="store_true", dest="update",
-                     help="Update an existing epub with new chapter, give epub filename instead of storyurl.",)
+                     help="Update an existing epub with new chapters, give epub filename instead of storyurl.",)
    parser.add_option("--update-cover",
                      action="store_true", dest="updatecover",
                      help="Update cover in an existing epub, otherwise existing cover (if any) is used on update.  Only valid with --update-epub.",)
    parser.add_option("--force",
                      action="store_true", dest="force",
-                     help="Force overwrite or update of an existing epub, download and overwrite all chapters.",)
+                     help="Force overwrite of an existing epub, download and overwrite all chapters.",)
    parser.add_option("-l", "--list",
                      action="store_true", dest="list",
                      help="Get list of valid story URLs from page given.",)
@@ -155,9 +155,7 @@ def main():
        return
 
    try:
-
        adapter = adapters.getAdapter(configuration,url)
-
        adapter.setChaptersRange(options.begin,options.end)
            
        ## Check for include_images and absence of PIL, give warning.
@@ -173,7 +171,6 @@ def main():
                    print "You have include_images enabled, but Python Image Library(PIL) isn't found.\nImages will be included full size in original format.\nContinue? (y/n)?"
                    if not sys.stdin.readline().strip().lower().startswith('y'):
                        return
-               
 
        ## three tries, that's enough if both user/pass & is_adult needed,
        ## or a couple tries of one or the other

fanficdownloader/adapters/__init__.py

@@ -113,6 +113,8 @@ import adapter_pommedesangcom
 import adapter_restrictedsectionorg
 import adapter_imagineeficcom
 import adapter_buffynfaithnet
+import adapter_psychficcom
+import adapter_hennethannunnet
 
 ## This bit of complexity allows adapters to be added by just adding
 ## importing.  It eliminates the long if/else clauses we used to need
@@ -149,7 +151,7 @@ getNormalStoryURL.__dummyconfig = None
 
 def getAdapter(config,url):
 
-    logger.debug("trying url:"+url)
+    #logger.debug("trying url:"+url)
     (cls,fixedurl) = getClassFor(url)
     logger.debug("fixedurl:"+fixedurl)
     if cls:
@@ -185,11 +187,11 @@ def getClassFor(url):
     cls = getClassFromList(domain)
     if not cls and domain.startswith("www."):
         domain = domain.replace("www.","")
-        logger.debug("trying site:without www: "+domain)
+        #logger.debug("trying site:without www: "+domain)
         cls = getClassFromList(domain)
         fixedurl = fixedurl.replace("http://www.","http://")
     if not cls:
-        logger.debug("trying site:www."+domain)
+        #logger.debug("trying site:www."+domain)
         cls = getClassFromList("www."+domain)
         fixedurl = fixedurl.replace("http://","http://www.")
         

fanficdownloader/adapters/adapter_archiveofourownorg.py

@@ -82,7 +82,8 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
 
     def getSiteURLPattern(self):
         # http://archiveofourown.org/collections/Smallville_Slash_Archive/works/159770
-        return re.escape("http://")+re.escape(self.getSiteDomain())+r"(/collections/[^/]+)?/works/(?P<id>\d+)"
+        # Discard leading zeros from story ID numbers--AO3 doesn't use them in it's own chapter URLs.
+        return re.escape("http://")+re.escape(self.getSiteDomain())+r"(/collections/[^/]+)?/works/0*(?P<id>\d+)"
         
     ## Login
     def needToLoginCheck(self, data):

fanficdownloader/adapters/adapter_archiveskyehawkecom.py

@@ -66,14 +66,15 @@ class ArchiveSkyeHawkeComAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'archive.skyehawke.com'
 
+    @classmethod
+    def getAcceptDomains(cls):
+        return ['archive.skyehawke.com','www.skyehawke.com']
+
     def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/story.php?no=1234"
+        return "http://archive.skyehawke.com/story.php?no=1234 http://www.skyehawke.com/archive/story.php?no=1234  http://skyehawke.com/archive/story.php?no=1234"
 
     def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/story.php?no=")+r"\d+$"
-
-
-        
+        return re.escape("http://")+r"(archive|www)\.skyehawke\.com/(archive/)?story\.php\?no=\d+$"
 
     ## Getting the chapter list and the meta data, plus 'is adult' checking.
     def extractChapterUrlsAndMetadata(self):

fanficdownloader/adapters/adapter_ficwadcom.py

@@ -43,10 +43,10 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
 
     @staticmethod
     def getSiteDomain():
-        return 'www.ficwad.com'
+        return 'ficwad.com'
 
     def getSiteExampleURLs(self):
-        return "http://www.ficwad.com/story/137169"
+        return "http://ficwad.com/story/137169"
 
     def getSiteURLPattern(self):
         return re.escape(r"http://"+self.getSiteDomain())+"/story/\d+?$"

fanficdownloader/adapters/adapter_fimfictionnet.py

@@ -181,10 +181,15 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
 
             self.setCoverImage(self.url,coverurl)
 
-        # the fimfic API gives bbcode for desc, not html.
-        # btw, bbcode honors newlines, html doesn't.  change newlines to br tags.
-        self.setDescription(self.url,
-                            bbcodeparser().parse(storyMetadata["description"]).html(doDeepCopy=False).replace('\r','').replace('\n','<br />'))
+        self.setDescription(self.url,soup.find("div", {"class":"description"}))
+        # if "description" in storyMetadata and storyMetadata["description"]:
+        #     # the fimfic API gives bbcode for desc, not html.
+        #     # btw, bbcode honors newlines, html doesn't.  change newlines to br tags.
+        #     self.setDescription(self.url,
+        #                         bbcodeparser().parse(storyMetadata["description"]).html(doDeepCopy=False).replace('\r','').replace('\n','<br />'))
+        # elif "short_description" in storyMetadata and storyMetadata["short_description"]:
+        #     self.setDescription(self.url,
+        #                         bbcodeparser().parse(storyMetadata["short_description"]).html(doDeepCopy=False).replace('\r','').replace('\n','<br />'))
         
         # Dates are in Unix time
         # Take the publish date from the first chapter posted
@@ -219,7 +224,7 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
             
     def getChapterText(self, url):
         logger.debug('Getting chapter text from: %s' % url)
-        soup = bs.BeautifulSoup(self._fetchUrl(url),selfClosingTags=('br','hr')).find('div', {'id' : 'chapter_container'})
+        soup = bs.BeautifulSoup(self._fetchUrl(url),selfClosingTags=('br','hr')).find('div', {'class' : 'chapter_content'})
         if soup == None:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
         return self.utf8FromSoup(url,soup)

fanficdownloader/adapters/adapter_hennethannunnet.py

@@ -0,0 +1,171 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return HennethAnnunNetAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class HennethAnnunNetAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/stories/chapter.cfm?stid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','htan')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%m/%d/%y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.henneth-annun.net'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/stories/chapter.cfm?stid=1234"
+
+    def getSiteURLPattern(self):
+        return "http://"+self.getSiteDomain()+"/stories/chapter(_view)?.cfm\?stid="+r"\d+$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+            
+        if "We're sorry. This story is not available." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: This story is not available.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('h2', {'id':'page_heading'})
+        self.story.setMetadata('title',a.string)
+
+        # Find the chapters: chapter_view.cfm?stid=6663&spordinal=1" 
+        for chapter in soup.findAll('a', href=re.compile(r'chapter_view.cfm\?stid='+self.story.getMetadata('storyId')+"&spordinal=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/stories/'+chapter['href']))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+        
+        self.story.setMetadata('numWords', soup.find('tr', {'class':'foot'}).findAll('td')[1].text)
+
+        self.setDescription(url,soup.find('div', {'id':'summary'}))
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        info = soup.find('div', {'id':'storyinformation'})
+        labels=info.findAll('b')
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+                
+            if 'Completion' in label:
+                if 'Complete' in value.string:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Rating' in label:
+                self.story.setMetadata('rating', value.string)
+
+            if 'Era:' in label:
+                self.story.addToList('category',value.string)
+
+            if 'Genre' in label:
+                self.story.addToList('genre',value.string)
+                
+        labels=info.findAll('strong')
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+            
+            if 'Author' in label:
+                value=value.nextSibling
+                self.story.setMetadata('authorId',value['href'].split('=')[1])
+                self.story.setMetadata('authorUrl','http://'+self.host+'/'+value['href'])
+                self.story.setMetadata('author',value.string)
+                
+            if 'Post' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated:' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+                
+        for char in soup.findAll('a', href=re.compile(r"/resources/bios_view.cfm\?scid=\d+")):
+            self.story.addToList('characters',stripHTML(char))
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'class' : 'block chapter'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_psychficcom.py

@@ -0,0 +1,246 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return PsychFicComAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class PsychFicComAdapter(BaseSiteAdapter): 
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','psyf')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%B %d, %Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'www.psychfic.com'
+
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=4"
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid & error in url.
+                addurl = addurl.replace("&","&")
+                url = self.url+'&index=1'+addurl
+                logger.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',a.string)
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        self.story.setMetadata('authorId',a['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('author',a.string)
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.text
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ""
+                while not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                for cat in cats:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
+            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
+            i=1
+            for a in storyas:
+                # skip 'report this' and 'TOC' links
+                if 'contact.php' not in a['href'] and 'index' not in a['href']:
+                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                        self.setSeries(series_name, i)
+                        break
+                    i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_restrictedsectionorg.py

@@ -205,10 +205,26 @@ class RestrictedSectionOrgSiteAdapter(BaseSiteAdapter):
         params['accept.x'] = 1
         params['accept.y'] = 1
 
-        data = self._postUrl(url, params)
-        if "I certify that I am over the age of 18 and that accessing the following story will not violate the laws of my country or local ordinances." in data:
-            raise exceptions.FailedToLogin(url,params['username'])
-        return data
+        excpt=None
+        for sleeptime in [0.5, 1.5, 4, 9]:
+            time.sleep(sleeptime)	
+            try:
+                data = self._postUrl(url, params)
+                if data == "Unable to connect to the database":
+                    raise exceptions.FailedToDownload("Site reported 'Unable to connect to the database'")
+                if "I certify that I am over the age of 18 and that accessing the following story will not violate the laws of my country or local ordinances." in data:
+                    raise exceptions.FailedToLogin(url,params['username'])
+                return data
+            except exceptions.FailedToLogin, ftl:
+                # no need to retry these.
+                raise(ftl)
+            except Exception, e:
+                excpt=e
+                logger.warn("Caught an exception reading URL: %s  Exception %s."%(unicode(url),unicode(e)))
+
+        logger.error("Giving up on %s" %url)
+        logger.exception(excpt)
+        raise(excpt)
         
     # grab the text for an individual chapter.
     def getChapterText(self, url):
@@ -216,6 +232,13 @@ class RestrictedSectionOrgSiteAdapter(BaseSiteAdapter):
         logger.debug('Getting chapter text from: %s' % url)
 
         data = self._postUrlUP(url)
+        #print("data:%s"%data)
+        
+        # some stories have html that confuses the parser.  For story
+        # text we don't care about anything before '<table id="page"'
+        # and seems to clear the issue.
+        data = data[data.index('<table id="page"'):]
+        
         soup = bs.BeautifulSoup(data)
         
         div = soup.find('td',{'id':'page_content'})

fanficdownloader/adapters/adapter_test1.py

@@ -101,9 +101,10 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
                 }
             self.story.setMetadata('language',langs[idnum%len(langs)])
             self.setSeries('The Great Test',idnum)
-        
+        if idnum == 0:
+            self.setSeries("A Nook Hyphen Test "+self.story.getMetadata('dateCreated'),idnum)
+            
         self.story.setMetadata('rating','Tweenie')
-        
 
         if self.story.getMetadata('storyId') == '673':
             self.story.addToList('author','Author From List')
@@ -114,6 +115,10 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
         
             self.story.addToList('authorUrl','http://author/url')
             self.story.addToList('authorUrl','http://author/url-2')
+            self.story.addToList('category','Power Rangers')
+            self.story.addToList('category','SG-1')
+            self.story.addToList('genre','Porn')
+            self.story.addToList('genre','Drama')
         else:
             self.story.setMetadata('authorId','98765')
             self.story.setMetadata('authorUrl','http://author/url')
@@ -161,9 +166,9 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
                             ('Chapter 3, Over Cinnabar',self.url+"&chapter=4"),
                             ('Chapter 4',self.url+"&chapter=5"),
                             ('Chapter 5',self.url+"&chapter=6"),
-                            #('Chapter 6',self.url+"&chapter=7"),
-                            #('Chapter 7',self.url+"&chapter=8"),
-                            #('Chapter 8',self.url+"&chapter=9"),
+                            ('Chapter 6',self.url+"&chapter=7"),
+                            ('Chapter 7',self.url+"&chapter=8"),
+                            ('Chapter 8',self.url+"&chapter=9"),
                             #('Chapter 9',self.url+"&chapter=0"),
                             #('Chapter 0',self.url+"&chapter=a"),
                             #('Chapter a',self.url+"&chapter=b"),
@@ -186,9 +191,6 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
 
     def getChapterText(self, url):
         logger.debug('Getting chapter text from: %s' % url)
-        if self.story.getMetadata('storyId') == '667':
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!" % url)
-
         if self.story.getMetadata('storyId').startswith('670') or \
                 self.story.getMetadata('storyId').startswith('672'):
             time.sleep(1.0)
@@ -201,20 +203,40 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
 <p>http://test1.com?sid=664 - Crazy string title</p>
 <p>http://test1.com?sid=665 - raises AdultCheckRequired</p>
 <p>http://test1.com?sid=666 - raises StoryDoesNotExist</p>
-<p>http://test1.com?sid=667 - raises FailedToDownload on chapter 1</p>
+<p>http://test1.com?sid=667 - raises FailedToDownload on chapters 2+</p>
 <p>http://test1.com?sid=668 - raises FailedToLogin unless username='Me'</p>
 <p>http://test1.com?sid=669 - Succeeds with Updated Date=now</p>
 <p>http://test1.com?sid=670 - Succeeds, but sleeps 2sec on each chapter</p>
+
+
+
+
 <p>http://test1.com?sid=671 - Succeeds, but sleeps 2sec metadata only</p>
 <p>http://test1.com?sid=672 - Succeeds, quick meta, sleeps 2sec chapters only</p>
-<p>http://test1.com?sid=673 - Succeeds, multiple authors</p>
+<p>http://test1.com?sid=673 - Succeeds, multiple authors, extra categories, genres</p>
+<p>http://test1.com?sid=0 - Succeeds, generates some text specifically for testing hyphenation problems with Nook STR/STRwG</p>
 <p>Odd sid's will be In-Progress, evens complete.  sid&lt;10 will be assigned one of four languages and included in a series.</p>
 </div>
+'''
+        elif self.story.getMetadata('storyId') == '0':
+            text=u'''
+<h3>45. Pronglet Returns to Hogwarts: Chapter 7</h3>
+<br />
+    eyes… but I’m not convinced we should automatically<br />
+<br /><br />
+<b>Thanks to the latest to recommend me: Alastor</b><br />
+<br /><br />
+    “Sure, invite her along. Does she have children?”<br />
+<br />
 '''
         else:
+            if self.story.getMetadata('storyId') == '667':
+                raise exceptions.FailedToDownload("Error downloading Chapter: %s!" % url)
+
             text=u'''
 <div>
 <h3>Chapter title from site</h3>
+<p>Timestamp:'''+datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")+'''</p>
 <p>Lorem '''+self.crazystring+u''' <i>italics</i>, <b>bold</b>, <u>underline</u> consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p>
 br breaks<br><br>
 Puella Magi Madoka Magica/魔法少女まどか★マギカ

fanficdownloader/adapters/adapter_tthfanficorg.py

@@ -100,8 +100,8 @@ class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter):
     
         if "Stories Published" not in d : #Member Account
             logger.info("Failed to login to URL %s as %s" % (loginUrl,
-                                                              params['penname']))
-            raise exceptions.FailedToLogin(url,params['penname'])
+                                                             params['urealname']))
+            raise exceptions.FailedToLogin(self.url,params['urealname'])
             return False
         else:
             return True

fanficdownloader/configurable.py

@@ -74,7 +74,8 @@ class Configuration(ConfigParser.SafeConfigParser):
             # internal stuff.
             'langcode',
             'output_css',
-            'authorHTML'
+            'authorHTML',
+            'lastupdate'
             ]
 
     def addConfigSection(self,section):

fanficdownloader/story.py

@@ -432,8 +432,13 @@ class Story(Configurable):
                                   map(removeAllEntities,retlist) )
 
         if retlist:
-            # remove dups and sort.
-            return sorted(list(set(retlist)))
+            if listname in ('author','authorUrl'):
+                # need to retain order for author & authorUrl so the
+                # two match up.
+                return retlist
+            else:
+                # remove dups and sort.
+                return sorted(list(set(retlist)))
         else:
             return []
 

fanficdownloader/writers/writer_epub.py

@@ -21,6 +21,7 @@ import StringIO
 import zipfile
 from zipfile import ZipFile, ZIP_STORED, ZIP_DEFLATED
 import urllib
+import re
 
 ## XML isn't as forgiving as HTML, so rather than generate as strings,
 ## use DOM to generate the XML files.
@@ -657,7 +658,12 @@ div { margin: 0pt; padding: 0pt; }
                 # as one line.  This causes problems for nook(at
                 # least) when the chapter size starts getting big
                 # (200k+)
-                fullhtml = fullhtml.replace('</p>','</p>\n').replace('<br />','<br />\n')
+                #fullhtml = fullhtml.replace('</p>','</p>\n').replace('<br />','<br />\n')
+                # The replaces above added tons of extra newlines
+                # during *each* epub update.  The regexp version adds
+                # only one and removes any extra.
+                fullhtml = re.sub(r'(</p>|<br />)\n*',r'\1\n',fullhtml)
+
                 outputepub.writestr("OEBPS/file%04d.xhtml"%(index+1),fullhtml.encode('utf-8'))
                 del fullhtml
 

index.html

@@ -57,8 +57,7 @@
 	  <h3>Changes:</h3>
 	  <p>
 	    <ul>
-	      <li>Fix for a change in fimfiction.net's handling of password protected stories.</li>
-	      <li>Fix for thewriterscoffeeshop.com's changed date format.</li>
+	      <li>Change how fimfiction.net story descriptions are collected.</li>
 	    </ul>
 	  </p>
 	  <p>
@@ -69,7 +68,7 @@
 	    If you have any problems with this application, please
 	    report them in
 	    the <a href="http://groups.google.com/group/fanfic-downloader">FanFictionDownLoader Google Group</a>.  The
-	    <a href="http://4-4-39.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
+	    <a href="http://4-4-45.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
 	  </p>
 	  <div id='error'>
 	    {{ error_message }}
@@ -293,6 +292,8 @@
 	  <dd>
 	    Use the URL of the story's summary, such as
 	    <br /><a href="http://archive.skyehawke.com/story.php?no=17466">http://archive.skyehawke.com/story.php?no=17466</a>.
+	    <br /><a href="http://www.skyehawke.com/archive/story.php?no=17466">http://www.skyehawke.com/archive/story.php?no=17466</a>.
+	    <br /><a href="http://skyehawke.com/archive/story.php?no=17466">http://skyehawke.com/archive/story.php?no=17466</a>.
 	  </dd>
 	  <dt>www.libraryofmoria.com</dt>
 	  <dd>
@@ -601,6 +602,16 @@
 	    <br />Or, use the URL of any story chapter, such as
 	    <br /><a href="http://buffynfaith.net/fanfictions/index.php?act=vie&id=1234&ch=2">http://buffynfaith.net/fanfictions/index.php?act=vie&amp;id=1234&amp;ch=2</a>
 	  </dd>
+	  <dt>www.henneth-annun.net</dt>
+	  <dd>
+	    Use the URL of the story's chapter list, such as
+	    <br /><a href="http://www.henneth-annun.net/stories/chapter.cfm?stid=1234">http://www.henneth-annun.net/stories/chapter.cfm?stid=1234</a>
+	  </dd>
+	  <dt>http://www.psychfic.com</dt>
+	  <dd>
+	    Use the URL of the story's chapter list, such as
+	    <br /><a href="http://www.psychfic.com/viewstory.php?sid=1234">http://www.psychfic.com/viewstory.php?sid=1234</a>
+	  </dd>
 	</dl>
 
 	<p>

makeplugin.py

@@ -27,7 +27,7 @@ if __name__=="__main__":
     exclude=['*.pyc','*~','*.xcf','*[0-9].png']
     # from top dir. 'w' for overwrite
     createZipFile(filename,"w",
-                  ['plugin-defaults.ini','plugin-example.ini','epubmerge.py','fanficdownloader'],
+                  ['plugin-defaults.ini','plugin-example.ini','fanficdownloader'],
                   exclude=exclude)
     #from calibre-plugin dir. 'a' for append
     os.chdir('calibre-plugin')

plugin-defaults.ini

@@ -157,7 +157,7 @@ extratags: FanFiction
 ## metadata part(s) to look at, 2) a regular expression to match the
 ## template, and 3) the name of the GC setting to use, which must
 ## match exactly.  Use this parameter in [defaults], or by site eg,
-## [www.ficwad.com]
+## [ficwad.com]
 ## Make sure to keep at least one space at the start of each line and
 ## to escape % to %%, if used.
 ## template => regexp to match => GC Setting to use.
@@ -296,12 +296,13 @@ background_color: ffffff
 ## Allow customization of CSS.  Make sure to keep at least one space
 ## at the start of each line and to escape % to %%.  Also need
 ## background_color to be in the same section, if included in CSS.
-## 'adobe-text-layout: optimizeSpeed;' prevents hyphenation on newer Nooks
+## 'adobe-hyphenate: none;' prevents hyphenation on newer Nooks
+## STR(wG) (1.2.1+ for sure)
 output_css: 
  body { background-color: #%(background_color)s;
         text-align: justify;
         margin: 2%%;
-	adobe-text-layout: optimizeSpeed; }
+	adobe-hyphenate: none; }
  pre { font-size: x-small; }
  sml { font-size: small; }
  h1 { text-align: center; }
@@ -343,7 +344,7 @@ output_css:
 ## It can be either a 'file:' or 'http:' url.
 ## Note that if you enable make_firstimage_cover in [epub], but want
 ## to use default_cover_image for a specific site, use the site:format
-## section, for example: [www.ficwad.com:epub]
+## section, for example: [ficwad.com:epub]
 ## default_cover_image is a python string Template string with
 ## ${title}, ${author} etc, same as titlepage_entries. Unless
 ## allow_unsafe_filename is true, invalid filename chars will be
@@ -1007,7 +1008,7 @@ extratags:
 ## for examples of how to use them.
 extra_valid_entries:reviews,favs,follows
 
-[www.ficwad.com]
+[ficwad.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -1047,6 +1048,10 @@ extracategories:Harry Potter
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+[www.henneth-annun.net]
+## Site dedicated to these categories/characters/ships
+extracategories:The Hobbit
+
 [www.hpfandom.net]
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter
@@ -1134,6 +1139,15 @@ extracategories:Harry Potter
 ## Site dedicated to these categories/characters/ships
 extracategories:Prison Break
 
+[www.psychfic.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Psych
+
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
 [www.qaf-fic.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:Queer as Folk

plugin-example.ini

@@ -51,7 +51,7 @@
 ## default is false
 #collect_series: true
 
-[www.ficwad.com]
+[ficwad.com]
 #username:YourUsername
 #password:YourPassword