Update CLI download zip.

maplebook using base_efiction.

app.yaml

@@ -1,6 +1,6 @@
 # ffd-retief-hrd fanfictiondownloader
 application: fanfictiondownloader
-version: 4-4-64
+version: 2-0-08
 runtime: python27
 api_version: 1
 threadsafe: true

calibre-plugin/__init__.py

@@ -7,6 +7,22 @@ __license__   = 'GPL v3'
 __copyright__ = '2013, Jim Miller'
 __docformat__ = 'restructuredtext en'
 
+import sys
+if sys.version_info >= (2, 7):
+    import logging
+    logger = logging.getLogger(__name__)
+    loghandler=logging.StreamHandler()
+    loghandler.setFormatter(logging.Formatter("FFDL:%(levelname)s:%(filename)s(%(lineno)d):%(message)s"))
+    logger.addHandler(loghandler)
+    loghandler.setLevel(logging.DEBUG)
+    logger.setLevel(logging.DEBUG)
+
+# pulls in translation files for _() strings
+try:
+    load_translations()
+except NameError:
+    pass # load_translations() added in calibre 1.9
+
 # The class that all Interface Action plugin wrappers must inherit from
 from calibre.customize import InterfaceActionBase
 
@@ -23,11 +39,11 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
     calibre utilities to run without needing to load the GUI libraries.
     '''
     name                = 'FanFictionDownLoader'
-    description         = 'UI plugin to download FanFiction stories from various sites.'
+    description         = _('UI plugin to download FanFiction stories from various sites.')
     supported_platforms = ['windows', 'osx', 'linux']
     author              = 'Jim Miller'
-    version             = (1, 7, 30)
-    minimum_calibre_version = (0, 8, 57)
+    version             = (2, 0, 8)
+    minimum_calibre_version = (1, 48, 0)
 
     #: This field defines the GUI plugin class that contains all the code
     #: that actually does something. Its format is module_path:class_name

calibre-plugin/common_utils.py

@@ -8,12 +8,19 @@ __copyright__ = '2011, Grant Drake <grant.drake@gmail.com>'
 __docformat__ = 'restructuredtext en'
 
 import os
-from PyQt4 import QtGui
-from PyQt4.Qt import (Qt, QIcon, QPixmap, QLabel, QDialog, QHBoxLayout,
-                      QTableWidgetItem, QFont, QLineEdit, QComboBox,
-                      QVBoxLayout, QDialogButtonBox, QStyledItemDelegate, QDateTime,
-                      QTextEdit,
-                      QListWidget, QAbstractItemView)
+try:
+    from PyQt5 import QtWidgets as QtGui
+    from PyQt5.Qt import (Qt, QIcon, QPixmap, QLabel, QDialog, QHBoxLayout,
+                          QTableWidgetItem, QFont, QLineEdit, QComboBox,
+                          QVBoxLayout, QDialogButtonBox, QStyledItemDelegate, QDateTime,
+                          QTextEdit, QListWidget, QAbstractItemView)
+except ImportError as e:
+    from PyQt4 import QtGui
+    from PyQt4.Qt import (Qt, QIcon, QPixmap, QLabel, QDialog, QHBoxLayout,
+                          QTableWidgetItem, QFont, QLineEdit, QComboBox,
+                          QVBoxLayout, QDialogButtonBox, QStyledItemDelegate, QDateTime,
+                          QTextEdit, QListWidget, QAbstractItemView)
+    
 from calibre.constants import iswindows
 from calibre.gui2 import gprefs, error_dialog, UNDEFINED_QDATETIME, info_dialog
 from calibre.gui2.actions import menu_action_unique_name

calibre-plugin/config.py

@@ -4,32 +4,84 @@ from __future__ import (unicode_literals, division, absolute_import,
                         print_function)
 
 __license__   = 'GPL v3'
-__copyright__ = '2012, Jim Miller'
+__copyright__ = '2014, Jim Miller'
 __docformat__ = 'restructuredtext en'
 
+import logging
+logger = logging.getLogger(__name__)
+
 import traceback, copy, threading
 from collections import OrderedDict
 
-from PyQt4.Qt import (QDialog, QWidget, QVBoxLayout, QHBoxLayout, QLabel,
-                      QLineEdit, QFont, QWidget, QTextEdit, QComboBox,
-                      QCheckBox, QPushButton, QTabWidget, QVariant, QScrollArea,
-                      QDialogButtonBox, QGroupBox )
+try:
+    from PyQt5.Qt import (QDialog, QWidget, QVBoxLayout, QHBoxLayout, QLabel,
+                          QLineEdit, QFont, QWidget, QTextEdit, QComboBox,
+                          QCheckBox, QPushButton, QTabWidget, QScrollArea,
+                          QDialogButtonBox, QGroupBox )
+except ImportError as e:
+    from PyQt4.Qt import (QDialog, QWidget, QVBoxLayout, QHBoxLayout, QLabel,
+                          QLineEdit, QFont, QWidget, QTextEdit, QComboBox,
+                          QCheckBox, QPushButton, QTabWidget, QScrollArea,
+                          QDialogButtonBox, QGroupBox )
+try:
+    from calibre.gui2 import QVariant
+    del QVariant
+except ImportError:
+    is_qt4 = False
+    convert_qvariant = lambda x: x
+else:
+    is_qt4 = True
+    def convert_qvariant(x):
+        vt = x.type()
+        if vt == x.String:
+            return unicode(x.toString())
+        if vt == x.List:
+            return [convert_qvariant(i) for i in x.toList()]
+        return x.toPyObject()
 
 from calibre.gui2.ui import get_gui
 from calibre.gui2 import dynamic, info_dialog
+from calibre.constants import numeric_version as calibre_version
+
+# pulls in translation files for _() strings
+try:
+    load_translations()
+except NameError:
+    pass # load_translations() added in calibre 1.9
+
+# There are a number of things used several times that shouldn't be
+# translated.  This is just a way to make that easier by keeping them
+# out of the _() strings.
+# I'm tempted to override _() to include them...
+no_trans = { 'pini':'personal.ini',
+             'imgset':'\n\n[epub]\ninclude_images:true\nkeep_summary_html:true\nmake_firstimage_cover:true\n\n',
+             'gcset':'generate_cover_settings',
+             'ccset':'custom_columns_settings',
+             'gc':'Generate Cover',
+             'rl':'Reading List',
+             'cp':'Count Pages',
+             'cmplt':'Completed',
+             'inprog':'In-Progress',
+             'lul':'Last Updated',
+             'lus':'lastupdate',
+             'is':'include_subject',
+             'isa':'is_adult',
+             'u':'username',
+             'p':'password',
+             }
 
 from calibre_plugins.fanfictiondownloader_plugin.prefs import prefs, PREFS_NAMESPACE
 from calibre_plugins.fanfictiondownloader_plugin.dialogs \
-    import (UPDATE, UPDATEALWAYS, OVERWRITE, collision_order, RejectListDialog,
+    import (UPDATE, UPDATEALWAYS, collision_order, save_collisions, RejectListDialog,
             EditTextDialog, RejectUrlEntry)
     
 from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.adapters \
-    import (getConfigSections, getNormalStoryURL)
+    import getConfigSections
 
 from calibre_plugins.fanfictiondownloader_plugin.common_utils \
     import ( KeyboardConfigDialog, PrefsViewerDialog )
 
-from calibre.gui2.complete import MultiCompleteLineEdit
+from calibre.gui2.complete2 import EditWithComplete  #MultiCompleteLineEdit
 
 class RejectURLList:
     def __init__(self,prefs):
@@ -121,15 +173,21 @@ class ConfigWidget(QWidget):
         self.l = QVBoxLayout()
         self.setLayout(self.l)
 
-        label = QLabel('<a href="http://code.google.com/p/fanficdownloader/wiki/FanFictionDownloaderSupportedsites">List of Supported Sites</a> -- <a href="http://code.google.com/p/fanficdownloader/wiki/FanFictionDownloaderFAQs">FAQs</a>')
+        label = QLabel('<a href="http://code.google.com/p/fanficdownloader/wiki/FanFictionDownloaderSupportedsites">'+_('List of Supported Sites')+'</a> -- <a href="http://code.google.com/p/fanficdownloader/wiki/FanFictionDownloaderFAQs">'+_('FAQs')+'</a>')
         label.setOpenExternalLinks(True)
         self.l.addWidget(label)
 
-        tab_widget = QTabWidget(self)
-        self.l.addWidget(tab_widget)
 
+        self.scroll_area = QScrollArea(self)
+        self.scroll_area.setFrameShape(QScrollArea.NoFrame)
+        self.scroll_area.setWidgetResizable(True)
+        self.l.addWidget(self.scroll_area)
+
+        tab_widget = QTabWidget(self)
+        self.scroll_area.setWidget(tab_widget)
+        
         self.basic_tab = BasicTab(self, plugin_action)
-        tab_widget.addTab(self.basic_tab, 'Basic')
+        tab_widget.addTab(self.basic_tab, _('Basic'))
 
         self.personalini_tab = PersonalIniTab(self, plugin_action)
         tab_widget.addTab(self.personalini_tab, 'personal.ini')
@@ -150,25 +208,29 @@ class ConfigWidget(QWidget):
             self.countpages_tab.setEnabled(False)
 
         self.std_columns_tab = StandardColumnsTab(self, plugin_action)
-        tab_widget.addTab(self.std_columns_tab, 'Standard Columns')
+        tab_widget.addTab(self.std_columns_tab, _('Standard Columns'))
 
         self.cust_columns_tab = CustomColumnsTab(self, plugin_action)
-        tab_widget.addTab(self.cust_columns_tab, 'Custom Columns')
+        tab_widget.addTab(self.cust_columns_tab, _('Custom Columns'))
 
         self.other_tab = OtherTab(self, plugin_action)
-        tab_widget.addTab(self.other_tab, 'Other')
+        tab_widget.addTab(self.other_tab, _('Other'))
 
 
     def save_settings(self):
 
         # basic
         prefs['fileform'] = unicode(self.basic_tab.fileform.currentText())
-        prefs['collision'] = unicode(self.basic_tab.collision.currentText())
+        prefs['collision'] = save_collisions[unicode(self.basic_tab.collision.currentText())]
         prefs['updatemeta'] = self.basic_tab.updatemeta.isChecked()
         prefs['updatecover'] = self.basic_tab.updatecover.isChecked()
         prefs['updateepubcover'] = self.basic_tab.updateepubcover.isChecked()
         prefs['keeptags'] = self.basic_tab.keeptags.isChecked()
+        prefs['suppressauthorsort'] = self.basic_tab.suppressauthorsort.isChecked()
+        prefs['suppresstitlesort'] = self.basic_tab.suppresstitlesort.isChecked()
+        prefs['mark'] = self.basic_tab.mark.isChecked()
         prefs['showmarked'] = self.basic_tab.showmarked.isChecked()
+        prefs['autoconvert'] = self.basic_tab.autoconvert.isChecked()
         prefs['urlsfromclip'] = self.basic_tab.urlsfromclip.isChecked()
         prefs['updatedefault'] = self.basic_tab.updatedefault.isChecked()
         prefs['deleteotherforms'] = self.basic_tab.deleteotherforms.isChecked()
@@ -176,7 +238,10 @@ class ConfigWidget(QWidget):
         prefs['includeimages'] = self.basic_tab.includeimages.isChecked()
         prefs['lookforurlinhtml'] = self.basic_tab.lookforurlinhtml.isChecked()
         prefs['checkforseriesurlid'] = self.basic_tab.checkforseriesurlid.isChecked()
+        prefs['checkforurlchange'] = self.basic_tab.checkforurlchange.isChecked()
         prefs['injectseries'] = self.basic_tab.injectseries.isChecked()
+        prefs['smarten_punctuation'] = self.basic_tab.smarten_punctuation.isChecked()
+        prefs['reject_always'] = self.basic_tab.reject_always.isChecked()
 
         if self.readinglist_tab:
             # lists
@@ -200,12 +265,13 @@ class ConfigWidget(QWidget):
         prefs['gcnewonly'] = self.generatecover_tab.gcnewonly.isChecked()
         gc_site_settings = {}
         for (site,combo) in self.generatecover_tab.gc_dropdowns.iteritems():
-            val = unicode(combo.itemData(combo.currentIndex()).toString())
+            val = unicode(convert_qvariant(combo.itemData(combo.currentIndex())))
             if val != 'none':
                 gc_site_settings[site] = val
                 #print("gc_site_settings[%s]:%s"%(site,gc_site_settings[site]))
         prefs['gc_site_settings'] = gc_site_settings
         prefs['allow_gc_from_ini'] = self.generatecover_tab.allow_gc_from_ini.isChecked()
+        prefs['gc_polish_cover'] = self.generatecover_tab.gc_polish_cover.isChecked()
 
         # Count Pages tab
         countpagesstats = []
@@ -231,12 +297,12 @@ class ConfigWidget(QWidget):
 
         # Custom Columns tab
         # error column
-        prefs['errorcol'] = unicode(self.cust_columns_tab.errorcol.itemData(self.cust_columns_tab.errorcol.currentIndex()).toString())
+        prefs['errorcol'] = unicode(convert_qvariant(self.cust_columns_tab.errorcol.itemData(self.cust_columns_tab.errorcol.currentIndex())))
 
         # cust cols tab
         colsmap = {}
         for (col,combo) in self.cust_columns_tab.custcol_dropdowns.iteritems():
-            val = unicode(combo.itemData(combo.currentIndex()).toString())
+            val = unicode(convert_qvariant(combo.itemData(combo.currentIndex())))
             if val != 'none':
                 colsmap[col] = val
                 #print("colsmap[%s]:%s"%(col,colsmap[col]))
@@ -269,17 +335,17 @@ class BasicTab(QWidget):
         topl = QVBoxLayout()
         self.setLayout(topl)
 
-        label = QLabel('These settings control the basic features of the plugin--downloading FanFiction.')
+        label = QLabel(_('These settings control the basic features of the plugin--downloading FanFiction.'))
         label.setWordWrap(True)
         topl.addWidget(label)
 
-        defs_gb = groupbox = QGroupBox("Defaults Options on Download")
+        defs_gb = groupbox = QGroupBox(_("Defaults Options on Download"))
         self.l = QVBoxLayout()
         groupbox.setLayout(self.l)
 
-        tooltip = "On each download, FFDL offers an option to select the output format. <br />This sets what that option will default to."
+        tooltip = _("On each download, FFDL offers an option to select the output format. <br />This sets what that option will default to.")
         horz = QHBoxLayout()
-        label = QLabel('Default Output &Format:')
+        label = QLabel(_('Default Output &Format:'))
         label.setToolTip(tooltip)
         horz.addWidget(label)
         self.fileform = QComboBox(self)
@@ -294,15 +360,15 @@ class BasicTab(QWidget):
         horz.addWidget(self.fileform)
         self.l.addLayout(horz)
 
-        tooltip = "On each download, FFDL offers an option of what happens if that story already exists. <br />This sets what that option will default to."
+        tooltip = _("On each download, FFDL offers an option of what happens if that story already exists. <br />This sets what that option will default to.")
         horz = QHBoxLayout()
-        label = QLabel('Default If Story Already Exists?')
+        label = QLabel(_('Default If Story Already Exists?'))
         label.setToolTip(tooltip)
         horz.addWidget(label)
         self.collision = QComboBox(self)
         # add collision options
         self.set_collisions()
-        i = self.collision.findText(prefs['collision'])
+        i = self.collision.findText(save_collisions[prefs['collision']])
         if i > -1:
             self.collision.setCurrentIndex(i)
         self.collision.setToolTip(tooltip)
@@ -310,116 +376,153 @@ class BasicTab(QWidget):
         horz.addWidget(self.collision)
         self.l.addLayout(horz)
 
-        self.updatemeta = QCheckBox('Default Update Calibre &Metadata?',self)
-        self.updatemeta.setToolTip("On each download, FFDL offers an option to update Calibre's metadata (title, author, URL, tags, custom columns, etc) from the web site. <br />This sets whether that will default to on or off. <br />Columns set to 'New Only' in the column tabs will only be set for new books.")
+        self.updatemeta = QCheckBox(_('Default Update Calibre &Metadata?'),self)
+        self.updatemeta.setToolTip(_("On each download, FFDL offers an option to update Calibre's metadata (title, author, URL, tags, custom columns, etc) from the web site. <br />This sets whether that will default to on or off. <br />Columns set to 'New Only' in the column tabs will only be set for new books."))
         self.updatemeta.setChecked(prefs['updatemeta'])
         self.l.addWidget(self.updatemeta)
 
-        self.updateepubcover = QCheckBox('Default Update EPUB Cover when Updating EPUB?',self)
-        self.updateepubcover.setToolTip("On each download, FFDL offers an option to update the book cover image <i>inside</i> the EPUB from the web site when the EPUB is updated.<br />This sets whether that will default to on or off.")
+        self.updateepubcover = QCheckBox(_('Default Update EPUB Cover when Updating EPUB?'),self)
+        self.updateepubcover.setToolTip(_("On each download, FFDL offers an option to update the book cover image <i>inside</i> the EPUB from the web site when the EPUB is updated.<br />This sets whether that will default to on or off."))
         self.updateepubcover.setChecked(prefs['updateepubcover'])
         self.l.addWidget(self.updateepubcover)
 
-        cali_gb = groupbox = QGroupBox("Updating Calibre Options")
+        self.smarten_punctuation = QCheckBox(_('Smarten Punctuation (EPUB only)'),self)
+        self.smarten_punctuation.setToolTip(_("Run Smarten Punctuation from Calibre's Polish Book feature on each EPUB download and update."))
+        self.smarten_punctuation.setChecked(prefs['smarten_punctuation'])
+        if calibre_version >= (0, 9, 39):
+            self.l.addWidget(self.smarten_punctuation)
+
+        cali_gb = groupbox = QGroupBox(_("Updating Calibre Options"))
         self.l = QVBoxLayout()
         groupbox.setLayout(self.l)
 
-        self.deleteotherforms = QCheckBox('Delete other existing formats?',self)
-        self.deleteotherforms.setToolTip('Check this to automatically delete all other ebook formats when updating an existing book.\nHandy if you have both a Nook(epub) and Kindle(mobi), for example.')
+        self.deleteotherforms = QCheckBox(_('Delete other existing formats?'),self)
+        self.deleteotherforms.setToolTip(_('Check this to automatically delete all other ebook formats when updating an existing book.\nHandy if you have both a Nook(epub) and Kindle(mobi), for example.'))
         self.deleteotherforms.setChecked(prefs['deleteotherforms'])
         self.l.addWidget(self.deleteotherforms)
         
-        self.updatecover = QCheckBox('Update Calibre Cover when Updating Metadata?',self)
-        self.updatecover.setToolTip("Update calibre book cover image from EPUB when metadata is updated.  (EPUB only.)\nDoesn't go looking for new images on 'Update Calibre Metadata Only'.")
+        self.updatecover = QCheckBox(_('Update Calibre Cover when Updating Metadata?'),self)
+        self.updatecover.setToolTip(_("Update calibre book cover image from EPUB when metadata is updated.  (EPUB only.)\nDoesn't go looking for new images on 'Update Calibre Metadata Only'."))
         self.updatecover.setChecked(prefs['updatecover'])
         self.l.addWidget(self.updatecover)
 
-        self.keeptags = QCheckBox('Keep Existing Tags when Updating Metadata?',self)
-        self.keeptags.setToolTip("Existing tags will be kept and any new tags added.\nCompleted and In-Progress tags will be still be updated, if known.\nLast Updated tags will be updated if lastupdate in include_subject_tags.\n(If Tags is set to 'New Only' in the Standard Columns tab, this has no effect.)")
+        self.keeptags = QCheckBox(_('Keep Existing Tags when Updating Metadata?'),self)
+        self.keeptags.setToolTip(_("Existing tags will be kept and any new tags added.\n%(cmplt)s and %(inprog)s tags will be still be updated, if known.\n%(lul)s tags will be updated if %(lus)s in %(is)s.\n(If Tags is set to 'New Only' in the Standard Columns tab, this has no effect.)")%no_trans)
         self.keeptags.setChecked(prefs['keeptags'])
         self.l.addWidget(self.keeptags)
 
-        self.checkforseriesurlid = QCheckBox("Check for existing Series Anthology books?",self)
-        self.checkforseriesurlid.setToolTip("Check for existings Series Anthology books using each new story's series URL before downloading.\nOffer to skip downloading if a Series Anthology is found.")
+        self.suppressauthorsort = QCheckBox(_('Force Author into Author Sort?'),self)
+        self.suppressauthorsort.setToolTip(_("If checked, the author(s) as given will be used for the Author Sort, too.\nIf not checked, calibre will apply it's built in algorithm which makes 'Bob Smith' sort as 'Smith, Bob', etc."))
+        self.suppressauthorsort.setChecked(prefs['suppressauthorsort'])
+        self.l.addWidget(self.suppressauthorsort)
+
+        self.suppresstitlesort = QCheckBox(_('Force Title into Title Sort?'),self)
+        self.suppresstitlesort.setToolTip(_("If checked, the title as given will be used for the Title Sort, too.\nIf not checked, calibre will apply it's built in algorithm which makes 'The Title' sort as 'Title, The', etc."))
+        self.suppresstitlesort.setChecked(prefs['suppresstitlesort'])
+        self.l.addWidget(self.suppresstitlesort)
+
+        self.checkforseriesurlid = QCheckBox(_("Check for existing Series Anthology books?"),self)
+        self.checkforseriesurlid.setToolTip(_("Check for existings Series Anthology books using each new story's series URL before downloading.\nOffer to skip downloading if a Series Anthology is found."))
         self.checkforseriesurlid.setChecked(prefs['checkforseriesurlid'])
         self.l.addWidget(self.checkforseriesurlid)
 
-        self.lookforurlinhtml = QCheckBox("Search EPUB text for Story URL?",self)
-        self.lookforurlinhtml.setToolTip("Look for first valid story URL inside EPUB text if not found in metadata.\nSomewhat risky, could find wrong URL depending on EPUB content.\nAlso finds and corrects bad ffnet URLs from ficsaver.com files.")
+        self.checkforurlchange = QCheckBox(_("Check for changed Story URL?"),self)
+        self.checkforurlchange.setToolTip(_("Warn you if an update will change the URL of an existing book.\nfanfiction.net URLs will change from http to https silently."))
+        self.checkforurlchange.setChecked(prefs['checkforurlchange'])
+        self.l.addWidget(self.checkforurlchange)
+
+        self.lookforurlinhtml = QCheckBox(_("Search EPUB text for Story URL?"),self)
+        self.lookforurlinhtml.setToolTip(_("Look for first valid story URL inside EPUB text if not found in metadata.\nSomewhat risky, could find wrong URL depending on EPUB content.\nAlso finds and corrects bad ffnet URLs from ficsaver.com files."))
         self.lookforurlinhtml.setChecked(prefs['lookforurlinhtml'])
         self.l.addWidget(self.lookforurlinhtml)
 
-        self.showmarked = QCheckBox("Show added/updated books when finished?",self)
-        self.showmarked.setToolTip("Show added/updated books only when finished.\nYou can also manually search for 'marked:ffdl_success'.\n'marked:ffdl_failed' is also available, or search 'marked:ffdl' for both.")
+        self.mark = QCheckBox(_("Mark added/updated books when finished?"),self)
+        self.mark.setToolTip(_("Mark added/updated books when finished.  Use with option below.\nYou can also manually search for 'marked:ffdl_success'.\n'marked:ffdl_failed' is also available, or search 'marked:ffdl' for both."))
+        self.mark.setChecked(prefs['mark'])
+        self.l.addWidget(self.mark)
+
+        self.showmarked = QCheckBox(_("Show Marked books when finished?"),self)
+        self.showmarked.setToolTip(_("Show Marked added/updated books only when finished.\nYou can also manually search for 'marked:ffdl_success'.\n'marked:ffdl_failed' is also available, or search 'marked:ffdl' for both."))
         self.showmarked.setChecked(prefs['showmarked'])
         self.l.addWidget(self.showmarked)
 
-        gui_gb = groupbox = QGroupBox("GUI Options")
+        self.autoconvert = QCheckBox(_("Automatically Convert new/update books?"),self)
+        self.autoconvert.setToolTip(_("Automatically call calibre's Convert for new/update books.\nConverts to the current output format as chosen in calibre's\nPreferences->Behavior settings."))
+        self.autoconvert.setChecked(prefs['autoconvert'])
+        self.l.addWidget(self.autoconvert)
+
+        gui_gb = groupbox = QGroupBox(_("GUI Options"))
         self.l = QVBoxLayout()
         groupbox.setLayout(self.l)
 
-        self.urlsfromclip = QCheckBox('Take URLs from Clipboard?',self)
-        self.urlsfromclip.setToolTip('Prefill URLs from valid URLs in Clipboard when Adding New.')
+        self.urlsfromclip = QCheckBox(_('Take URLs from Clipboard?'),self)
+        self.urlsfromclip.setToolTip(_('Prefill URLs from valid URLs in Clipboard when Adding New.'))
         self.urlsfromclip.setChecked(prefs['urlsfromclip'])
         self.l.addWidget(self.urlsfromclip)
 
-        self.updatedefault = QCheckBox('Default to Update when books selected?',self)
-        self.updatedefault.setToolTip('The top FanFictionDownLoader plugin button will start Update if\n'+
-                                      'books are selected.  If unchecked, it will always bring up \'Add New\'.')
+        self.updatedefault = QCheckBox(_('Default to Update when books selected?'),self)
+        self.updatedefault.setToolTip(_('The top FanFictionDownLoader plugin button will start Update if\nbooks are selected.  If unchecked, it will always bring up \'Add New\'.'))
         self.updatedefault.setChecked(prefs['updatedefault'])
         self.l.addWidget(self.updatedefault)
 
-        self.adddialogstaysontop = QCheckBox("Keep 'Add New from URL(s)' dialog on top?",self)
-        self.adddialogstaysontop.setToolTip("Instructs the OS and Window Manager to keep the 'Add New from URL(s)'\ndialog on top of all other windows.  Useful for dragging URLs onto it.")
+        self.adddialogstaysontop = QCheckBox(_("Keep 'Add New from URL(s)' dialog on top?"),self)
+        self.adddialogstaysontop.setToolTip(_("Instructs the OS and Window Manager to keep the 'Add New from URL(s)'\ndialog on top of all other windows.  Useful for dragging URLs onto it."))
         self.adddialogstaysontop.setChecked(prefs['adddialogstaysontop'])
         self.l.addWidget(self.adddialogstaysontop)
 
-        misc_gb = groupbox = QGroupBox("Misc Options")
+        misc_gb = groupbox = QGroupBox(_("Misc Options"))
         self.l = QVBoxLayout()
         groupbox.setLayout(self.l)
 
         # this is a cheat to make it easier for users to realize there's a new include_images features.
-        self.includeimages = QCheckBox("Include images in EPUBs?",self)
-        self.includeimages.setToolTip("Download and include images in EPUB stories.  This is equivalent to adding:\n\n[epub]\ninclude_images:true\nkeep_summary_html:true\nmake_firstimage_cover:true\n\n ...to the top of personal.ini.  Your settings in personal.ini will override this.")
+        self.includeimages = QCheckBox(_("Include images in EPUBs?"),self)
+        self.includeimages.setToolTip(_("Download and include images in EPUB stories.  This is equivalent to adding:%(imgset)s ...to the top of %(pini)s.  Your settings in %(pini)s will override this.")%no_trans)
         self.includeimages.setChecked(prefs['includeimages'])
         self.l.addWidget(self.includeimages)
 
-        self.injectseries = QCheckBox("Inject calibre Series when none found?",self)
-        self.injectseries.setToolTip("If no series is found, inject the calibre series (if there is one) so it appears on the FFDL title page(not cover).")
+        self.injectseries = QCheckBox(_("Inject calibre Series when none found?"),self)
+        self.injectseries.setToolTip(_("If no series is found, inject the calibre series (if there is one) so it appears on the FFDL title page(not cover)."))
         self.injectseries.setChecked(prefs['injectseries'])
         self.l.addWidget(self.injectseries)
 
-        rej_gb = groupbox = QGroupBox("Reject List")
+        rej_gb = groupbox = QGroupBox(_("Reject List"))
         self.l = QVBoxLayout()
         groupbox.setLayout(self.l)
 
-        self.rejectlist = QPushButton('Edit Reject URL List', self)
-        self.rejectlist.setToolTip("Edit list of URLs FFDL will automatically Reject.")
+        self.rejectlist = QPushButton(_('Edit Reject URL List'), self)
+        self.rejectlist.setToolTip(_("Edit list of URLs FFDL will automatically Reject."))
         self.rejectlist.clicked.connect(self.show_rejectlist)
         self.l.addWidget(self.rejectlist)
         
-        self.reject_urls = QPushButton('Add Reject URLs', self)
-        self.reject_urls.setToolTip("Add additional URLs to Reject as text.")
+        self.reject_urls = QPushButton(_('Add Reject URLs'), self)
+        self.reject_urls.setToolTip(_("Add additional URLs to Reject as text."))
         self.reject_urls.clicked.connect(self.add_reject_urls)
         self.l.addWidget(self.reject_urls)
         
-        self.reject_reasons = QPushButton('Edit Reject Reasons List', self)
-        self.reject_reasons.setToolTip("Customize the Reasons presented when Rejecting URLs")
+        self.reject_reasons = QPushButton(_('Edit Reject Reasons List'), self)
+        self.reject_reasons.setToolTip(_("Customize the Reasons presented when Rejecting URLs"))
         self.reject_reasons.clicked.connect(self.show_reject_reasons)
         self.l.addWidget(self.reject_reasons)
 
+        self.reject_always = QCheckBox(_('Reject Without Confirmation?'),self)
+        self.reject_always.setToolTip(_("Always reject URLs on the Reject List without stopping and asking."))
+        self.reject_always.setChecked(prefs['reject_always'])
+        self.l.addWidget(self.reject_always)
+
         topl.addWidget(defs_gb)
 
         horz = QHBoxLayout()
-        topl.addLayout(horz)
-        horz.addWidget(cali_gb)
-        horz.addWidget(rej_gb)
-        
-        horz = QHBoxLayout()
-        topl.addLayout(horz)
-        horz.addWidget(gui_gb)
-        horz.addWidget(misc_gb)
 
+        horz.addWidget(cali_gb)
+
+        vert = QVBoxLayout()
+        vert.addWidget(gui_gb)
+        vert.addWidget(misc_gb)
+        vert.addWidget(rej_gb)
+        
+        horz.addLayout(vert)
+        
+        topl.addLayout(horz)
         topl.insertStretch(-1)
         
     def set_collisions(self):
@@ -440,7 +543,7 @@ class BasicTab(QWidget):
         d = RejectListDialog(self,
                              rejecturllist.get_list(),
                              rejectreasons=rejecturllist.get_reject_reasons(),
-                             header="Edit Reject URLs List",
+                             header=_("Edit Reject URLs List"),
                              show_delete=False,
                              show_all_reasons=False)
         d.exec_()
@@ -454,22 +557,22 @@ class BasicTab(QWidget):
         d = EditTextDialog(self,
                            prefs['rejectreasons'],
                            icon=self.windowIcon(),
-                           title="Reject Reasons",
-                           label="Customize Reject List Reasons",
-                           tooltip="Customize the Reasons presented when Rejecting URLs")
+                           title=_("Reject Reasons"),
+                           label=_("Customize Reject List Reasons"),
+                           tooltip=_("Customize the Reasons presented when Rejecting URLs"))
         d.exec_()
         if d.result() == d.Accepted:
             prefs['rejectreasons'] = d.get_plain_text()
                 
     def add_reject_urls(self):
         d = EditTextDialog(self,
-                           "http://example.com/story.php?sid=5,Reason why I rejected it\nhttp://example.com/story.php?sid=6,Title by Author - Reason why I rejected it",
+                           "http://example.com/story.php?sid=5,"+_("Reason why I rejected it")+"\nhttp://example.com/story.php?sid=6,"+_("Title by Author")+" - "+_("Reason why I rejected it"),
                            icon=self.windowIcon(),
-                           title="Add Reject URLs",
-                           label="Add Reject URLs. Use: <b>http://...,note</b> or <b>http://...,title by author - note</b><br>Invalid story URLs will be ignored.",
-                           tooltip="One URL per line:\n<b>http://...,note</b>\n<b>http://...,title by author - note</b>",
+                           title=_("Add Reject URLs"),
+                           label=_("Add Reject URLs. Use: <b>http://...,note</b> or <b>http://...,title by author - note</b><br>Invalid story URLs will be ignored."),
+                           tooltip=_("One URL per line:\n<b>http://...,note</b>\n<b>http://...,title by author - note</b>"),
                            rejectreasons=rejecturllist.get_reject_reasons(),
-                           reasonslabel='Add this reason to all URLs added:')
+                           reasonslabel=_('Add this reason to all URLs added:'))
         d.exec_()
         if d.result() == d.Accepted:
             rejecturllist.add_text(d.get_plain_text(),d.get_reason_text())
@@ -484,7 +587,7 @@ class PersonalIniTab(QWidget):
         self.l = QVBoxLayout()
         self.setLayout(self.l)
 
-        label = QLabel('These settings provide more detailed control over what metadata will be displayed inside the ebook as well as let you set is_adult and user/password for different sites.')
+        label = QLabel(_('These settings provide more detailed control over what metadata will be displayed inside the ebook as well as let you set %(isa)s and %(u)s/%(p)s for different sites.')%no_trans)
         label.setWordWrap(True)
         self.l.addWidget(label)
         self.l.addSpacing(5)
@@ -495,15 +598,15 @@ class PersonalIniTab(QWidget):
         self.ini = QTextEdit(self)
         try:
             self.ini.setFont(QFont("Courier",
-                                   self.plugin_action.gui.font().pointSize()+1));
+                                   self.plugin_action.gui.font().pointSize()+1))
         except Exception as e:
-            print("Couldn't get font: %s"%e)
+            logger.error("Couldn't get font: %s"%e)
         self.ini.setLineWrapMode(QTextEdit.NoWrap)
         self.ini.setText(prefs['personal.ini'])
         self.l.addWidget(self.ini)
 
-        self.defaults = QPushButton('View Defaults (plugin-defaults.ini)', self)
-        self.defaults.setToolTip("View all of the plugin's configurable settings\nand their default settings.")
+        self.defaults = QPushButton(_('View Defaults')+' (plugin-defaults.ini)', self)
+        self.defaults.setToolTip(_("View all of the plugin's configurable settings\nand their default settings."))
         self.defaults.clicked.connect(self.show_defaults)
         self.l.addWidget(self.defaults)
         
@@ -521,25 +624,25 @@ class ShowDefaultsIniDialog(QDialog):
         self.resize(600, 500)
         self.l = QVBoxLayout()
         self.setLayout(self.l)
-        self.label = QLabel("Plugin Defaults (plugin-defaults.ini) (Read-Only)")
-        self.label.setToolTip("These are all of the plugin's configurable options\nand their default settings.")
+        self.label = QLabel(_("Plugin Defaults (%s) (Read-Only)")%'plugin-defaults.ini')
+        self.label.setToolTip(_("These are all of the plugin's configurable options\nand their default settings."))
         self.setWindowTitle(_('Plugin Defaults'))
         self.setWindowIcon(icon)
         self.l.addWidget(self.label)
         
         self.ini = QTextEdit(self)
-        self.ini.setToolTip("These are all of the plugin's configurable options\nand their default settings.")
+        self.ini.setToolTip(_("These are all of the plugin's configurable options\nand their default settings."))
         try:
             self.ini.setFont(QFont("Courier",
-                                   get_gui().font().pointSize()+1));
+                                   get_gui().font().pointSize()+1))
         except Exception as e:
-            print("Couldn't get font: %s"%e)
+            logger.error("Couldn't get font: %s"%e)
         self.ini.setLineWrapMode(QTextEdit.NoWrap)
         self.ini.setText(text)
         self.ini.setReadOnly(True)
         self.l.addWidget(self.ini)
         
-        self.ok_button = QPushButton('OK', self)
+        self.ok_button = QPushButton(_('OK'), self)
         self.ok_button.clicked.connect(self.hide)
         self.l.addWidget(self.ok_button)
         
@@ -559,45 +662,45 @@ class ReadingListTab(QWidget):
         except KeyError:
             reading_lists= []
             
-        label = QLabel('These settings provide integration with the Reading List Plugin.  Reading List can automatically send to devices and change custom columns.  You have to create and configure the lists in Reading List to be useful.')
+        label = QLabel(_('These settings provide integration with the %(rl)s Plugin.  %(rl)s can automatically send to devices and change custom columns.  You have to create and configure the lists in %(rl)s to be useful.')%no_trans)
         label.setWordWrap(True)
         self.l.addWidget(label)
         self.l.addSpacing(5)
         
-        self.addtolists = QCheckBox('Add new/updated stories to "Send to Device" Reading List(s).',self)
-        self.addtolists.setToolTip('Automatically add new/updated stories to these lists in the Reading List plugin.')
+        self.addtolists = QCheckBox(_('Add new/updated stories to "Send to Device" Reading List(s).'),self)
+        self.addtolists.setToolTip(_('Automatically add new/updated stories to these lists in the %(rl)s plugin.')%no_trans)
         self.addtolists.setChecked(prefs['addtolists'])
         self.l.addWidget(self.addtolists)
             
         horz = QHBoxLayout()
-        label = QLabel('"Send to Device" Reading Lists')
-        label.setToolTip("When enabled, new/updated stories will be automatically added to these lists.")
+        label = QLabel(_('"Send to Device" Reading Lists'))
+        label.setToolTip(_("When enabled, new/updated stories will be automatically added to these lists."))
         horz.addWidget(label)        
-        self.send_lists_box = MultiCompleteLineEdit(self)
-        self.send_lists_box.setToolTip("When enabled, new/updated stories will be automatically added to these lists.")
+        self.send_lists_box = EditWithComplete(self)
+        self.send_lists_box.setToolTip(_("When enabled, new/updated stories will be automatically added to these lists."))
         self.send_lists_box.update_items_cache(reading_lists)
         self.send_lists_box.setText(prefs['send_lists'])
         horz.addWidget(self.send_lists_box)
         self.l.addLayout(horz)
         
-        self.addtoreadlists = QCheckBox('Add new/updated stories to "To Read" Reading List(s).',self)
-        self.addtoreadlists.setToolTip('Automatically add new/updated stories to these lists in the Reading List plugin.\nAlso offers menu option to remove stories from the "To Read" lists.')
+        self.addtoreadlists = QCheckBox(_('Add new/updated stories to "To Read" Reading List(s).'),self)
+        self.addtoreadlists.setToolTip(_('Automatically add new/updated stories to these lists in the %(rl)s plugin.\nAlso offers menu option to remove stories from the "To Read" lists.')%no_trans)
         self.addtoreadlists.setChecked(prefs['addtoreadlists'])
         self.l.addWidget(self.addtoreadlists)
             
         horz = QHBoxLayout()
-        label = QLabel('"To Read" Reading Lists')
-        label.setToolTip("When enabled, new/updated stories will be automatically added to these lists.")
+        label = QLabel(_('"To Read" Reading Lists'))
+        label.setToolTip(_("When enabled, new/updated stories will be automatically added to these lists."))
         horz.addWidget(label)        
-        self.read_lists_box = MultiCompleteLineEdit(self)
-        self.read_lists_box.setToolTip("When enabled, new/updated stories will be automatically added to these lists.")
+        self.read_lists_box = EditWithComplete(self)
+        self.read_lists_box.setToolTip(_("When enabled, new/updated stories will be automatically added to these lists."))
         self.read_lists_box.update_items_cache(reading_lists)
         self.read_lists_box.setText(prefs['read_lists'])
         horz.addWidget(self.read_lists_box)
         self.l.addLayout(horz)
         
-        self.addtolistsonread = QCheckBox('Add stories back to "Send to Device" Reading List(s) when marked "Read".',self)
-        self.addtolistsonread.setToolTip('Menu option to remove from "To Read" lists will also add stories back to "Send to Device" Reading List(s)')
+        self.addtolistsonread = QCheckBox(_('Add stories back to "Send to Device" Reading List(s) when marked "Read".'),self)
+        self.addtolistsonread.setToolTip(_('Menu option to remove from "To Read" lists will also add stories back to "Send to Device" Reading List(s)'))
         self.addtolistsonread.setChecked(prefs['addtolistsonread'])
         self.l.addWidget(self.addtolistsonread)
             
@@ -619,7 +722,7 @@ class GenerateCoverTab(QWidget):
         except KeyError:
             gc_settings= []
             
-        label = QLabel('The Generate Cover plugin can create cover images for books using various metadata and configurations.  If you have GC installed, FFDL can run GC on new downloads and metadata updates.  Pick a GC setting by site or Default.')
+        label = QLabel(_('The %(gc)s plugin can create cover images for books using various metadata and configurations.  If you have GC installed, FFDL can run GC on new downloads and metadata updates.  Pick a GC setting by site or Default.')%no_trans)
         label.setWordWrap(True)
         self.l.addWidget(label)
         self.l.addSpacing(5)
@@ -637,41 +740,52 @@ class GenerateCoverTab(QWidget):
 
         sitelist = getConfigSections()
         sitelist.sort()
-        sitelist.insert(0,u"Default")
+        sitelist.insert(0,_("Default"))
         for site in sitelist:
             horz = QHBoxLayout()
             label = QLabel(site)
-            if site == u"Default":
-                s = "On Metadata update, run Generate Cover with this setting, if not selected for specific site."
+            if site == _("Default"):
+                s = _("On Metadata update, run %(gc)s with this setting, if not selected for specific site.")%no_trans
             else:
-                s = "On Metadata update, run Generate Cover with this setting for %s stories."%site
+                no_trans['site']=site # not ideal, but, meh.
+                s = _("On Metadata update, run %(gc)s with this setting for %(site)s stories.")%no_trans
 
             label.setToolTip(s)
             horz.addWidget(label)
             dropdown = QComboBox(self)
             dropdown.setToolTip(s)
-            dropdown.addItem('',QVariant('none'))
+            dropdown.addItem('','none')
             for setting in gc_settings:
-                dropdown.addItem(setting,QVariant(setting))
-            self.gc_dropdowns[site] = dropdown
+                dropdown.addItem(setting,setting)
+            if site == _("Default"):
+                self.gc_dropdowns["Default"] = dropdown
+                if 'Default' in prefs['gc_site_settings']:
+                    dropdown.setCurrentIndex(dropdown.findData(prefs['gc_site_settings']['Default']))
+            else:
+                self.gc_dropdowns[site] = dropdown
             if site in prefs['gc_site_settings']:
-                dropdown.setCurrentIndex(dropdown.findData(QVariant(prefs['gc_site_settings'][site])))
+                dropdown.setCurrentIndex(dropdown.findData(prefs['gc_site_settings'][site]))
 
             horz.addWidget(dropdown)
             self.sl.addLayout(horz)
         
         self.sl.insertStretch(-1)
         
-        self.gcnewonly = QCheckBox("Run Generate Cover Only on New Books",self)
-        self.gcnewonly.setToolTip("Default is to run GC any time the calibre metadata is updated.")
+        self.gcnewonly = QCheckBox(_("Run %(gc)s Only on New Books")%no_trans,self)
+        self.gcnewonly.setToolTip(_("Default is to run GC any time the calibre metadata is updated."))
         self.gcnewonly.setChecked(prefs['gcnewonly'])
         self.l.addWidget(self.gcnewonly)
 
-        self.allow_gc_from_ini = QCheckBox('Allow generate_cover_settings from personal.ini to override',self)
-        self.allow_gc_from_ini.setToolTip("The personal.ini parameter generate_cover_settings allows you to choose a GC setting based on metadata rather than site, but it's much more complex.<br \>generate_cover_settings is ignored when this is off.")
+        self.allow_gc_from_ini = QCheckBox(_('Allow %(gcset)s from %(pini)s to override')%no_trans,self)
+        self.allow_gc_from_ini.setToolTip(_("The %(pini)s parameter %(gcset)s allows you to choose a GC setting based on metadata rather than site, but it's much more complex.<br \>%(gcset)s is ignored when this is off.")%no_trans)
         self.allow_gc_from_ini.setChecked(prefs['allow_gc_from_ini'])
         self.l.addWidget(self.allow_gc_from_ini)
             
+        self.gc_polish_cover = QCheckBox(_("Use calibre's Polish feature to inject/update the cover"),self)
+        self.gc_polish_cover.setToolTip(_("Calibre's Polish feature will be used to inject or update the generated cover into the ebook, EPUB only."))
+        self.gc_polish_cover.setChecked(prefs['gc_polish_cover'])
+        self.l.addWidget(self.gc_polish_cover)
+            
 class CountPagesTab(QWidget):
 
     def __init__(self, parent_dialog, plugin_action):
@@ -682,39 +796,41 @@ class CountPagesTab(QWidget):
         self.l = QVBoxLayout()
         self.setLayout(self.l)
 
-        label = QLabel('These settings provide integration with the Count Pages Plugin.  Count Pages can automatically update custom columns with page, word and reading level statistics.  You have to create and configure the columns in Count Pages first.')
+        label = QLabel(_('These settings provide integration with the %(cp)s Plugin.  %(cp)s can automatically update custom columns with page, word and reading level statistics.  You have to create and configure the columns in %(cp)s first.')%no_trans)
         label.setWordWrap(True)
         self.l.addWidget(label)
         self.l.addSpacing(5)
 
-        label = QLabel('If any of the settings below are checked, when stories are added or updated, the Count Pages Plugin will be called to update the checked statistics.')
+        label = QLabel(_('If any of the settings below are checked, when stories are added or updated, the %(cp)s Plugin will be called to update the checked statistics.')%no_trans)
         label.setWordWrap(True)
         self.l.addWidget(label)
         self.l.addSpacing(5)
-        
+
+        # the same for all settings.  Mostly.
+        tooltip = _('Which column and algorithm to use are configured in %(cp)s.')%no_trans
         # 'PageCount', 'WordCount', 'FleschReading', 'FleschGrade', 'GunningFog'
         self.pagecount = QCheckBox('Page Count',self)
-        self.pagecount.setToolTip('Which column and algorithm to use are configured in Count Pages.')
+        self.pagecount.setToolTip(tooltip)
         self.pagecount.setChecked('PageCount' in prefs['countpagesstats'])
         self.l.addWidget(self.pagecount)
             
         self.wordcount = QCheckBox('Word Count',self)
-        self.wordcount.setToolTip('Which column and algorithm to use are configured in Count Words.\nWill overwrite word count from FFDL metadata if set to update the same custom column.')
+        self.wordcount.setToolTip(tooltip+"\n"+_('Will overwrite word count from FFDL metadata if set to update the same custom column.'))
         self.wordcount.setChecked('WordCount' in prefs['countpagesstats'])
         self.l.addWidget(self.wordcount)
 
         self.fleschreading = QCheckBox('Flesch Reading Ease',self)
-        self.fleschreading.setToolTip('Which column and algorithm to use are configured in Count Pages.')
+        self.fleschreading.setToolTip(tooltip)
         self.fleschreading.setChecked('FleschReading' in prefs['countpagesstats'])
         self.l.addWidget(self.fleschreading)
         
         self.fleschgrade = QCheckBox('Flesch-Kincaid Grade Level',self)
-        self.fleschgrade.setToolTip('Which column and algorithm to use are configured in Count Pages.')
+        self.fleschgrade.setToolTip(tooltip)
         self.fleschgrade.setChecked('FleschGrade' in prefs['countpagesstats'])
         self.l.addWidget(self.fleschgrade)
         
         self.gunningfog = QCheckBox('Gunning Fog Index',self)
-        self.gunningfog.setToolTip('Which column and algorithm to use are configured in Count Pages.')
+        self.gunningfog.setToolTip(tooltip)
         self.gunningfog.setChecked('GunningFog' in prefs['countpagesstats'])
         self.l.addWidget(self.gunningfog)
         
@@ -730,26 +846,23 @@ class OtherTab(QWidget):
         self.l = QVBoxLayout()
         self.setLayout(self.l)
 
-        label = QLabel("These controls aren't plugin settings as such, but convenience buttons for setting Keyboard shortcuts and getting all the FanFictionDownLoader confirmation dialogs back again.")
+        label = QLabel(_("These controls aren't plugin settings as such, but convenience buttons for setting Keyboard shortcuts and getting all the FanFictionDownLoader confirmation dialogs back again."))
         label.setWordWrap(True)
         self.l.addWidget(label)
         self.l.addSpacing(5)
         
-        keyboard_shortcuts_button = QPushButton('Keyboard shortcuts...', self)
-        keyboard_shortcuts_button.setToolTip(_(
-                    'Edit the keyboard shortcuts associated with this plugin'))
+        keyboard_shortcuts_button = QPushButton(_('Keyboard shortcuts...'), self)
+        keyboard_shortcuts_button.setToolTip(_('Edit the keyboard shortcuts associated with this plugin'))
         keyboard_shortcuts_button.clicked.connect(parent_dialog.edit_shortcuts)
         self.l.addWidget(keyboard_shortcuts_button)
 
         reset_confirmation_button = QPushButton(_('Reset disabled &confirmation dialogs'), self)
-        reset_confirmation_button.setToolTip(_(
-                    'Reset all show me again dialogs for the FanFictionDownLoader plugin'))
+        reset_confirmation_button.setToolTip(_('Reset all show me again dialogs for the FanFictionDownLoader plugin'))
         reset_confirmation_button.clicked.connect(self.reset_dialogs)
         self.l.addWidget(reset_confirmation_button)
         
-        view_prefs_button = QPushButton('&View library preferences...', self)
-        view_prefs_button.setToolTip(_(
-                    'View data stored in the library database for this plugin'))
+        view_prefs_button = QPushButton(_('&View library preferences...'), self)
+        view_prefs_button.setToolTip(_('View data stored in the library database for this plugin'))
         view_prefs_button.clicked.connect(self.view_prefs)
         self.l.addWidget(view_prefs_button)
         
@@ -809,35 +922,35 @@ permitted_values['text'] = permitted_values['enumeration']
 permitted_values['comments'] = permitted_values['enumeration']
 
 titleLabels = {
-    'category':'Category',
-    'genre':'Genre',
-    'language':'Language',
-    'status':'Status',
-    'status-C':'Status:Completed',
-    'status-I':'Status:In-Progress',
-    'series':'Series',
-    'characters':'Characters',
-    'ships':'Relationships',
-    'datePublished':'Published',
-    'dateUpdated':'Updated',
-    'dateCreated':'Created',
-    'rating':'Rating',
-    'warnings':'Warnings',
-    'numChapters':'Chapters',
-    'numWords':'Words',
-    'site':'Site',
-    'storyId':'Story ID',
-    'authorId':'Author ID',
-    'extratags':'Extra Tags',
-    'title':'Title',
-    'storyUrl':'Story URL',
-    'description':'Description',
-    'author':'Author',
-    'authorUrl':'Author URL',
-    'formatname':'File Format',
-    'formatext':'File Extension',
-    'siteabbrev':'Site Abbrev',
-    'version':'FFDL Version'
+    'category':_('Category'),
+    'genre':_('Genre'),
+    'language':_('Language'),
+    'status':_('Status'),
+    'status-C':_('Status:%(cmplt)s')%no_trans,
+    'status-I':_('Status:%(inprog)s')%no_trans,
+    'series':_('Series'),
+    'characters':_('Characters'),
+    'ships':_('Relationships'),
+    'datePublished':_('Published'),
+    'dateUpdated':_('Updated'),
+    'dateCreated':_('Created'),
+    'rating':_('Rating'),
+    'warnings':_('Warnings'),
+    'numChapters':_('Chapters'),
+    'numWords':_('Words'),
+    'site':_('Site'),
+    'storyId':_('Story ID'),
+    'authorId':_('Author ID'),
+    'extratags':_('Extra Tags'),
+    'title':_('Title'),
+    'storyUrl':_('Story URL'),
+    'description':_('Description'),
+    'author':_('Author'),
+    'authorUrl':_('Author URL'),
+    'formatname':_('File Format'),
+    'formatext':_('File Extension'),
+    'siteabbrev':_('Site Abbrev'),
+    'version':_('FFDL Version')
     }
 
 class CustomColumnsTab(QWidget):
@@ -852,7 +965,7 @@ class CustomColumnsTab(QWidget):
         self.l = QVBoxLayout()
         self.setLayout(self.l)
 
-        label = QLabel("If you have custom columns defined, they will be listed below.  Choose a metadata value type to fill your columns automatically.")
+        label = QLabel(_("If you have custom columns defined, they will be listed below.  Choose a metadata value type to fill your columns automatically."))
         label.setWordWrap(True)
         self.l.addWidget(label)
         self.l.addSpacing(5)
@@ -877,23 +990,23 @@ class CustomColumnsTab(QWidget):
                 #     print("column['%s'] => %s"%(k,v))
                 horz = QHBoxLayout()
                 label = QLabel(column['name'])
-                label.setToolTip("Update this %s column(%s) with..."%(key,column['datatype']))
+                label.setToolTip(_("Update this %s column(%s) with...")%(key,column['datatype']))
                 horz.addWidget(label)
                 dropdown = QComboBox(self)
-                dropdown.addItem('',QVariant('none'))
+                dropdown.addItem('','none')
                 for md in permitted_values[column['datatype']]:
-                    dropdown.addItem(titleLabels[md],QVariant(md))
+                    dropdown.addItem(titleLabels[md],md)
                 self.custcol_dropdowns[key] = dropdown
                 if key in prefs['custom_cols']:
-                    dropdown.setCurrentIndex(dropdown.findData(QVariant(prefs['custom_cols'][key])))
+                    dropdown.setCurrentIndex(dropdown.findData(prefs['custom_cols'][key]))
                 if column['datatype'] == 'enumeration':
-                    dropdown.setToolTip("Metadata values valid for this type of column.\nValues that aren't valid for this enumeration column will be ignored.")
+                    dropdown.setToolTip(_("Metadata values valid for this type of column.")+"\n"+_("Values that aren't valid for this enumeration column will be ignored."))
                 else:
-                    dropdown.setToolTip("Metadata values valid for this type of column.")
+                    dropdown.setToolTip(_("Metadata values valid for this type of column."))
                 horz.addWidget(dropdown)
 
-                newonlycheck = QCheckBox("New Only",self)
-                newonlycheck.setToolTip("Write to %s(%s) only for new\nbooks, not updates to existing books."%(column['name'],key))
+                newonlycheck = QCheckBox(_("New Only"),self)
+                newonlycheck.setToolTip(_("Write to %s(%s) only for new\nbooks, not updates to existing books.")%(column['name'],key))
                 self.custcol_newonlycheck[key] = newonlycheck
                 if key in prefs['custom_cols_newonly']:
                     newonlycheck.setChecked(prefs['custom_cols_newonly'][key])
@@ -904,28 +1017,28 @@ class CustomColumnsTab(QWidget):
         self.sl.insertStretch(-1)
 
         self.l.addSpacing(5)
-        self.allow_custcol_from_ini = QCheckBox('Allow custom_columns_settings from personal.ini to override',self)
-        self.allow_custcol_from_ini.setToolTip("The personal.ini parameter custom_columns_settings allows you to set custom columns to site specific values that aren't common to all sites.<br \>custom_columns_settings is ignored when this is off.")
+        self.allow_custcol_from_ini = QCheckBox(_('Allow %(ccset)s from %(pini)s to override')%no_trans,self)
+        self.allow_custcol_from_ini.setToolTip(_("The %(pini)s parameter %(ccset)s allows you to set custom columns to site specific values that aren't common to all sites.<br />%(ccset)s is ignored when this is off.")%no_trans)
         self.allow_custcol_from_ini.setChecked(prefs['allow_custcol_from_ini'])
         self.l.addWidget(self.allow_custcol_from_ini)
         
         self.l.addSpacing(5)
-        label = QLabel("Special column:")
+        label = QLabel(_("Special column:"))
         label.setWordWrap(True)
         self.l.addWidget(label)
         
         horz = QHBoxLayout()
-        label = QLabel("Update/Overwrite Error Column:")
-        tooltip="When an update or overwrite of an existing story fails, record the reason in this column.\n(Text and Long Text columns only.)"
+        label = QLabel(_("Update/Overwrite Error Column:"))
+        tooltip=_("When an update or overwrite of an existing story fails, record the reason in this column.\n(Text and Long Text columns only.)")
         label.setToolTip(tooltip)
         horz.addWidget(label)
         self.errorcol = QComboBox(self)
         self.errorcol.setToolTip(tooltip)
-        self.errorcol.addItem('',QVariant('none'))
+        self.errorcol.addItem('','none')
         for key, column in custom_columns.iteritems():
             if column['datatype'] in ('text','comments'):
-                self.errorcol.addItem(column['name'],QVariant(key))
-        self.errorcol.setCurrentIndex(self.errorcol.findData(QVariant(prefs['errorcol'])))
+                self.errorcol.addItem(column['name'],key)
+        self.errorcol.setCurrentIndex(self.errorcol.findData(prefs['errorcol']))
         horz.addWidget(self.errorcol)
         self.l.addLayout(horz)
 
@@ -941,21 +1054,21 @@ class StandardColumnsTab(QWidget):
 
         columns=OrderedDict()
         
-        columns["title"]="Title"
-        columns["authors"]="Author(s)"
-        columns["publisher"]="Publisher"
-        columns["tags"]="Tags"
-        columns["languages"]="Languages"
-        columns["pubdate"]="Published Date"
-        columns["timestamp"]="Date"
-        columns["comments"]="Comments"
-        columns["series"]="Series"
-        columns["identifiers"]="Ids(url id only)"
+        columns["title"]=_("Title")
+        columns["authors"]=_("Author(s)")
+        columns["publisher"]=_("Publisher")
+        columns["tags"]=_("Tags")
+        columns["languages"]=_("Languages")
+        columns["pubdate"]=_("Published Date")
+        columns["timestamp"]=_("Date")
+        columns["comments"]=_("Comments")
+        columns["series"]=_("Series")
+        columns["identifiers"]=_("Ids(url id only)")
 
         self.l = QVBoxLayout()
         self.setLayout(self.l)
 
-        label = QLabel("The standard calibre metadata columns are listed below.  You may choose whether FFDL will fill each column automatically on updates or only for new books.")
+        label = QLabel(_("The standard calibre metadata columns are listed below.  You may choose whether FFDL will fill each column automatically on updates or only for new books."))
         label.setWordWrap(True)
         self.l.addWidget(label)
         self.l.addSpacing(5)
@@ -968,8 +1081,8 @@ class StandardColumnsTab(QWidget):
             #label.setToolTip("Update this %s column(%s) with..."%(key,column['datatype']))
             horz.addWidget(label)
 
-            newonlycheck = QCheckBox("New Only",self)
-            newonlycheck.setToolTip("Write to %s only for new\nbooks, not updates to existing books."%column)
+            newonlycheck = QCheckBox(_("New Only"),self)
+            newonlycheck.setToolTip(_("Write to %s only for new\nbooks, not updates to existing books.")%column)
             self.stdcol_newonlycheck[key] = newonlycheck
             if key in prefs['std_cols_newonly']:
                 newonlycheck.setChecked(prefs['std_cols_newonly'][key])

calibre-plugin/dialogs.py

@@ -7,22 +7,58 @@ __license__   = 'GPL v3'
 __copyright__ = '2011, Jim Miller'
 __docformat__ = 'restructuredtext en'
 
+import logging
+logger = logging.getLogger(__name__)
+
 import traceback, re
 from functools import partial
 
+import logging
+logger = logging.getLogger(__name__)
+
 import urllib
 import email
 
-from PyQt4 import QtGui
-from PyQt4.Qt import (QDialog, QTableWidget, QVBoxLayout, QHBoxLayout, QGridLayout,
-                      QPushButton, QString, QLabel, QCheckBox, QIcon, QLineEdit,
-                      QComboBox, QVariant, QProgressDialog, QTimer, QDialogButtonBox,
-                      QPixmap, Qt, QAbstractItemView, SIGNAL, QTextEdit, pyqtSignal,
-                      QGroupBox, QFrame)
+try:
+    from PyQt5 import QtWidgets as QtGui
+    from PyQt5.Qt import (QDialog, QTableWidget, QVBoxLayout, QHBoxLayout, QGridLayout,
+                          QPushButton, QLabel, QCheckBox, QIcon, QLineEdit,
+                          QComboBox, QProgressDialog, QTimer, QDialogButtonBox,
+                          QPixmap, Qt, QAbstractItemView, QTextEdit, pyqtSignal,
+                          QGroupBox, QFrame)
+except ImportError as e:
+    from PyQt4 import QtGui
+    from PyQt4.Qt import (QDialog, QTableWidget, QVBoxLayout, QHBoxLayout, QGridLayout,
+                          QPushButton, QLabel, QCheckBox, QIcon, QLineEdit,
+                          QComboBox, QProgressDialog, QTimer, QDialogButtonBox,
+                          QPixmap, Qt, QAbstractItemView, QTextEdit, pyqtSignal,
+                          QGroupBox, QFrame)
+
+try:
+    from calibre.gui2 import QVariant
+    del QVariant
+except ImportError:
+    is_qt4 = False
+    convert_qvariant = lambda x: x
+else:
+    is_qt4 = True
+    def convert_qvariant(x):
+        vt = x.type()
+        if vt == x.String:
+            return unicode(x.toString())
+        if vt == x.List:
+            return [convert_qvariant(i) for i in x.toList()]
+        return x.toPyObject()
 
 from calibre.gui2.dialogs.confirm_delete import confirm
 from calibre.gui2.complete2 import EditWithComplete
 
+# pulls in translation files for _() strings
+try:
+    load_translations()
+except NameError:
+    pass # load_translations() added in calibre 1.9
+
 from calibre_plugins.fanfictiondownloader_plugin.common_utils \
     import (ReadOnlyTableWidgetItem, ReadOnlyTextIconWidgetItem, SizePersistedDialog,
             ImageTitleLayout, get_icon)
@@ -30,13 +66,13 @@ from calibre_plugins.fanfictiondownloader_plugin.common_utils \
 from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.geturls import get_urls_from_html, get_urls_from_text
 from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.adapters import getNormalStoryURL
 
-SKIP=u'Skip'
-ADDNEW=u'Add New Book'
-UPDATE=u'Update EPUB if New Chapters'
-UPDATEALWAYS=u'Update EPUB Always'
-OVERWRITE=u'Overwrite if Newer'
-OVERWRITEALWAYS=u'Overwrite Always'
-CALIBREONLY=u'Update Calibre Metadata Only'
+SKIP=_('Skip')
+ADDNEW=_('Add New Book')
+UPDATE=_('Update EPUB if New Chapters')
+UPDATEALWAYS=_('Update EPUB Always')
+OVERWRITE=_('Overwrite if Newer')
+OVERWRITEALWAYS=_('Overwrite Always')
+CALIBREONLY=_('Update Calibre Metadata Only')
 collision_order=[SKIP,
                  ADDNEW,
                  UPDATE,
@@ -45,6 +81,32 @@ collision_order=[SKIP,
                  OVERWRITEALWAYS,
                  CALIBREONLY,]
 
+# best idea I've had for how to deal with config/pref saving the
+# collision name in english.
+SAVE_SKIP='Skip'
+SAVE_ADDNEW='Add New Book'
+SAVE_UPDATE='Update EPUB if New Chapters'
+SAVE_UPDATEALWAYS='Update EPUB Always'
+SAVE_OVERWRITE='Overwrite if Newer'
+SAVE_OVERWRITEALWAYS='Overwrite Always'
+SAVE_CALIBREONLY='Update Calibre Metadata Only'
+save_collisions={
+    SKIP:SAVE_SKIP,
+    ADDNEW:SAVE_ADDNEW,
+    UPDATE:SAVE_UPDATE,
+    UPDATEALWAYS:SAVE_UPDATEALWAYS,
+    OVERWRITE:SAVE_OVERWRITE,
+    OVERWRITEALWAYS:SAVE_OVERWRITEALWAYS,
+    CALIBREONLY:SAVE_CALIBREONLY,
+    SAVE_SKIP:SKIP,
+    SAVE_ADDNEW:ADDNEW,
+    SAVE_UPDATE:UPDATE,
+    SAVE_UPDATEALWAYS:UPDATEALWAYS,
+    SAVE_OVERWRITE:OVERWRITE,
+    SAVE_OVERWRITEALWAYS:OVERWRITEALWAYS,
+    SAVE_CALIBREONLY:CALIBREONLY,
+    }
+    
 anthology_collision_order=[UPDATE,
                            UPDATEALWAYS,
                            OVERWRITEALWAYS]
@@ -98,6 +160,7 @@ class RejectUrlEntry:
     def fullnote(self):
         retval = ""
         if self.title and self.auth:
+            # don't translate--ends up being saved and confuses regex above.
             retval = retval + "%s by %s"%(self.title,self.auth)
             if self.note:
                 retval = retval + " - "
@@ -107,19 +170,6 @@ class RejectUrlEntry:
             
         return retval
 
-# This is a more than slightly kludgey way to get
-# EditWithComplete to *not* alpha-order the reasons, but leave
-# them in the order entered.  If
-# calibre.gui2.complete2.CompleteModel.set_items ever changes,
-# this function will need to also.
-def complete_model_set_items_kludge(self, items):
-    items = [unicode(x.strip()) for x in items]
-    items = [x for x in items if x]
-    items = tuple(items)
-    self.all_items = self.current_items = items
-    self.current_prefix = ''
-    self.reset()
-    
 class NotGoingToDownload(Exception):
     def __init__(self,error,icon='dialog_error.png'):
         self.error=error
@@ -157,14 +207,14 @@ class DroppableQTextEdit(QTextEdit):
                             urllist.extend(get_urls_from_text(part.get_payload(decode=True)))
                 else:
                     urllist.extend(get_urls_from_text("%s"%msg))
-        
         if urllist:
             self.append("\n".join(urllist))
+            return None
         return QTextEdit.dropEvent(self,event)
         
     def canInsertFromMimeData(self, source):
         if source.hasUrls():
-            return True;
+            return True
         else:
             return QTextEdit.canInsertFromMimeData(self,source)
 
@@ -186,7 +236,7 @@ class AddNewDialog(SizePersistedDialog):
         self.l = QVBoxLayout()
         self.setLayout(self.l)
 
-        self.setWindowTitle('FanFictionDownLoader')
+        self.setWindowTitle(_('FanFictionDownLoader'))
         self.setWindowIcon(icon)
 
         self.toplabel=QLabel("Toplabel")
@@ -203,12 +253,12 @@ class AddNewDialog(SizePersistedDialog):
         # elements to show again when doing *update* merge
         self.mergeupdateshow = []
 
-        self.groupbox = QGroupBox("Show Download Options")
+        self.groupbox = QGroupBox(_("Show Download Options"))
         self.groupbox.setCheckable(True)
         self.groupbox.setChecked(False)
         self.groupbox.setFlat(True)
-        print("style:%s"%self.groupbox.styleSheet())
-        self.groupbox.setStyleSheet(gpstyle);
+        #print("style:%s"%self.groupbox.styleSheet())
+        self.groupbox.setStyleSheet(gpstyle)
 
         self.gbf = QFrame()
         self.gbl = QVBoxLayout()
@@ -222,7 +272,7 @@ class AddNewDialog(SizePersistedDialog):
         self.groupbox.toggled.connect(self.gbf.setVisible)
         
         horz = QHBoxLayout()
-        label = QLabel('Output &Format:')
+        label = QLabel(_('Output &Format:'))
         self.mergehide.append(label)
         
         self.fileform = QComboBox(self)
@@ -230,7 +280,7 @@ class AddNewDialog(SizePersistedDialog):
         self.fileform.addItem('mobi')
         self.fileform.addItem('html')
         self.fileform.addItem('txt')
-        self.fileform.setToolTip('Choose output format to create.  May set default from plugin configuration.')
+        self.fileform.setToolTip(_('Choose output format to create.  May set default from plugin configuration.'))
         self.fileform.activated.connect(self.set_collisions)
         
         horz.addWidget(label)
@@ -246,7 +296,7 @@ class AddNewDialog(SizePersistedDialog):
         self.collision.setToolTip("CollisionToolTip")
         # add collision options
         self.set_collisions()
-        i = self.collision.findText(prefs['collision'])
+        i = self.collision.findText(save_collisions[prefs['collision']])
         if i > -1:
             self.collision.setCurrentIndex(i)
         self.collisionlabel.setBuddy(self.collision)
@@ -258,15 +308,15 @@ class AddNewDialog(SizePersistedDialog):
         self.mergeupdateshow.append(self.collision)
 
         horz = QHBoxLayout()
-        self.updatemeta = QCheckBox('Update Calibre &Metadata?',self)
-        self.updatemeta.setToolTip("Update metadata for existing stories in Calibre from web site?\n(Columns set to 'New Only' in the column tabs will only be set for new books.)")
+        self.updatemeta = QCheckBox(_('Update Calibre &Metadata?'),self)
+        self.updatemeta.setToolTip(_("Update metadata for existing stories in Calibre from web site?\n(Columns set to 'New Only' in the column tabs will only be set for new books.)"))
         self.updatemeta.setChecked(prefs['updatemeta'])
         horz.addWidget(self.updatemeta)
         self.mergehide.append(self.updatemeta)
         self.mergeupdateshow.append(self.updatemeta)
 
-        self.updateepubcover = QCheckBox('Update EPUB Cover?',self)
-        self.updateepubcover.setToolTip('Update book cover image from site or defaults (if found) <i>inside</i> the EPUB when EPUB is updated.')
+        self.updateepubcover = QCheckBox(_('Update EPUB Cover?'),self)
+        self.updateepubcover.setToolTip(_('Update book cover image from site or defaults (if found) <i>inside</i> the EPUB when EPUB is updated.'))
         self.updateepubcover.setChecked(prefs['updateepubcover'])
         horz.addWidget(self.updateepubcover)
         self.mergehide.append(self.updateepubcover)
@@ -313,10 +363,10 @@ class AddNewDialog(SizePersistedDialog):
         self.groupbox.setVisible(not(self.merge and self.newmerge))
         
         if self.merge:
-            self.toplabel.setText('Story URL(s) for anthology, one per line:')
-            self.url.setToolTip('URLs for stories to include in the anthology, one per line.\nWill take URLs from clipboard, but only valid URLs.')
-            self.collisionlabel.setText('If Story Already Exists in Anthology?')
-            self.collision.setToolTip("What to do if there's already an existing story with the same URL in the anthology.")
+            self.toplabel.setText(_('Story URL(s) for anthology, one per line:'))
+            self.url.setToolTip(_('URLs for stories to include in the anthology, one per line.\nWill take URLs from clipboard, but only valid URLs.'))
+            self.collisionlabel.setText(_('If Story Already Exists in Anthology?'))
+            self.collision.setToolTip(_("What to do if there's already an existing story with the same URL in the anthology."))
             for widget in self.mergehide:
                 widget.setVisible(False)
             if not self.newmerge:
@@ -325,10 +375,10 @@ class AddNewDialog(SizePersistedDialog):
         else:
             for widget in self.mergehide:
                 widget.setVisible(True)
-            self.toplabel.setText('Story URL(s), one per line:')
-            self.url.setToolTip('URLs for stories, one per line.\nWill take URLs from clipboard, but only valid URLs.\nAdd [1,5] after the URL to limit the download to chapters 1-5.')
-            self.collisionlabel.setText('If Story Already Exists?')
-            self.collision.setToolTip("What to do if there's already an existing story with the same URL or title and author.")
+            self.toplabel.setText(_('Story URL(s), one per line:'))
+            self.url.setToolTip(_('URLs for stories, one per line.\nWill take URLs from clipboard, but only valid URLs.\nAdd [1,5] after the URL to limit the download to chapters 1-5.'))
+            self.collisionlabel.setText(_('If Story Already Exists?'))
+            self.collision.setToolTip(_("What to do if there's already an existing story with the same URL or title and author."))
 
         # Need to re-able after hiding/showing
         self.setAcceptDrops(True)
@@ -344,10 +394,10 @@ class AddNewDialog(SizePersistedDialog):
 
         # add collision options
         self.set_collisions()
-        
-        i = self.collision.findText(self.prefs['collision'])
+        i = self.collision.findText(save_collisions[self.prefs['collision']])
         if i > -1:
             self.collision.setCurrentIndex(i)
+            
         self.updatemeta.setChecked(self.prefs['updatemeta'])
             
         if not self.merge:
@@ -383,6 +433,7 @@ class AddNewDialog(SizePersistedDialog):
             'collision': unicode(self.collision.currentText()),
             'updatemeta': self.updatemeta.isChecked(),
             'updateepubcover': self.updateepubcover.isChecked(),
+            'smarten_punctuation':self.prefs['smarten_punctuation']
                 }
         
         if self.merge:
@@ -427,18 +478,18 @@ class CollectURLDialog(SizePersistedDialog):
         self.url.setText(url_text)
         self.l.addWidget(self.url,1,1,1,2)
    
-        self.indiv_button = QPushButton('For Individual Books', self)
-        self.indiv_button.setToolTip('Get URLs and go to dialog for individual story downloads.')
+        self.indiv_button = QPushButton(_('For Individual Books'), self)
+        self.indiv_button.setToolTip(_('Get URLs and go to dialog for individual story downloads.'))
         self.indiv_button.clicked.connect(self.indiv)
         self.l.addWidget(self.indiv_button,2,0)
 
-        self.merge_button = QPushButton('For Anthology Epub', self)
-        self.merge_button.setToolTip('Get URLs and go to dialog for Anthology download.\nRequires EpubMerge 1.3.1+ plugin.')
+        self.merge_button = QPushButton(_('For Anthology Epub'), self)
+        self.merge_button.setToolTip(_('Get URLs and go to dialog for Anthology download.\nRequires %s plugin.')%'EpubMerge 1.3.1+')
         self.merge_button.clicked.connect(self.merge)
         self.l.addWidget(self.merge_button,2,1)
         self.merge_button.setEnabled(epubmerge_plugin!=None)
 
-        self.cancel_button = QPushButton('Cancel', self)
+        self.cancel_button = QPushButton(_('Cancel'), self)
         self.cancel_button.clicked.connect(self.cancel)
         self.l.addWidget(self.cancel_button,2,2)
 
@@ -470,29 +521,29 @@ class UserPassDialog(QDialog):
         self.setLayout(self.l)
 
         if exception.passwdonly:
-            self.setWindowTitle('Password')
-            self.l.addWidget(QLabel("Author requires a password for this story(%s)."%exception.url),0,0,1,2)
+            self.setWindowTitle(_('Password'))
+            self.l.addWidget(QLabel(_("Author requires a password for this story(%s).")%exception.url),0,0,1,2)
             # user isn't used, but it's easier to still have it for
             # post processing.
             self.user = FakeLineEdit()
         else:
-            self.setWindowTitle('User/Password')
-            self.l.addWidget(QLabel("%s requires you to login to download this story."%site),0,0,1,2)
+            self.setWindowTitle(_('User/Password'))
+            self.l.addWidget(QLabel(_("%s requires you to login to download this story.")%site),0,0,1,2)
         
-            self.l.addWidget(QLabel("User:"),1,0)
+            self.l.addWidget(QLabel(_("User:")),1,0)
             self.user = QLineEdit(self)
             self.l.addWidget(self.user,1,1)
    
-        self.l.addWidget(QLabel("Password:"),2,0)
+        self.l.addWidget(QLabel(_("Password:")),2,0)
         self.passwd = QLineEdit(self)
         self.passwd.setEchoMode(QLineEdit.Password)
         self.l.addWidget(self.passwd,2,1)
    
-        self.ok_button = QPushButton('OK', self)
+        self.ok_button = QPushButton(_('OK'), self)
         self.ok_button.clicked.connect(self.ok)
         self.l.addWidget(self.ok_button,3,0)
 
-        self.cancel_button = QPushButton('Cancel', self)
+        self.cancel_button = QPushButton(_('Cancel'), self)
         self.cancel_button.clicked.connect(self.cancel)
         self.l.addWidget(self.cancel_button,3,1)
 
@@ -514,12 +565,12 @@ class LoopProgressDialog(QProgressDialog):
                  book_list,
                  foreach_function,
                  finish_function,
-                 init_label="Fetching metadata for stories...",
-                 win_title="Downloading metadata for stories",
-                 status_prefix="Fetched metadata for"):
+                 init_label=_("Fetching metadata for stories..."),
+                 win_title=_("Downloading metadata for stories"),
+                 status_prefix=_("Fetched metadata for")):
         QProgressDialog.__init__(self,
                                  init_label,
-                                 QString(), 0, len(book_list), gui)
+                                 _('Cancel'), 0, len(book_list), gui)
         self.setWindowTitle(win_title)
         self.setMinimumWidth(500)
         self.book_list = book_list
@@ -534,9 +585,9 @@ class LoopProgressDialog(QProgressDialog):
         self.exec_()
 
     def updateStatus(self):
-        self.setLabelText("%s %d of %d"%(self.status_prefix,self.i+1,len(self.book_list)))
+        self.setLabelText("%s %d / %d"%(self.status_prefix,self.i+1,len(self.book_list)))
         self.setValue(self.i+1)
-        print(self.labelText())
+        #print(self.labelText())
 
     def do_loop(self):
 
@@ -557,7 +608,7 @@ class LoopProgressDialog(QProgressDialog):
         except Exception as e:
             book['good']=False
             book['comment']=unicode(e)
-            print("Exception: %s:%s"%(book,unicode(e)))
+            logger.error("Exception: %s:%s"%(book,unicode(e)))
             traceback.print_exc()
             
         self.updateStatus()
@@ -618,7 +669,8 @@ class UpdateExistingDialog(SizePersistedDialog):
     def __init__(self, gui, header, prefs, icon, books,
                  save_size_name='fanfictiondownloader_plugin:update list dialog'):
         SizePersistedDialog.__init__(self, gui, save_size_name)
-        
+
+        self.prefs = prefs
         self.setWindowTitle(header)
         self.setWindowIcon(icon)
         
@@ -639,7 +691,7 @@ class UpdateExistingDialog(SizePersistedDialog):
         spacerItem = QtGui.QSpacerItem(20, 40, QtGui.QSizePolicy.Minimum, QtGui.QSizePolicy.Expanding)
         button_layout.addItem(spacerItem)
         self.remove_button = QtGui.QToolButton(self)
-        self.remove_button.setToolTip('Remove selected books from the list')
+        self.remove_button.setToolTip(_('Remove selected books from the list'))
         self.remove_button.setIcon(get_icon('list_remove.png'))
         self.remove_button.clicked.connect(self.remove_from_list)
         button_layout.addWidget(self.remove_button)
@@ -648,7 +700,7 @@ class UpdateExistingDialog(SizePersistedDialog):
 
         options_layout = QHBoxLayout()
 
-        groupbox = QGroupBox("Show Download Options")
+        groupbox = QGroupBox(_("Show Download Options"))
         groupbox.setCheckable(True)
         groupbox.setChecked(False)
         groupbox.setFlat(True)
@@ -665,7 +717,7 @@ class UpdateExistingDialog(SizePersistedDialog):
         gbf.setVisible(False)
         groupbox.toggled.connect(gbf.setVisible)
         
-        label = QLabel('Output &Format:')
+        label = QLabel(_('Output &Format:'))
         gbl.addWidget(label)
         self.fileform = QComboBox(self)
         self.fileform.addItem('epub')
@@ -673,30 +725,30 @@ class UpdateExistingDialog(SizePersistedDialog):
         self.fileform.addItem('html')
         self.fileform.addItem('txt')
         self.fileform.setCurrentIndex(self.fileform.findText(prefs['fileform']))
-        self.fileform.setToolTip('Choose output format to create.  May set default from plugin configuration.')
+        self.fileform.setToolTip(_('Choose output format to create.  May set default from plugin configuration.'))
         self.fileform.activated.connect(self.set_collisions)
         label.setBuddy(self.fileform)
         gbl.addWidget(self.fileform)
         
-        label = QLabel('Update Mode:')
+        label = QLabel(_('Update Mode:'))
         gbl.addWidget(label)
         self.collision = QComboBox(self)
-        self.collision.setToolTip("What sort of update to perform.  May set default from plugin configuration.")
+        self.collision.setToolTip(_("What sort of update to perform.  May set default from plugin configuration."))
         # add collision options
         self.set_collisions()
-        i = self.collision.findText(prefs['collision'])
+        i = self.collision.findText(save_collisions[prefs['collision']])
         if i > -1:
             self.collision.setCurrentIndex(i)
         label.setBuddy(self.collision)
         gbl.addWidget(self.collision)
 
-        self.updatemeta = QCheckBox('Update Calibre &Metadata?',self)
-        self.updatemeta.setToolTip("Update metadata for existing stories in Calibre from web site?\n(Columns set to 'New Only' in the column tabs will only be set for new books.)")
+        self.updatemeta = QCheckBox(_('Update Calibre &Metadata?'),self)
+        self.updatemeta.setToolTip(_("Update metadata for existing stories in Calibre from web site?\n(Columns set to 'New Only' in the column tabs will only be set for new books.)"))
         self.updatemeta.setChecked(prefs['updatemeta'])
         gbl.addWidget(self.updatemeta)
                 
-        self.updateepubcover = QCheckBox('Update EPUB Cover?',self)
-        self.updateepubcover.setToolTip('Update book cover image from site or defaults (if found) <i>inside</i> the EPUB when EPUB is updated.')
+        self.updateepubcover = QCheckBox(_('Update EPUB Cover?'),self)
+        self.updateepubcover.setToolTip(_('Update book cover image from site or defaults (if found) <i>inside</i> the EPUB when EPUB is updated.'))
         self.updateepubcover.setChecked(prefs['updateepubcover'])
         gbl.addWidget(self.updateepubcover)
 
@@ -736,6 +788,7 @@ class UpdateExistingDialog(SizePersistedDialog):
             'collision': unicode(self.collision.currentText()),
             'updatemeta': self.updatemeta.isChecked(),
             'updateepubcover': self.updateepubcover.isChecked(),
+            'smarten_punctuation':self.prefs['smarten_punctuation']
             }
 
 class StoryListTableWidget(QTableWidget):
@@ -748,7 +801,7 @@ class StoryListTableWidget(QTableWidget):
         self.clear()
         self.setAlternatingRowColors(True)
         self.setRowCount(len(books))
-        header_labels = ['','Title', 'Author', 'URL', 'Comment']
+        header_labels = ['',_('Title'), _('Author'), 'URL', _('Comment')]
         self.setColumnCount(len(header_labels))
         self.setHorizontalHeaderLabels(header_labels)
         self.horizontalHeader().setStretchLastSection(True)
@@ -787,11 +840,11 @@ class StoryListTableWidget(QTableWidget):
             icon = get_icon(book['icon'])
 
         status_cell = IconWidgetItem(None,icon,val)
-        status_cell.setData(Qt.UserRole, QVariant(val))
+        status_cell.setData(Qt.UserRole, val)
         self.setItem(row, 0, status_cell)
         
         title_cell = ReadOnlyTableWidgetItem(book['title'])
-        title_cell.setData(Qt.UserRole, QVariant(row))
+        title_cell.setData(Qt.UserRole, row)
         self.setItem(row, 1, title_cell)
         
         self.setItem(row, 2, AuthorTableWidgetItem(", ".join(book['author']), ", ".join(book['author_sort'])))
@@ -806,7 +859,7 @@ class StoryListTableWidget(QTableWidget):
         books = []
         #print("=========================\nbooks:%s"%self.books)
         for row in range(self.rowCount()):
-            rnum = self.item(row, 1).data(Qt.UserRole).toPyObject()
+            rnum = convert_qvariant(self.item(row, 1).data(Qt.UserRole))
             book = self.books[rnum]
             books.append(book)
         return books
@@ -816,9 +869,9 @@ class StoryListTableWidget(QTableWidget):
         rows = self.selectionModel().selectedRows()
         if len(rows) == 0:
             return
-        message = '<p>Are you sure you want to remove this book from the list?'
+        message = '<p>'+_('Are you sure you want to remove this book from the list?')
         if len(rows) > 1:
-            message = '<p>Are you sure you want to remove the selected %d books from the list?'%len(rows)
+            message = '<p>'+_('Are you sure you want to remove the selected %d books from the list?')%len(rows)
         if not confirm(message,'fanfictiondownloader_delete_item', self):
             return
         first_sel_row = self.currentRow()
@@ -844,7 +897,7 @@ class RejectListTableWidget(QTableWidget):
         self.clear()
         self.setAlternatingRowColors(True)
         self.setRowCount(len(reject_list))
-        header_labels = ['URL', 'Title', 'Author', 'Note']
+        header_labels = ['URL', _('Title'), _('Author'), _('Note')]
         self.setColumnCount(len(header_labels))
         self.setHorizontalHeaderLabels(header_labels)
         self.horizontalHeader().setStretchLastSection(True)
@@ -872,21 +925,18 @@ class RejectListTableWidget(QTableWidget):
     def populate_table_row(self, row, rej):
 
         url_cell = ReadOnlyTableWidgetItem(rej.url)
-        url_cell.setData(Qt.UserRole, QVariant(rej.book_id))
+        url_cell.setData(Qt.UserRole, rej.book_id)
         self.setItem(row, 0, url_cell)
         self.setItem(row, 1, ReadOnlyTableWidgetItem(rej.title))
         self.setItem(row, 2, ReadOnlyTableWidgetItem(rej.auth))
         
-        note_cell = EditWithComplete(self)
-        note_cell.lineEdit().mcompleter.model().set_items = \
-            partial(complete_model_set_items_kludge,
-                    note_cell.lineEdit().mcompleter.model())
+        note_cell = EditWithComplete(self,sort_func=lambda x:1)
         
         items = [rej.note]+self.rejectreasons
         note_cell.update_items_cache(items)
         note_cell.show_initial_value(rej.note)
         note_cell.set_separator(None)
-        note_cell.setToolTip('Select or Edit Reject Note.')
+        note_cell.setToolTip(_('Select or Edit Reject Note.'))
         self.setCellWidget(row, 3, note_cell)
         
     def remove_selected_rows(self):
@@ -894,9 +944,9 @@ class RejectListTableWidget(QTableWidget):
         rows = self.selectionModel().selectedRows()
         if len(rows) == 0:
             return
-        message = '<p>Are you sure you want to remove this URL from the list?'
+        message = '<p>'+_('Are you sure you want to remove this URL from the list?')
         if len(rows) > 1:
-            message = '<p>Are you sure you want to remove the %d selected URLs from the list?'%len(rows)
+            message = '<p>'+_('Are you sure you want to remove the %d selected URLs from the list?')%len(rows)
         if not confirm(message,'ffdl_rejectlist_delete_item_again', self):
             return
         first_sel_row = self.currentRow()
@@ -914,7 +964,7 @@ class RejectListTableWidget(QTableWidget):
 class RejectListDialog(SizePersistedDialog):
     def __init__(self, gui, reject_list,
                  rejectreasons=[],
-                 header="List of Books to Reject",
+                 header=_("List of Books to Reject"),
                  icon='rotate-right.png',
                  show_delete=True,
                  show_all_reasons=True,
@@ -927,7 +977,7 @@ class RejectListDialog(SizePersistedDialog):
         layout = QVBoxLayout(self)
         self.setLayout(layout)
         title_layout = ImageTitleLayout(self, icon, header,
-                                        '<i></i>FFDL will remember these URLs and display the note and offer to reject them if you try to download them again later.')
+                                        '<i></i>'+_('FFDL will remember these URLs and display the note and offer to reject them if you try to download them again later.'))
         layout.addLayout(title_layout)
         rejects_layout = QHBoxLayout()
         layout.addLayout(rejects_layout)
@@ -941,7 +991,7 @@ class RejectListDialog(SizePersistedDialog):
         button_layout.addItem(spacerItem)
         
         self.remove_button = QtGui.QToolButton(self)
-        self.remove_button.setToolTip('Remove selected URL(s) from the list')
+        self.remove_button.setToolTip(_('Remove selected URL(s) from the list'))
         self.remove_button.setIcon(get_icon('list_remove.png'))
         self.remove_button.clicked.connect(self.remove_from_list)
         button_layout.addWidget(self.remove_button)
@@ -950,20 +1000,17 @@ class RejectListDialog(SizePersistedDialog):
         button_layout.addItem(spacerItem1)
 
         if show_all_reasons:
-            self.reason_edit = EditWithComplete(self)
-            self.reason_edit.lineEdit().mcompleter.model().set_items = \
-                partial(complete_model_set_items_kludge,
-                        self.reason_edit.lineEdit().mcompleter.model())
+            self.reason_edit = EditWithComplete(self,sort_func=lambda x:1)
             
             items = ['']+rejectreasons
             self.reason_edit.update_items_cache(items)
             self.reason_edit.show_initial_value('')
             self.reason_edit.set_separator(None)
-            self.reason_edit.setToolTip("This will be added to whatever note you've set for each URL above.")
+            self.reason_edit.setToolTip(_("This will be added to whatever note you've set for each URL above."))
             
             horz = QHBoxLayout()
-            label = QLabel("Add this reason to all URLs added:")
-            label.setToolTip("This will be added to whatever note you've set for each URL above.")
+            label = QLabel(_("Add this reason to all URLs added:"))
+            label.setToolTip(_("This will be added to whatever note you've set for each URL above."))
             horz.addWidget(label)
             horz.addWidget(self.reason_edit)
             horz.insertStretch(-1)
@@ -972,8 +1019,8 @@ class RejectListDialog(SizePersistedDialog):
         options_layout = QHBoxLayout()
 
         if show_delete:
-            self.deletebooks = QCheckBox('Delete Books (including books without FanFiction URLs)?',self)
-            self.deletebooks.setToolTip("Delete the selected books after adding them to the Rejected URLs list.")
+            self.deletebooks = QCheckBox(_('Delete Books (including books without FanFiction URLs)?'),self)
+            self.deletebooks.setToolTip(_("Delete the selected books after adding them to the Rejected URLs list."))
             self.deletebooks.setChecked(True)
             options_layout.addWidget(self.deletebooks)
         
@@ -995,7 +1042,7 @@ class RejectListDialog(SizePersistedDialog):
         rejectrows = []
         for row in range(self.rejects_table.rowCount()):
             url = unicode(self.rejects_table.item(row, 0).text()).strip()
-            book_id = self.rejects_table.item(row, 0).data(Qt.UserRole).toPyObject()
+            book_id =convert_qvariant(self.rejects_table.item(row, 0).data(Qt.UserRole))
             title = unicode(self.rejects_table.item(row, 1).text()).strip()
             auth = unicode(self.rejects_table.item(row, 2).text()).strip()
             note = unicode(self.rejects_table.cellWidget(row, 3).currentText()).strip()
@@ -1005,7 +1052,7 @@ class RejectListDialog(SizePersistedDialog):
     def get_reject_list_ids(self):
         rejectrows = []
         for row in range(self.rejects_table.rowCount()):
-            book_id = self.rejects_table.item(row, 0).data(Qt.UserRole).toPyObject()
+            book_id = convert_qvariant(self.rejects_table.item(row, 0).data(Qt.UserRole))
             if book_id:
                 rejectrows.append(book_id)
         return rejectrows
@@ -1047,11 +1094,7 @@ class EditTextDialog(QDialog):
             self.textedit.setToolTip(tooltip)
 
         if rejectreasons or reasonslabel:
-            self.reason_edit = EditWithComplete(self)
-
-            self.reason_edit.lineEdit().mcompleter.model().set_items = \
-                partial(complete_model_set_items_kludge,
-                        self.reason_edit.lineEdit().mcompleter.model())
+            self.reason_edit = EditWithComplete(self,sort_func=lambda x:1)
             
             items = ['']+rejectreasons
             self.reason_edit.update_items_cache(items)

calibre-plugin/jobs.py

@@ -4,16 +4,25 @@ from __future__ import (unicode_literals, division, absolute_import,
                         print_function)
 
 __license__   = 'GPL v3'
-__copyright__ = '2012, Jim Miller'
+__copyright__ = '2014, Jim Miller'
 __copyright__ = '2011, Grant Drake <grant.drake@gmail.com>'
 __docformat__ = 'restructuredtext en'
 
+import logging
+logger = logging.getLogger(__name__)
+
 import time, os, traceback
 
 from StringIO import StringIO
 
 from calibre.utils.ipc.server import Server
 from calibre.utils.ipc.job import ParallelJob
+from calibre.constants import numeric_version as calibre_version
+
+# for smarten punc
+from calibre.ebooks.oeb.polish.main import polish, ALL_OPTS
+from calibre.utils.logging import Log
+from collections import namedtuple
 
 from calibre_plugins.fanfictiondownloader_plugin.dialogs import (NotGoingToDownload,
     OVERWRITE, OVERWRITEALWAYS, UPDATE, UPDATEALWAYS, ADDNEW, SKIP, CALIBREONLY)
@@ -37,27 +46,23 @@ def do_download_worker(book_list, options,
     '''
     server = Server(pool_size=cpus)
 
-    print(options['version'])
+    logger.info(options['version'])
     total = 0
     alreadybad = []
     # Queue all the jobs
-    print("Adding jobs for URLs:")
+    logger.info("Adding jobs for URLs:")
     for book in book_list:
-        print("%s"%book['url'])
+        logger.info("%s"%book['url'])
         if book['good']:
             total += 1
             args = ['calibre_plugins.fanfictiondownloader_plugin.jobs',
                     'do_download_for_worker',
                     (book,options)]
-            job = ParallelJob('arbitrary',
+            job = ParallelJob('arbitrary_n',
                               "url:(%s) id:(%s)"%(book['url'],book['calibre_id']),
                               done=None,
                               args=args)
             job._book = book
-            # job._book_id = book_id
-            # job._title = title
-            # job._modified_date = modified_date
-            # job._existing_isbn = existing_isbn
             server.add_job(job)
         else:
             # was already bad before the subprocess ever started.
@@ -65,7 +70,7 @@ def do_download_worker(book_list, options,
     
     # This server is an arbitrary_n job, so there is a notifier available.
     # Set the % complete to a small number to avoid the 'unavailable' indicator
-    notification(0.01, 'Downloading FanFiction Stories')
+    notification(0.01, _('Downloading FanFiction Stories'))
 
     # dequeue the job results as they arrive, saving the results
     count = 0
@@ -77,28 +82,20 @@ def do_download_worker(book_list, options,
         if not job.is_finished:
             continue
         # A job really finished. Get the information.
-        output_book = job.result
-        #print("output_book:%s"%output_book)
         book_list.remove(job._book)
         book_list.append(job.result)
         book_id = job._book['calibre_id']
-        #title = job._title
         count = count + 1
-        notification(float(count)/total, 'Downloaded Story')
+        notification(float(count)/total, '%d of %d stories finished downloading'%(count,total))
         # Add this job's output to the current log
-        print('Logfile for book ID %s (%s)'%(book_id, job._book['title']))
-        print(job.details)
+        logger.info('Logfile for book ID %s (%s)'%(book_id, job._book['title']))
+        logger.info(job.details)
 
         if count >= total:
-            # All done!  Output some lists for convenience of some users.
-            print("Successfully downloaded:")
-            for book in book_list:
-                if book['good']:
-                    print("%s %s"%(book['title'],book['url']))
-            print("\nUnsuccessful:")
-            for book in book_list:
-                if not book['good']:
-                    print("%s %s"%(book['title'],book['url']))
+            logger.info("\n"+_("Successful:")+"\n%s\n"%("\n".join([book['url'] for book in 
+                                                      filter(lambda x: x['good'], book_list) ] ) ) )
+            logger.info("\n"+_("Unsuccessful:")+"\n%s\n"%("\n".join([book['url'] for book in 
+                                                        filter(lambda x: not x['good'], book_list) ] ) ) )
             break
 
     server.close()
@@ -106,13 +103,12 @@ def do_download_worker(book_list, options,
     # return the book list as the job result
     return book_list
 
-def do_download_for_worker(book,options):
+def do_download_for_worker(book,options,notification=lambda x,y:x):
     '''
-    Child job, to extract isbn from formats for this specific book,
-    when run as a worker job
+    Child job, to download story when run as a worker job
     '''
     try:
-        book['comment'] = 'Download started...'
+        book['comment'] = _('Download started...')
 
         configuration = get_ffdl_config(book['url'],
                                         options['fileform'],
@@ -121,8 +117,8 @@ def do_download_for_worker(book,options):
         if not options['updateepubcover'] and 'epub_for_update' in book and options['collision'] in (UPDATE, UPDATEALWAYS):
             configuration.set("overrides","never_make_cover","true")
 
-        # images only for epub, even if the user mistakenly turned it
-        # on else where.
+        # images only for epub, html, even if the user mistakenly
+        # turned it on else where.
         if options['fileform'] not in ("epub","html"):
             configuration.set("overrides","include_images","false")
         
@@ -132,6 +128,10 @@ def do_download_for_worker(book,options):
         adapter.password = book['password']
         adapter.setChaptersRange(book['begin'],book['end'])
         
+        adapter.load_cookiejar(options['cookiejarfile'])
+        logger.debug("cookiejar:%s"%adapter.cookiejar)
+        adapter.set_pagecache(options['pagecache'])
+        
         story = adapter.getStoryMetadataOnly()
         if 'calibre_series' in book:
             adapter.setSeries(book['calibre_series'][0],book['calibre_series'][1])
@@ -146,7 +146,7 @@ def do_download_for_worker(book,options):
 
         ## No need to download at all.  Shouldn't ever get down here.
         if options['collision'] in (CALIBREONLY):
-            print("Skipping CALIBREONLY 'update' down inside worker--this shouldn't be happening...")
+            logger.info("Skipping CALIBREONLY 'update' down inside worker--this shouldn't be happening...")
             book['comment'] = 'Metadata collected.'
             
         ## checks were done earlier, it's new or not dup or newer--just write it.
@@ -169,7 +169,7 @@ def do_download_for_worker(book,options):
                 if adapter.logfile:
                     adapter.logfile = adapter.logfile.replace("span id","span notid")
             
-            print("write to %s"%outfile)
+            logger.info("write to %s"%outfile)
             writer.writeStory(outfilename=outfile, forceOverwrite=True)
             book['comment'] = 'Download %s completed, %s chapters.'%(options['fileform'],story.getMetadata("numChapters"))
             
@@ -178,7 +178,7 @@ def do_download_for_worker(book,options):
 
             # update now handled by pre-populating the old images and
             # chapters in the adapter rather than merging epubs.
-            urlchaptercount = int(story.getMetadata('numChapters'))
+            urlchaptercount = int(story.getMetadata('numChapters').replace(',',''))
             (url,
              chaptercount,
              adapter.oldchapters,
@@ -190,22 +190,39 @@ def do_download_for_worker(book,options):
             # dup handling from ffdl_plugin needed for anthology updates.
             if options['collision'] == UPDATE:
                 if chaptercount == urlchaptercount:
-                    book['comment']="Already contains %d chapters.  Reuse as is."%chaptercount
+                    book['comment']=_("Already contains %d chapters.  Reuse as is.")%chaptercount
                     book['outfile'] = book['epub_for_update'] # for anthology merge ops.
                     return book
 
             # dup handling from ffdl_plugin needed for anthology updates.
             if chaptercount > urlchaptercount:
-                raise NotGoingToDownload("Existing epub contains %d chapters, web site only has %d. Use Overwrite to force update." % (chaptercount,urlchaptercount),'dialog_error.png')
-                            
-            print("Do update - epub(%d) vs url(%d)" % (chaptercount, urlchaptercount))
-            print("write to %s"%outfile)
+                raise NotGoingToDownload(_("Existing epub contains %d chapters, web site only has %d. Use Overwrite to force update.") % (chaptercount,urlchaptercount),'dialog_error.png')
+
+            if not (options['collision'] == UPDATEALWAYS and chaptercount == urlchaptercount) \
+                    and adapter.getConfig("do_update_hook"):
+                chaptercount = adapter.hookForUpdates(chaptercount)
+
+            logger.info("Do update - epub(%d) vs url(%d)" % (chaptercount, urlchaptercount))
+            logger.info("write to %s"%outfile)
 
             writer.writeStory(outfilename=outfile, forceOverwrite=True)
             
-            book['comment'] = 'Update %s completed, added %s chapters for %s total.'%\
+            book['comment'] = _('Update %s completed, added %s chapters for %s total.')%\
                 (options['fileform'],(urlchaptercount-chaptercount),urlchaptercount)
         
+        if options['smarten_punctuation'] and options['fileform'] == "epub" \
+                and calibre_version >= (0, 9, 39):
+            # do smarten_punctuation from calibre's polish feature
+            data = {'smarten_punctuation':True}
+            opts = ALL_OPTS.copy()
+            opts.update(data)
+            O = namedtuple('Options', ' '.join(ALL_OPTS.iterkeys()))
+            opts = O(**opts)
+            
+            log = Log(level=Log.DEBUG)
+            # report = []
+            polish({outfile:outfile}, opts, log, logger.info) # report.append
+        
     except NotGoingToDownload as d:
         book['good']=False
         book['comment']=unicode(d)
@@ -216,8 +233,8 @@ def do_download_for_worker(book,options):
         book['comment']=unicode(e)
         book['icon']='dialog_error.png'
         book['status'] = 'Error'
-        print("Exception: %s:%s"%(book,unicode(e)))
+        logger.info("Exception: %s:%s"%(book,unicode(e)))
         traceback.print_exc()
-
+        
     #time.sleep(10)
     return book

calibre-plugin/prefs.py

@@ -25,12 +25,17 @@ default_prefs['rejecturls'] = ''
 default_prefs['rejectreasons'] = '''Sucked
 Boring
 Dup from another site'''
+default_prefs['reject_always'] = False
 
 default_prefs['updatemeta'] = True
 default_prefs['updatecover'] = False
 default_prefs['updateepubcover'] = False
 default_prefs['keeptags'] = False
+default_prefs['suppressauthorsort'] = False
+default_prefs['suppresstitlesort'] = False
+default_prefs['mark'] = False
 default_prefs['showmarked'] = False
+default_prefs['autoconvert'] = False
 default_prefs['urlsfromclip'] = True
 default_prefs['updatedefault'] = True
 default_prefs['fileform'] = 'epub'
@@ -40,7 +45,9 @@ default_prefs['adddialogstaysontop'] = False
 default_prefs['includeimages'] = False
 default_prefs['lookforurlinhtml'] = False
 default_prefs['checkforseriesurlid'] = True
+default_prefs['checkforurlchange'] = True
 default_prefs['injectseries'] = False
+default_prefs['smarten_punctuation'] = False
 
 default_prefs['send_lists'] = ''
 default_prefs['read_lists'] = ''
@@ -51,6 +58,7 @@ default_prefs['addtolistsonread'] = False
 default_prefs['gcnewonly'] = False
 default_prefs['gc_site_settings'] = {}
 default_prefs['allow_gc_from_ini'] = True
+default_prefs['gc_polish_cover'] = False
 
 default_prefs['countpagesstats'] = []
 

downloader.py

@@ -23,6 +23,7 @@ import getpass
 import string
 import ConfigParser
 from subprocess import call
+import pprint
 
 import logging
 if sys.version_info >= (2, 7):
@@ -132,14 +133,31 @@ def main(argv,
        parser.error("-u/--update-epub only works with epub")
 
    ## Attempt to update an existing epub.
+   chaptercount = None
+   output_filename = None
    if options.update:
-       (url,chaptercount) = get_dcsource_chaptercount(args[0])
-       print "Updating %s, URL: %s" % (args[0],url)
-       output_filename = args[0]
+       try:
+           (url,chaptercount) = get_dcsource_chaptercount(args[0])
+           if not url:
+               print "No story URL found in epub to update."
+               return
+           print "Updating %s, URL: %s" % (args[0],url)
+           output_filename = args[0]
+       except:
+           # if there's an error reading the update file, maybe it's a URL?
+           # we'll look for an existing outputfile down below.
+           url = args[0]
    else:
        url = args[0]
-           
-   configuration = Configuration(adapters.getConfigSectionFor(url),options.format)
+
+   try:
+       configuration = Configuration(adapters.getConfigSectionFor(url),options.format)
+   except exceptions.UnknownSite, e:
+       if options.list or options.normalize:
+           # list for page doesn't have to be a supported site.
+           configuration = Configuration("test1.com",options.format)
+       else:
+           raise e
 
    conflist = []
    homepath = join(expanduser("~"),".fanficdownloader")
@@ -174,8 +192,8 @@ def main(argv,
    if options.force:
        configuration.set("overrides","always_overwrite","true")
 
-   if options.update:
-       configuration.set("overrides","output_filename",args[0])
+   if options.update and chaptercount:
+       configuration.set("overrides","output_filename",output_filename)
        
    if options.update and not options.updatecover:
        configuration.set("overrides","never_make_cover","true")
@@ -192,16 +210,26 @@ def main(argv,
 
    if options.list or options.normalize:
        retlist = get_urls_from_page(args[0], configuration, normalize=options.normalize)
-       print "\n".join(retlist)
-               
+       print "\n".join(retlist)               
        return
 
    try:
        adapter = adapters.getAdapter(configuration,url)
        adapter.setChaptersRange(options.begin,options.end)
-           
-       ## Check for include_images and absence of PIL, give warning.
-       if adapter.getConfig('include_images'):
+
+       # check for updating from URL (vs from file)
+       if options.update and not chaptercount:
+           try:
+               writer = writers.getWriter("epub",configuration,adapter)
+               output_filename=writer.getOutputFileName()
+               (noturl,chaptercount) = get_dcsource_chaptercount(output_filename)
+               print "Updating %s, URL: %s" % (output_filename,url)
+           except:
+               options.update = False
+               pass
+
+       ## Check for include_images without no_image_processing. In absence of PIL, give warning.
+       if adapter.getConfig('include_images') and not adapter.getConfig('no_image_processing'):
            try:
                from calibre.utils.magick import Image
                logging.debug("Using calibre.utils.magick")
@@ -235,32 +263,37 @@ def main(argv,
 
        if options.update and not options.force:
            urlchaptercount = int(adapter.getStoryMetadataOnly().getMetadata('numChapters'))
-           
+
            if chaptercount == urlchaptercount and not options.metaonly:
-               print "%s already contains %d chapters." % (args[0],chaptercount)
+               print "%s already contains %d chapters." % (output_filename,chaptercount)
            elif chaptercount > urlchaptercount:
-               print "%s contains %d chapters, more than source: %d." % (args[0],chaptercount,urlchaptercount)
+               print "%s contains %d chapters, more than source: %d." % (output_filename,chaptercount,urlchaptercount)
+           elif chaptercount == 0:
+               print "%s doesn't contain any recognizable chapters, probably from a different source.  Not updating." % (output_filename)
            else:
+               # update now handled by pre-populating the old
+               # images and chapters in the adapter rather than
+               # merging epubs.
+               (url,
+                chaptercount,
+                adapter.oldchapters,
+                adapter.oldimgs,
+                adapter.oldcover,
+                adapter.calibrebookmark,
+                adapter.logfile) = get_update_data(output_filename)
+
                print "Do update - epub(%d) vs url(%d)" % (chaptercount, urlchaptercount)
-               if not options.metaonly:
 
-                   # update now handled by pre-populating the old
-                   # images and chapters in the adapter rather than
-                   # merging epubs.
-                   (url,
-                    chaptercount,
-                    adapter.oldchapters,
-                    adapter.oldimgs,
-                    adapter.oldcover,
-                    adapter.calibrebookmark,
-                    adapter.logfile) = get_update_data(args[0])
+               if not (options.update and chaptercount == urlchaptercount) \
+                       and adapter.getConfig("do_update_hook"):
+                   chaptercount = adapter.hookForUpdates(chaptercount)
 
-                   writeStory(configuration,adapter,"epub")
+               writeStory(configuration,adapter,"epub")
                    
        else:
            # regular download
            if options.metaonly:
-               print adapter.getStoryMetadataOnly()
+               pprint.pprint(adapter.getStoryMetadataOnly().getAllMetadata())
            
            output_filename=writeStory(configuration,adapter,options.format,options.metaonly)
        

fanficdownloader/HtmlTagStack.py

@@ -0,0 +1,57 @@
+# coding: utf-8
+
+import re
+import codecs
+
+stack = []
+
+def get_end_tag(tag):
+    if len(tag) > 0 and tag.find(u'<') > -1 and tag.rfind(u'>') > -1:
+        return re.sub(r'.*<([^\ >]+).*', r'</\1>', tag)
+    return u''
+
+def get_tag_name(tag):
+    if len(tag) > 0 and tag.find(u'<') > -1 and tag.rfind(u'>') > -1:
+        return re.sub(r'</*([^\ >]+).*', r'\1', tag)
+    return u''
+
+def push(tag):
+    if len(tag) > 0 and tag.find(u'<') > -1 and tag.rfind(u'>') > -1:
+        stack.append(tag)
+
+def pop():
+    if len(stack) > 0:
+        return stack.pop()
+    return u''
+
+def pop_end_tag():
+    return unicode(get_end_tag(pop()))
+
+def spool_end():
+    html = u''
+    for tag in reversed(stack):
+        html += get_end_tag(tag)
+    return html
+
+def spool_start():
+    html = u''
+    for item in stack:
+        html += item
+    return html
+
+def has_elements():
+    return len(stack) > 0
+
+def get_last():
+    # t = pop()
+    # push(t)
+    # return t
+    if len(stack) > 0:
+        return stack[len(stack)-1]
+    return u''
+
+def flush():
+    del stack[:]
+
+def get_stack():
+    return stack

fanficdownloader/__init__.py

@@ -4,13 +4,16 @@ try:
     # just a way to switch between web service and CLI/PI
     import google.appengine.api 
 except:
-    import sys
-    if sys.version_info >= (2, 7):
-        import logging
-        logger = logging.getLogger(__name__)
-        loghandler=logging.StreamHandler()
-        loghandler.setFormatter(logging.Formatter("FFDL:%(levelname)s:%(filename)s(%(lineno)d):%(message)s"))
-        logger.addHandler(loghandler)
-        loghandler.setLevel(logging.DEBUG)
-        logger.setLevel(logging.DEBUG)
+    try: # just a way to switch between CLI and PI
+        import calibre.constants
+    except:
+        import sys
+        if sys.version_info >= (2, 7):
+            import logging
+            logger = logging.getLogger(__name__)
+            loghandler=logging.StreamHandler()
+            loghandler.setFormatter(logging.Formatter("FFDL:%(levelname)s:%(filename)s(%(lineno)d):%(message)s"))
+            logger.addHandler(loghandler)
+            loghandler.setLevel(logging.DEBUG)
+            logger.setLevel(logging.DEBUG)
 

fanficdownloader/adapters/__init__.py

@@ -119,13 +119,33 @@ import adapter_nickandgregnet
 import adapter_potterheadsanonymouscom
 import adapter_simplyundeniablecom
 import adapter_scarheadnet
+import adapter_fictionpadcom
+import adapter_storiesonlinenet
+import adapter_trekiverseorg
+import adapter_literotica
+import adapter_voracity2eficcom
+import adapter_spikeluvercom
+import adapter_bloodshedversecom
+import adapter_nocturnallightnet
+import adapter_fanfichu
+import adapter_fanfictioncsodaidokhu
+import adapter_fictionmaniatv
+import adapter_bdsmgeschichten
+import adapter_tolkienfanfiction
+import adapter_themaplebookshelf
+import adapter_fannation
+import adapter_sheppardweircom
+import adapter_samandjacknet
+import adapter_csiforensicscom
+import adapter_lotrfanfictioncom
 
 ## This bit of complexity allows adapters to be added by just adding
 ## importing.  It eliminates the long if/else clauses we used to need
 ## to pick out the adapter.
-    
+
 ## List of registered site adapters.
 __class_list = []
+__domain_map = {}
 
 def imports():
     for name, val in globals().items():
@@ -135,9 +155,19 @@ def imports():
 for x in imports():
     if "fanficdownloader.adapters.adapter_" in x:
         #print x
-        __class_list.append(sys.modules[x].getClass())
+        cls = sys.modules[x].getClass()
+        __class_list.append(cls)
+        for site in cls.getAcceptDomains():
+            __domain_map[site]=cls
 
 def getNormalStoryURL(url):
+    r = getNormalStoryURLSite(url)
+    if r:
+        return r[0]
+    else:
+        return None
+
+def getNormalStoryURLSite(url):
     if not getNormalStoryURL.__dummyconfig:
         getNormalStoryURL.__dummyconfig = Configuration("test1.com","EPUB")
     # pulling up an adapter is pretty low over-head.  If
@@ -145,20 +175,23 @@ def getNormalStoryURL(url):
     try:
         adapter = getAdapter(getNormalStoryURL.__dummyconfig,url)
         url = adapter.url
+        site = adapter.getSiteDomain()
         del adapter
-        return url
+        return (url,site)
     except:
-        return None;
+        return None
 
 # kludgey function static/singleton
 getNormalStoryURL.__dummyconfig = None
 
-def getAdapter(config,url):
+def getAdapter(config,url,anyurl=False):
 
     #logger.debug("trying url:"+url)
     (cls,fixedurl) = getClassFor(url)
     #logger.debug("fixedurl:"+fixedurl)
     if cls:
+        if anyurl:
+            fixedurl = cls.getSiteExampleURLs().split()[0]
         adapter = cls(config,fixedurl) # raises InvalidStoryURL
         return adapter
     # No adapter found.
@@ -177,18 +210,20 @@ def getConfigSectionFor(url):
     (cls,fixedurl) = getClassFor(url)
     if cls:
         return cls.getConfigSection()
-    
+
     # No adapter found.
     raise exceptions.UnknownSite( url, [cls.getSiteDomain() for cls in __class_list] )
 
 def getClassFor(url):
     ## fix up leading protocol.
-    fixedurl = re.sub(r"(?i)^[htps]+[:/]+","http://",url.strip())
+    fixedurl = re.sub(r"(?i)^[htp]+(s?)[:/]+",r"http\1://",url.strip())
+    if fixedurl.startswith("//"):
+        fixedurl = "http:%s"%url
     if not fixedurl.startswith("http"):
         fixedurl = "http://%s"%url
     ## remove any trailing '#' locations.
     fixedurl = re.sub(r"#.*$","",fixedurl)
-    
+
     parsedUrl = up.urlparse(fixedurl)
     domain = parsedUrl.netloc.lower()
     if( domain != parsedUrl.netloc ):
@@ -199,17 +234,19 @@ def getClassFor(url):
         domain = domain.replace("www.","")
         #logger.debug("trying site:without www: "+domain)
         cls = getClassFromList(domain)
-        fixedurl = fixedurl.replace("http://www.","http://")
+        fixedurl = re.sub(r"^http(s?)://www\.",r"http\1://",fixedurl)
     if not cls:
         #logger.debug("trying site:www."+domain)
         cls = getClassFromList("www."+domain)
-        fixedurl = fixedurl.replace("http://","http://www.")
-        
-    fixedurl = cls.stripURLParameters(fixedurl)
-    
+        fixedurl = re.sub(r"^http(s?)://",r"http\1://www.",fixedurl)
+
+    if cls:
+        fixedurl = cls.stripURLParameters(fixedurl)
+
     return (cls,fixedurl)
-    
+
 def getClassFromList(domain):
-    for cls in __class_list:
-        if cls.matchesSite(domain):
-            return cls
+    try:
+        return __domain_map[domain]
+    except KeyError:
+        pass # return none.

fanficdownloader/adapters/adapter_adastrafanficcom.py

@@ -15,6 +15,7 @@
 # limitations under the License.
 #
 
+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -42,7 +43,7 @@ class AdAstraFanficComSiteAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -53,12 +54,19 @@ class AdAstraFanficComSiteAdapter(BaseSiteAdapter):
         return 'www.adastrafanfic.com'
 
     @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
         return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
 
+    def use_pagecache(self):
+        '''
+        adapters that will work with the page cache need to implement
+        this and change it to True.
+        '''
+        return True
+    
     def extractChapterUrlsAndMetadata(self):
 
         if self.is_adult or self.getConfig("is_adult"):
@@ -90,7 +98,7 @@ class AdAstraFanficComSiteAdapter(BaseSiteAdapter):
 
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php"))

fanficdownloader/adapters/adapter_archiveofourownorg.py

@@ -51,13 +51,13 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # get storyId from url--url validation guarantees query correct
         m = re.match(self.getSiteURLPattern(),url)
         if m:
             self.story.setMetadata('storyId',m.group('id'))
-            logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+            
             # normalized story URL.
             self._setURL('http://' + self.getSiteDomain() + '/works/'+self.story.getMetadata('storyId'))
         else:
@@ -77,14 +77,14 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'archiveofourown.org'
 
-    @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/works/123456 http://"+self.getSiteDomain()+"/collections/Some_Archive/works/123456 http://"+self.getSiteDomain()+"/works/123456/chapters/78901"
+    @classmethod
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/works/123456 http://"+cls.getSiteDomain()+"/collections/Some_Archive/works/123456 http://"+cls.getSiteDomain()+"/works/123456/chapters/78901"
 
     def getSiteURLPattern(self):
         # http://archiveofourown.org/collections/Smallville_Slash_Archive/works/159770
         # Discard leading zeros from story ID numbers--AO3 doesn't use them in it's own chapter URLs.
-        return re.escape("http://")+re.escape(self.getSiteDomain())+r"(/collections/[^/]+)?/works/0*(?P<id>\d+)"
+        return r"https?://"+re.escape(self.getSiteDomain())+r"(/collections/[^/]+)?/works/0*(?P<id>\d+)"
         
     ## Login
     def needToLoginCheck(self, data):
@@ -123,6 +123,13 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
         else:
             return True
 
+    def use_pagecache(self):
+        '''
+        adapters that will work with the page cache need to implement
+        this and change it to True.
+        '''
+        return True
+
     ## Getting the chapter list and the meta data, plus 'is adult' checking.
     def extractChapterUrlsAndMetadata(self):
 
@@ -145,51 +152,67 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
             
         except urllib2.HTTPError, e:
             if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.meta)
+                raise exceptions.StoryDoesNotExist(self.url)
             else:
                 raise e
-                
+
+        if "Sorry, we couldn&#x27;t find the work you were looking for." in data:
+            raise exceptions.StoryDoesNotExist(self.url)
+            
         if self.needToLoginCheck(data):
             # need to log in for this one.
             self.performLogin(url,data)
-            data = self._fetchUrl(url)
-            meta = self._fetchUrl(metaurl)
+            data = self._fetchUrl(url,usecache=False)
+            meta = self._fetchUrl(metaurl,usecache=False)
             
         # use BeautifulSoup HTML parser to make everything easier to find.
         soup = bs.BeautifulSoup(data)
+        for tag in soup.findAll('div',id='admin-banner'):
+            tag.extract()
         metasoup = bs.BeautifulSoup(meta)
+        for tag in metasoup.findAll('div',id='admin-banner'):
+            tag.extract()
 
         # Now go hunting for all the meta data and the chapter list.
         
         ## Title
-        a = soup.find('a', href=re.compile(r"^/works/\d+$"))
-        self.story.setMetadata('title',a.string)
+        a = soup.find('a', href=re.compile(r"/works/\d+$"))
+        self.story.setMetadata('title',stripHTML(a))
 		
         # Find authorid and URL from... author url.
-        alist = soup.findAll('a', href=re.compile(r"^/users/\w+/pseuds/\w+"))
+        alist = soup.findAll('a', href=re.compile(r"/users/\w+/pseuds/\w+"))
         if len(alist) < 1: # ao3 allows for author 'Anonymous' with no author link.
             self.story.setMetadata('author','Anonymous')
             self.story.setMetadata('authorUrl','http://archiveofourown.org/')
             self.story.setMetadata('authorId','0')
         else:
             for a in alist:
-                self.story.addToList('authorId',a['href'].split('/')[2])
-                self.story.addToList('authorUrl','http://'+self.host+a['href'])
+                self.story.addToList('authorId',a['href'].split('/')[-1])
+                self.story.addToList('authorUrl',a['href'])
                 self.story.addToList('author',a.text)
 
+        newestChapter = None
+        self.newestChapterNum = None # save for comparing during update.
+        # Scan all chapters to find the oldest and newest, on AO3 it's
+        # possible for authors to insert new chapters out-of-order or
+        # change the dates of earlier ones by editing them--That WILL
+        # break epub update.
         # Find the chapters:
         chapters=soup.findAll('a', href=re.compile(r'/works/'+self.story.getMetadata('storyId')+"/chapters/\d+$"))
         self.story.setMetadata('numChapters',len(chapters))
         logger.debug("numChapters: (%s)"%self.story.getMetadata('numChapters'))
-        for x in range(0,len(chapters)):
-            # just in case there's tags, like <i> in chapter titles.
-            chapter=chapters[x]
-            if len(chapters)==1:
-                self.chapterUrls.append((self.story.getMetadata('title'),'http://'+self.host+chapter['href']+addurl))
-            else:
+        if len(chapters)==1:
+            self.chapterUrls.append((self.story.getMetadata('title'),'http://'+self.host+chapters[0]['href']+addurl))
+        else:
+            for index, chapter in enumerate(chapters):
+                # strip just in case there's tags, like <i> in chapter titles.
                 self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+chapter['href']+addurl))
-
-
+                # (2013-09-21)
+                date = stripHTML(chapter.findNext('span'))[1:-1]
+                chapterDate = makeDate(date,self.dateformat)
+                if newestChapter == None or chapterDate > newestChapter:
+                    newestChapter = chapterDate
+                    self.newestChapterNum = index
 
         a = metasoup.find('blockquote',{'class':'userstuff'})
         if a != None:
@@ -210,10 +233,7 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
         if a != None:
             warnings = a.findAll('a',{'class':"tag"})
             for warning in warnings:
-                if warning.string == "Author Chose Not To Use Archive Warnings":
-                    warning.string = "No Archive Warnings Apply"
-                if warning.string != "No Archive Warnings Apply":
-                    self.story.addToList('warnings',warning.string)
+                self.story.addToList('warnings',warning.string)
 		
         a = metasoup.find('dd',{'class':"freeform tags"})
         if a != None:
@@ -288,46 +308,58 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
                 self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
 
 		
-        try:
-            # Find Series name from series URL.
-            a = metasoup.find('dd',{'class':"series"})
-            b = a.find('a', href=re.compile(r"/series/\d+"))
-            series_name = b.string
-            series_url = 'http://'+self.host+b['href']
-            series_index = int(a.text.split(' ')[1])
-            self.setSeries(series_name, series_index)
-            self.story.setMetadata('seriesUrl',series_url)
-            
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
+        # Find Series name from series URL.
+        ddseries = metasoup.find('dd',{'class':"series"})
 
+        if ddseries:
+            for i, a in enumerate(ddseries.findAll('a', href=re.compile(r"/series/\d+"))):
+                series_name = stripHTML(a)
+                series_url = 'http://'+self.host+a['href']
+                series_index = int(stripHTML(a.previousSibling).replace(', ','').split(' ')[1]) # "Part # of" or ", Part #"
+                self.story.setMetadata('series%02d'%i,"%s [%s]"%(series_name,series_index))
+                self.story.setMetadata('series%02dUrl'%i,series_url)
+                if i == 0:
+                    self.setSeries(series_name, series_index)
+                    self.story.setMetadata('seriesUrl',series_url)
+
+    def hookForUpdates(self,chaptercount):
+        if self.oldchapters and len(self.oldchapters) > self.newestChapterNum:
+            print("Existing epub has %s chapters\nNewest chapter is %s.  Discarding old chapters from there on."%(len(self.oldchapters), self.newestChapterNum+1))
+            self.oldchapters = self.oldchapters[:self.newestChapterNum]
+        return len(self.oldchapters)
+            
     # grab the text for an individual chapter.
     def getChapterText(self, url):
         logger.debug('Getting chapter text from: %s' % url)
 		
-        chapter=bs.BeautifulSoup('<div class="story"></div>')
+        chapter=bs.BeautifulSoup('<div class="story"></div>').find('div')
         data = self._fetchUrl(url)
         soup = bs.BeautifulSoup(data,selfClosingTags=('br','hr'))
-		
-        headnotes = soup.find('div', {'class' : "preface group"}).find('div', {'class' : "notes module"})
-        if headnotes != None:
-            headnotes = headnotes.find('blockquote', {'class' : "userstuff"})
+
+        exclude_notes=self.getConfigList('exclude_notes')
+
+        if 'authorheadnotes' not in exclude_notes:
+            headnotes = soup.find('div', {'class' : "preface group"}).find('div', {'class' : "notes module"})
             if headnotes != None:
-                chapter.append("<b>Author's Note:</b>")
-                chapter.append(headnotes)
+                headnotes = headnotes.find('blockquote', {'class' : "userstuff"})
+                if headnotes != None:
+                    chapter.append("<b>Author's Note:</b>")
+                    chapter.append(headnotes)
         
-        chapsumm = soup.find('div', {'id' : "summary"})
-        if chapsumm != None:
-            chapsumm = chapsumm.find('blockquote')
-            chapter.append("<b>Summary for the Chapter:</b>")
-            chapter.append(chapsumm)
-        chapnotes = soup.find('div', {'id' : "notes"})
-        if chapnotes != None:
-            chapnotes = chapnotes.find('blockquote')
+        if 'chaptersummary' not in exclude_notes:
+            chapsumm = soup.find('div', {'id' : "summary"})
+            if chapsumm != None:
+                chapsumm = chapsumm.find('blockquote')
+                chapter.append("<b>Summary for the Chapter:</b>")
+                chapter.append(chapsumm)
+                
+        if 'chapterheadnotes' not in exclude_notes:
+            chapnotes = soup.find('div', {'id' : "notes"})
             if chapnotes != None:
-                chapter.append("<b>Notes for the Chapter:</b>")
-                chapter.append(chapnotes)
+                chapnotes = chapnotes.find('blockquote')
+                if chapnotes != None:
+                    chapter.append("<b>Notes for the Chapter:</b>")
+                    chapter.append(chapnotes)
 		
         text = soup.find('div', {'class' : "userstuff module"})
         chtext = text.find('h3', {'class' : "landmark heading"})
@@ -335,17 +367,19 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
             chtext.extract()
         chapter.append(text)
 		
-        chapfoot = soup.find('div', {'class' : "end notes module", 'role' : "complementary"})
-        if chapfoot != None:
-            chapfoot = chapfoot.find('blockquote')
-            chapter.append("<b>Notes for the Chapter:</b>")
-            chapter.append(chapfoot)
+        if 'chapterfootnotes' not in exclude_notes:
+            chapfoot = soup.find('div', {'class' : "end notes module", 'role' : "complementary"})
+            if chapfoot != None:
+                chapfoot = chapfoot.find('blockquote')
+                chapter.append("<b>Notes for the Chapter:</b>")
+                chapter.append(chapfoot)
 		
-        footnotes = soup.find('div', {'id' : "work_endnotes"})
-        if footnotes != None:
-            footnotes = footnotes.find('blockquote')
-            chapter.append("<b>Author's Note:</b>")
-            chapter.append(footnotes)
+        if 'authorfootnotes' not in exclude_notes:
+            footnotes = soup.find('div', {'id' : "work_endnotes"})
+            if footnotes != None:
+                footnotes = footnotes.find('blockquote')
+                chapter.append("<b>Author's Note:</b>")
+                chapter.append(footnotes)
 			
         if None == soup:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)

fanficdownloader/adapters/adapter_archiveskyehawkecom.py

@@ -49,7 +49,7 @@ class ArchiveSkyeHawkeComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/story.php?no='+self.story.getMetadata('storyId'))
@@ -71,7 +71,7 @@ class ArchiveSkyeHawkeComAdapter(BaseSiteAdapter):
         return ['archive.skyehawke.com','www.skyehawke.com']
 
     @classmethod
-    def getSiteExampleURLs(self):
+    def getSiteExampleURLs(cls):
         return "http://archive.skyehawke.com/story.php?no=1234 http://www.skyehawke.com/archive/story.php?no=1234  http://skyehawke.com/archive/story.php?no=1234"
 
     def getSiteURLPattern(self):
@@ -91,8 +91,6 @@ class ArchiveSkyeHawkeComAdapter(BaseSiteAdapter):
             else:
                 raise e
 
-        data = self._fetchUrl(url)
-            
         # use BeautifulSoup HTML parser to make everything easier to find.
         soup = bs.BeautifulSoup(data)
         # print data
@@ -101,7 +99,7 @@ class ArchiveSkyeHawkeComAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('div', {'class':"story border"}).find('span',{'class':'left'})
-        title=a.text.split('"')[1]
+        title=stripHTML(a).split('"')[1]
         self.story.setMetadata('title',title)
         
         # Find authorid and URL from... author url.

fanficdownloader/adapters/adapter_ashwindersycophanthexcom.py

@@ -15,6 +15,7 @@
 # limitations under the License.
 #
 
+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -48,7 +49,7 @@ class AshwinderSycophantHexComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
 		
         
         # normalized story URL.
@@ -67,8 +68,8 @@ class AshwinderSycophantHexComAdapter(BaseSiteAdapter):
         return 'ashwinder.sycophanthex.com'
 
     @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
         return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
@@ -157,7 +158,7 @@ class AshwinderSycophantHexComAdapter(BaseSiteAdapter):
         for info in asoup.findAll('table', {'width' : '100%', 'bordercolor' : re.compile(r'#')}):
             a = info.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
             if a != None:
-                self.story.setMetadata('title',a.string)
+                self.story.setMetadata('title',stripHTML(a))
                 break
 		
 

fanficdownloader/adapters/adapter_asr3slashzoneorg.py

@@ -15,6 +15,7 @@
 # limitations under the License.
 #
 
+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -46,7 +47,7 @@ class Asr3SlashzoneOrgAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/archive/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -64,8 +65,8 @@ class Asr3SlashzoneOrgAdapter(BaseSiteAdapter):
         return 'asr3.slashzone.org'
 
     @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/archive/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/archive/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
         return re.escape("http://"+self.getSiteDomain()+"/archive/viewstory.php?sid=")+r"\d+$"
@@ -128,7 +129,7 @@ class Asr3SlashzoneOrgAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))

fanficdownloader/adapters/adapter_bdsmgeschichten.py

@@ -0,0 +1,346 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2014 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+import urlparse
+import time
+
+from .. import BeautifulSoup as bs
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter, makeDate
+
+def _translate_date_german_english(date):
+    fullmon = {"Januar":"01",
+               "Februar":"02",
+               u"März":"03",
+               "April":"04",
+               "Mai":"05",
+               "Juni":"06",
+               "Juli":"07",
+               "August":"08",
+               "September":"09",
+               "Oktober":"10",
+               "November":"11",
+               "Dezember":"12"}
+    for (name,num) in fullmon.items():
+        date = date.replace(name,num)
+    return date
+
+_REGEX_TRAILING_DIGIT = re.compile("(\d+)$")
+_REGEX_DASH_TO_END = re.compile("-[^-]+$")
+_REGEX_CHAPTER_TITLE = re.compile(ur"""
+    \s*
+    [\u2013-]?
+    \s*
+    ([\dIVX-]+)?
+    \.?
+    \s*
+    [\[\(]?
+    \s*
+    (Teil|Kapitel|Tag)?
+    \s*
+    ([\dIVX-]+)?
+    \s*
+    [\]\)]?
+    \s*
+    $
+""", re.VERBOSE)
+_INITIAL_STEP = 5
+
+class BdsmGeschichtenAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["utf8", "Windows-1252"]
+
+        self.story.setMetadata('siteabbrev','bdsmgesch')
+
+        # Replace possible chapter numbering
+        chapterMatch = _REGEX_TRAILING_DIGIT.search(url)
+        if chapterMatch is None:
+            self.maxChapter = 1
+        else:
+            self.maxChapter = int(chapterMatch.group(1))
+        # url = re.sub(_REGEX_TRAILING_DIGIT, "1", url)
+
+        # set storyId
+        self.story.setMetadata('storyId', re.compile(self.getSiteURLPattern()).match(url).group('storyId'))
+
+        # normalize URL
+        self._setURL('http://%s/%s' % (self.getSiteDomain(), self.story.getMetadata('storyId')))
+
+        self.dateformat = '%d. %m %Y - %H:%M'
+
+    @staticmethod
+    def getSiteDomain():
+        return 'bdsm-geschichten.net'
+
+    @classmethod
+    def getAcceptDomains(cls):
+        return ['www.bdsm-geschichten.net', 'www.bdsm-geschichten.net']
+
+    @classmethod
+    def getSiteExampleURLs(cls):
+        return "http://www.bdsm-geschichten.net/title-of-story-1 http://bdsm-geschichten.net/title-of-story-1"
+
+    def getSiteURLPattern(self):
+        return r"http://(www\.)?bdsm-geschichten.net/(?P<storyId>[a-zA-Z0-9_-]+)"
+
+    def extractChapterUrlsAndMetadata(self):
+
+        if not (self.is_adult or self.getConfig("is_adult")):
+            raise exceptions.AdultCheckRequired(self.url)
+
+        try:
+            data1 = self._fetchUrl(self.url)
+            soup = bs.BeautifulSoup(data1)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        #strip comments from soup
+        [comment.extract() for comment in soup.findAll(text=lambda text:isinstance(text, bs.Comment))]
+
+        # Cache the soups so we won't have to redownload in getChapterText later
+        self.soupsCache = {}
+        self.soupsCache[self.url] = soup
+
+        # author
+        authorDiv = soup.find("div", "author-pane-line author-name")
+        authorId = authorDiv.string.strip()
+        self.story.setMetadata('authorId', authorId)
+        self.story.setMetadata('author', authorId)
+        # TODO not really true need to be loggedin for this to work or fetch userid
+        self.story.setMetadata('authorUrl','http://'+self.host+'/'+authorId)
+
+        # TODO better metadata
+        date = soup.find("div", {"class": "submitted"}).string.strip()
+        date = re.sub(" &#151;.*", "", date)
+        date = _translate_date_german_english(date)
+        self.story.setMetadata('datePublished', makeDate(date, self.dateformat))
+        title1 = soup.find("h1", {'class': 'title'}).string
+
+
+        for tagLink in soup.find("ul", "taxonomy").findAll("a"):
+            self.story.addToList('category', tagLink.string)
+
+        ## Retrieve chapter soups
+        if self.getConfig('find_chapters') == 'guess':
+            self.chapterUrls = []
+            self._find_chapters_by_guessing(title1)
+        else:
+            self._find_chapters_by_parsing(soup)
+
+        firstChapterUrl = self.chapterUrls[0][1]
+        if firstChapterUrl in self.soupsCache:
+            firstChapterSoup = self.soupsCache[firstChapterUrl]
+            h1 = firstChapterSoup.find("h1").text
+        else:
+            h1 = soup.find("h1").text
+
+        h1 = re.sub(_REGEX_CHAPTER_TITLE, "", h1)
+        self.story.setMetadata('title', h1)
+        self.story.setMetadata('numChapters', len(self.chapterUrls))
+        return
+
+    def _find_chapters_by_parsing(self, soup):
+
+        # store original soup
+        origSoup = soup
+
+        #
+        # find first chapter
+        #
+        firstLink = None
+        firstLinkDiv = soup.find("div", "field-field-erster-teil")
+        if firstLinkDiv is not None:
+            firstLink = "http://%s%s" % (self.getSiteDomain(), firstLinkDiv.findNext("a")['href'])
+            logger.debug("Found first chapter right away <%s>" % firstLink)
+            try:
+                soup = bs.BeautifulSoup(self._fetchUrl(firstLink))
+                self.soupsCache[firstLink] = soup
+                self.chapterUrls.insert(0, (soup.find("h1").text, firstLink))
+            except urllib2.HTTPError, e:
+                if e.code == 404:
+                    raise exceptions.StoryDoesNotExist(self.url)
+                else:
+                    raise exceptions.StoryDoesNotExist(firstLink)
+        else:
+            logger.debug("DIDN'T find first chapter right away")
+            # parse previous Link until first
+            while True:
+                prevLink = None
+                prevLinkDiv = soup.find("div", "field-field-vorheriger-teil")
+                if prevLinkDiv is not None:
+                    prevLink = prevLinkDiv.find("a")
+                if prevLink is None:
+                    prevLink = soup.find("a", text=re.compile("&lt;&lt;&lt;")) # <<<
+                if prevLink is None:
+                    logger.debug("Couldn't find prev part")
+                    break
+                else:
+                    logger.debug("Previous Chapter <%s>" % prevLink)
+                    if type(prevLink) != bs.Tag or prevLink.name != "a":
+                        prevLink = prevLink.findParent("a")
+                    if prevLink is None or '#' in prevLink['href']:
+                        logger.debug("Couldn't find prev part (false positive) <%s>" % prevLink)
+                        break
+                    prevLink = prevLink['href']
+                try:
+                    soup = bs.BeautifulSoup(self._fetchUrl(prevLink))
+                    self.soupsCache[prevLink] = soup
+                    prevTtitle = soup.find("h1", {'class': 'title'}).string
+                    self.chapterUrls.insert(0, (prevTtitle, prevLink))
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(nextLink)
+                    else:
+                        raise e
+                firstLink = prevLink
+
+        # if first chapter couldn't be determined, assume the URL originally
+        # passed is the first chapter
+        if firstLink is None:
+            logger.debug("Couldn't set first chapter")
+            firstLink = self.url
+            self.chapterUrls.insert(0, (soup.find("h1").text, firstLink))
+
+        # set first URL
+        logger.debug("Set first link: %s" % firstLink)
+        self._setURL(firstLink)
+        self.story.setMetadata('storyId', re.compile(self.getSiteURLPattern()).match(firstLink).group('storyId'))
+
+        #
+        # Parse next chapters
+        #
+        while True:
+            nextLink = None
+            nextLinkDiv = soup.find("div", "field-field-naechster-teil")
+            if nextLinkDiv is not None:
+                nextLink = nextLinkDiv.find("a")
+            if nextLink is None:
+                nextLink = soup.find("a", text=re.compile("&gt;&gt;&gt;"))
+            if nextLink is None:
+                nextLink = soup.find("a", text=re.compile("Fortsetzung"))
+
+            if nextLink is None:
+                logger.debug("Couldn't find next part")
+                break
+            else:
+                if type(nextLink) != bs.Tag or nextLink.name != "a":
+                    nextLink = nextLink.findParent("a")
+                if nextLink is None or '#' in nextLink['href']:
+                    logger.debug("Couldn't find next part (false positive) <%s>" % nextLink)
+                    break
+                nextLink = nextLink['href']
+
+            if not nextLink.startswith('http:'):
+                nextLink = 'http://' + self.getSiteDomain() + nextLink
+
+            for loadedChapter in self.chapterUrls:
+                if loadedChapter[0] == nextLink:
+                    logger.debug("ERROR: Repeating chapter <%s> Try to fix it" % nextLink)
+                    nextLinkMatch = _REGEX_TRAILING_DIGIT.match(nextLink)
+                    if nextLinkMatch is not None:
+                        curChap = nextLinkMatch.group(1)
+                        nextLink = re.sub(_REGEX_TRAILING_DIGIT, str(int(curChap) + 1), nextLink)
+                    else:
+                        break
+            try:
+                data = self._fetchUrl(nextLink)
+                soup = bs.BeautifulSoup(data)
+            except urllib2.HTTPError, e:
+                if e.code == 404:
+                    raise exceptions.StoryDoesNotExist(nextLink)
+                else:
+                    raise e
+            title2 = soup.find("h1", {'class': 'title'}).string
+            self.chapterUrls.append((title2, nextLink))
+            logger.debug("Grabbing next chapter URL " + nextLink)
+            self.soupsCache[nextLink] = soup
+            # [comment.extract() for comment in soup.findAll(text=lambda text:isinstance(text, bs.Comment))]
+        logger.debug("Chapters: %s" % self.chapterUrls)
+
+
+    def _find_chapters_by_guessing(self, title1):
+        step = _INITIAL_STEP
+        curMax = self.maxChapter + step
+        lastHit = True
+        while True:
+            nextChapterUrl = re.sub(_REGEX_TRAILING_DIGIT, str(curMax), self.url) 
+            if nextChapterUrl == self.url:
+                logger.debug("Unable to guess next chapter because URL doesn't end in numbers")
+                break;
+            try:
+                logger.debug("Trying chapter URL " + nextChapterUrl)
+                data = self._fetchUrl(nextChapterUrl)
+                hit = True
+            except urllib2.HTTPError, e:
+                if e.code == 404:
+                    hit = False
+                else:
+                    raise e
+            if hit:
+                logger.debug("Found chapter URL " + nextChapterUrl)
+                self.maxChapter = curMax
+                self.soupsCache[nextChapterUrl] = bs.BeautifulSoup(data)
+                if not lastHit:
+                    break
+                lastHit = curMax
+                curMax += step
+            else:
+                lastHit = False
+                curMax -= 1
+            logger.debug(curMax)
+
+        for i in xrange(1, self.maxChapter):
+            nextChapterUrl = re.sub(_REGEX_TRAILING_DIGIT, str(i), self.url)
+            nextChapterTitle = re.sub("1", str(i), title1)
+            self.chapterUrls.append((nextChapterTitle, nextChapterUrl))
+
+    def getChapterText(self, url):
+
+        if url in self.soupsCache:
+            logger.debug('Getting chapter <%s> from cache' % url)
+            soup = self.soupsCache[url]
+        else:
+            logger.debug('Downloading chapter <%s>' % url)
+            data1 = self._fetchUrl(url)
+            soup = bs.BeautifulSoup(data1)
+            #strip comments from soup
+            [comment.extract() for comment in soup.findAll(text=lambda text:isinstance(text, bs.Comment))]
+
+        # get story text
+        storyDiv1 = bs.Tag(soup, "div")
+        for para in soup.find("div", "full-node").find('div', 'content').findAll("p"):
+            storyDiv1.append(para)
+        storyDiv1.append('<br />')
+        storytext = self.utf8FromSoup(url,storyDiv1)
+
+        return storytext
+
+
+def getClass():
+    return BdsmGeschichtenAdapter

fanficdownloader/adapters/adapter_bloodshedversecom.py

@@ -0,0 +1,193 @@
+from datetime import timedelta
+import re
+import urllib2
+import urlparse
+
+from .. import BeautifulSoup
+from ..htmlcleanup import stripHTML
+
+from base_adapter import BaseSiteAdapter, makeDate
+from .. import exceptions
+
+
+def getClass():
+    return BloodshedverseComAdapter
+
+
+def _get_query_data(url):
+    components = urlparse.urlparse(url)
+    query_data = urlparse.parse_qs(components.query)
+    return dict((key, data[0]) for key, data in query_data.items())
+
+
+class BloodshedverseComAdapter(BaseSiteAdapter):
+    SITE_ABBREVIATION = 'bvc'
+    SITE_DOMAIN = 'bloodshedverse.com'
+
+    BASE_URL = 'http://' + SITE_DOMAIN + '/'
+    READ_URL_TEMPLATE = BASE_URL + 'stories.php?go=read&no=%s'
+
+    STARTED_DATETIME_FORMAT = '%m/%d/%Y'
+    UPDATED_DATETIME_FORMAT = '%m/%d/%Y %I:%M'
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        query_data = urlparse.parse_qs(self.parsedUrl.query)
+        story_no = query_data['no'][0]
+
+        self.story.setMetadata('storyId', story_no)
+        self._setURL(self.READ_URL_TEMPLATE % story_no)
+        self.story.setMetadata('siteabbrev', self.SITE_ABBREVIATION)
+
+    def _customized_fetch_url(self, url, exception=None, parameters=None):
+        if exception:
+            try:
+                data = self._fetchUrl(url, parameters)
+            except urllib2.HTTPError:
+                raise exception(self.url)
+        # Just let self._fetchUrl throw the exception, don't catch and
+        # customize it.
+        else:
+            data = self._fetchUrl(url, parameters)
+
+        return BeautifulSoup.BeautifulSoup(data)
+
+    @staticmethod
+    def getSiteDomain():
+        return BloodshedverseComAdapter.SITE_DOMAIN
+
+    @classmethod
+    def getSiteExampleURLs(cls):
+        return cls.READ_URL_TEMPLATE % 1234
+
+    def getSiteURLPattern(self):
+        return re.escape(self.BASE_URL + 'stories.php?go=') + r'(read|chapters)\&no=\d+$'
+
+    # Override stripURLParameters so the "no" parameter won't get stripped
+    @classmethod
+    def stripURLParameters(cls, url):
+        return url
+
+    def extractChapterUrlsAndMetadata(self):
+        soup = self._customized_fetch_url(self.url)
+
+        # Since no 404 error code we have to raise the exception ourselves.
+        # A title that is just 'by' indicates that there is no author name
+        # and no story title available.
+        if stripHTML(soup.title) == 'by':
+            raise exceptions.StoryDoesNotExist(self.url)
+
+        for option in soup.find('select', {'name': 'chapter'}):
+            title = stripHTML(option)
+            url = self.READ_URL_TEMPLATE % option['value']
+            self.chapterUrls.append((title, url))
+
+        # Get the URL to the author's page and find the correct story entry to
+        # scrape the metadata
+        author_url = urlparse.urljoin(self.url, soup.find('a', {'class': 'headline'})['href'])
+        soup = self._customized_fetch_url(author_url)
+
+        story_no = self.story.getMetadata('storyId')
+        # Ignore first list_box div, it only contains the author information
+        for list_box in soup('div', {'class': 'list_box'})[1:]:
+            url = list_box.find('a', {'class': 'fictitle'})['href']
+            query_data = _get_query_data(url)
+
+            # Found the div containing the story's metadata; break the loop and
+            # parse the element
+            if query_data['no'] == story_no:
+                break
+        else:
+            raise exceptions.FailedToDownload(self.url)
+
+        title_anchor = list_box.find('a', {'class': 'fictitle'})
+        self.story.setMetadata('title', stripHTML(title_anchor))
+
+        author_anchor = title_anchor.findNextSibling('a')
+        self.story.setMetadata('author', stripHTML(author_anchor))
+        self.story.setMetadata('authorId', _get_query_data(author_anchor['href'])['who'])
+        self.story.setMetadata('authorUrl', urlparse.urljoin(self.url, author_anchor['href']))
+
+        list_review = list_box.find('div', {'class': 'list_review'})
+        reviews = stripHTML(list_review.a).split(' ', 1)[0]
+        self.story.setMetadata('reviews', reviews)
+
+        summary_div = list_box.find('div', {'class': 'list_summary'})
+        if not self.getConfig('keep_summary_html'):
+            summary = ''.join(summary_div(text=True))
+        else:
+            summary = self.utf8FromSoup(author_url, summary_div)
+
+        self.story.setMetadata('description', summary)
+
+        # I'm assuming this to be the category, not sure what else it could be
+        first_listinfo = list_box.find('div', {'class': 'list_info'})
+        self.story.addToList('category', stripHTML(first_listinfo.a))
+
+        for list_info in first_listinfo.findNextSiblings('div', {'class': 'list_info'}):
+            for b_tag in list_info('b'):
+                key = b_tag.string.strip(': ')
+                # Strip colons from the beginning, superfluous spaces and minus
+                # characters from the end, and possibly trailing commas from
+                # the warnings if only one is present
+                value = b_tag.nextSibling.string.strip(': -,')
+
+                if key == 'Genre':
+                    for genre in value.split(', '):
+                        # Ignore the "none" genre
+                        if not genre == 'none':
+                            self.story.addToList('genre', genre)
+
+                elif key == 'Rating':
+                    self.story.setMetadata('rating', value)
+
+                elif key == 'Complete':
+                    self.story.setMetadata('status', 'Completed' if value == 'Yes' else 'In-Progress')
+
+                elif key == 'Warning':
+                    for warning in value.split(', '):
+                        # The string here starts with ", " before the actual list
+                        # of values sometimes, so check for an empty warning
+                        # and ignore the "none" warning.
+                        if not warning or warning == 'none':
+                            continue
+
+                        self.story.addToList('warnings', warning)
+
+                elif key == 'Chapters':
+                    self.story.setMetadata('numChapters', int(value))
+
+                elif key == 'Words':
+                    # Apparently only numChapters need to be an integer for
+                    # some strange reason. Remove possible ',' characters as to
+                    # not confuse the codebase down the line
+                    self.story.setMetadata('numWords', value.replace(',', ''))
+
+                elif key == 'Started':
+                    self.story.setMetadata('datePublished', makeDate(value, self.STARTED_DATETIME_FORMAT))
+
+                elif key == 'Updated':
+                    date_string, period = value.rsplit(' ', 1)
+                    date = makeDate(date_string, self.UPDATED_DATETIME_FORMAT)
+
+                    # Rather ugly hack to work around Calibre's changing of
+                    # Python's locale setting, causing am/pm to not be properly
+                    # parsed by strptime() when using a non-english locale
+                    if period == 'pm':
+                        date += timedelta(hours=12)
+                    self.story.setMetadata('dateUpdated', date)
+
+        if self.story.getMetadata('rating') == 'NC-17' and not (self.is_adult or self.getConfig('is_adult')):
+            raise exceptions.AdultCheckRequired(self.url)
+
+    def getChapterText(self, url):
+        soup = self._customized_fetch_url(url)
+        storytext_div = soup.find('div', {'class': 'storytext'})
+
+        if self.getConfig('strip_text_links'):
+            for anchor in storytext_div('a', {'class': 'FAtxtL'}):
+                navigable_string = BeautifulSoup.NavigableString(anchor.string)
+                anchor.replaceWith(navigable_string)
+
+        return self.utf8FromSoup(url, storytext_div)

fanficdownloader/adapters/adapter_bloodtiesfancom.py

@@ -15,6 +15,7 @@
 # limitations under the License.
 #
 
+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -70,7 +71,7 @@ class BloodTiesFansComAdapter(BaseSiteAdapter): # XXX
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
@@ -89,8 +90,8 @@ class BloodTiesFansComAdapter(BaseSiteAdapter): # XXX
         return 'bloodties-fans.com' # XXX
 
     @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/fiction/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/fiction/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
         return re.escape("http://"+self.getSiteDomain()+"/fiction/viewstory.php?sid=")+r"\d+$"
@@ -207,7 +208,7 @@ class BloodTiesFansComAdapter(BaseSiteAdapter): # XXX
         
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))

fanficdownloader/adapters/adapter_buffynfaithnet.py

@@ -58,7 +58,7 @@ class BuffyNFaithNetAdapter(BaseSiteAdapter):
         m = re.match(self.getSiteURLPattern(),url)
         if m:
             self.story.setMetadata('storyId',m.group('id'))
-            logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+            
             # normalized story URL. gets rid of chapter if there, left with ch 1 URL on this site
             nurl = "http://"+self.getSiteDomain()+"/fanfictions/index.php?act=vie&id="+self.story.getMetadata('storyId')
             self._setURL(nurl)
@@ -69,7 +69,7 @@ class BuffyNFaithNetAdapter(BaseSiteAdapter):
             raise exceptions.InvalidStoryURL(url,
                                              self.getSiteDomain(),
                                              self.getSiteExampleURLs())
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','bnfnet')
@@ -87,11 +87,11 @@ class BuffyNFaithNetAdapter(BaseSiteAdapter):
     
     def setHeader(self):
         "buffynfaith.net wants a Referer for images.  Used both above and below(after cookieproc added)"
-        self.opener.addheaders = [('Referer', 'http://'+self.getSiteDomain()+'/')]
+        self.opener.addheaders.append(('Referer', 'http://'+self.getSiteDomain()+'/'))
         
     @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/fanfictions/index.php?act=vie&id=1234 http://buffynfaith.net/fanfictions/index.php?act=ovr&id=1234 http://buffynfaith.net/fanfictions/index.php?act=vie&id=1234&ch=2"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/fanfictions/index.php?act=vie&id=1234 http://"+cls.getSiteDomain()+"/fanfictions/index.php?act=ovr&id=1234 http://"+cls.getSiteDomain()+"/fanfictions/index.php?act=vie&id=1234&ch=2"
 
     def getSiteURLPattern(self):
         #http://buffynfaith.net/fanfictions/index.php?act=vie&id=963
@@ -101,6 +101,13 @@ class BuffyNFaithNetAdapter(BaseSiteAdapter):
             r"(vie|ovr)&id=(?P<id>\d+)(&ch=(?P<ch>\d+))?$"
         return p
 
+    def use_pagecache(self):
+        '''
+        adapters that will work with the page cache need to implement
+        this and change it to True.
+        '''
+        return True
+    
     def extractChapterUrlsAndMetadata(self):
 
         dateformat = "%d %B %Y"
@@ -109,7 +116,6 @@ class BuffyNFaithNetAdapter(BaseSiteAdapter):
         
         #set a cookie to get past adult check
         if self.is_adult or self.getConfig("is_adult"):
-            cookieproc = urllib2.HTTPCookieProcessor()
             cookie = cl.Cookie(version=0, name='my_age', value='yes',
                                port=None, port_specified=False,
                                domain=self.getSiteDomain(), domain_specified=False, domain_initial_dot=False,
@@ -121,8 +127,7 @@ class BuffyNFaithNetAdapter(BaseSiteAdapter):
                                comment_url=None,
                                rest={'HttpOnly': None},
                                rfc2109=False)
-            cookieproc.cookiejar.set_cookie(cookie)
-            self.opener = urllib2.build_opener(cookieproc)        
+            self.cookiejar.set_cookie(cookie)
             self.setHeader()
 
         try:

fanficdownloader/adapters/adapter_castlefansorg.py

@@ -15,6 +15,7 @@
 # limitations under the License.
 #
 
+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -72,7 +73,7 @@ class CastleFansOrgAdapter(BaseSiteAdapter): # XXX
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
@@ -91,8 +92,8 @@ class CastleFansOrgAdapter(BaseSiteAdapter): # XXX
         return 'castlefans.org' # XXX
 
     @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/fanfic/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/fanfic/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
         return re.escape("http://"+self.getSiteDomain()+"/fanfic/viewstory.php?sid=")+r"\d+$"
@@ -180,7 +181,7 @@ class CastleFansOrgAdapter(BaseSiteAdapter): # XXX
         pagetitle = soup.find('div',{'id':'pagetitle'})
         ## Title
         a = pagetitle.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = pagetitle.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))

fanficdownloader/adapters/adapter_chaossycophanthexcom.py

@@ -15,6 +15,7 @@
 # limitations under the License.
 #
 
+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -48,7 +49,7 @@ class ChaosSycophantHexComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
 		
         
         # normalized story URL.
@@ -67,8 +68,8 @@ class ChaosSycophantHexComAdapter(BaseSiteAdapter):
         return 'chaos.sycophanthex.com'
 
     @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
         return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
@@ -116,7 +117,7 @@ class ChaosSycophantHexComAdapter(BaseSiteAdapter):
         ## Title
         pt = soup.find('div', {'id' : 'pagetitle'})
         a = pt.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.text)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = pt.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))

fanficdownloader/adapters/adapter_checkmatedcom.py

@@ -48,7 +48,7 @@ class CheckmatedComAdapter(BaseSiteAdapter):
 
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
 
         self._setURL('http://' + self.getSiteDomain() + '/story.php?story='+self.story.getMetadata('storyId'))
 
@@ -65,8 +65,8 @@ class CheckmatedComAdapter(BaseSiteAdapter):
         return 'www.checkmated.com'
 
     @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/story.php?story=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/story.php?story=1234"
 
     def getSiteURLPattern(self):
         return re.escape("http://"+self.getSiteDomain()+"/story.php?story=")+r"\d+$"
@@ -143,11 +143,11 @@ class CheckmatedComAdapter(BaseSiteAdapter):
         # Now go hunting for all the meta data and the chapter list.
 
         ## Title
-        a = soup.findAll('span', {'class' : 'storytitle'})
-        self.story.setMetadata('title',a[0].string)
+        a = soup.find('span', {'class' : 'storytitle'})
+        self.story.setMetadata('title',stripHTML(a))
 
         # Find authorid and URL from... author url.
-        a = a[1].find('a', href=re.compile(r"authors.php\?name\=\w+"))
+        a = a.parent.find('a', href=re.compile(r"authors.php\?name\=\w+"))
         self.story.setMetadata('authorId',a['href'].split('=')[1])
         self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
         self.story.setMetadata('author',a.string)

fanficdownloader/adapters/adapter_csiforensicscom.py

@@ -0,0 +1,56 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2014 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Software: eFiction
+import re
+from base_efiction_adapter import BaseEfictionAdapter
+
+class CSIForensicsComAdapter(BaseEfictionAdapter):
+
+    @staticmethod
+    def getSiteDomain():
+        return 'csi-forensics.com'
+
+    @classmethod
+    def getPathToArchive(self):
+        return ''
+
+    @classmethod
+    def getSiteAbbrev(self):
+        return 'csiforensics'
+        
+    @classmethod
+    def getDateFormat(self):
+        return "%d %b %Y" 
+   
+    def handleMetadataPair(self, key, value):
+        if key == 'Warnings':
+            for val in re.split("\s*,\s*", value):
+                if value == 'None':
+                    return
+                else:
+                    self.story.addToList('warnings', val)
+     
+        elif 'Categories' in key:
+            for val in re.split("\s*>\s*", value):
+                self.story.addToList('category', val)
+        else:
+            super(CSIForensicsComAdapter, self).handleMetadataPair(key, value) 
+                           
+def getClass():
+    return CSIForensicsComAdapter
+    

fanficdownloader/adapters/adapter_darksolaceorg.py

@@ -15,6 +15,7 @@
 # limitations under the License.
 #
 
+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -48,7 +49,7 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/elysian/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -70,8 +71,8 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
         return ['www.dark-solace.org','dark-solace.org']
 
     @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/elysian/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/elysian/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
         return re.escape("http://")+"(www\.)?"+re.escape(self.getSiteDomain()+"/elysian/viewstory.php?sid=")+r"\d+$"
@@ -94,11 +95,10 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
         else:
             params['penname'] = self.getConfig("username")
             params['password'] = self.getConfig("password")
-        params['rememberme'] = '1'
         params['action'] = 'login'
         params['submit'] = 'Submit'
     
-        loginUrl = 'http://' + self.getSiteDomain() + '/elysian/user.php'
+        loginUrl = 'http://www.' + self.getSiteDomain() + '/elysian/user.php'
         logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['penname']))
     
@@ -183,32 +183,37 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
 
         # first a tag in pagetitle is title
         self.story.setMetadata('title',stripHTML(div.find('a')))
-
-        # Find the chapters:
-        # chapters=soup.find('select', {'name' : 'chapter'})
-        # if chapters != None:
-        #     for chapter in chapters.findAll('option'):
-        #         self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/elysian/viewstory.php?sid='+self.story.getMetadata('storyId')+'&chapter='+chapter['value']))
-        # else:
-        #     self.chapterUrls.append((self.story.getMetadata('title'),url+"&chapter=1"))
+        div.find('a').extract()
+        # only thing left in div(pagetitle) now should be 'by' and rating.
+        rating = stripHTML(div)
+        if '[' in rating:
+            self.story.setMetadata('rating', rating[rating.index('[')+1:-1])
 
         for chapa in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+
                                                        self.story.getMetadata('storyId')+'&chapter=\d+')):
             self.chapterUrls.append((stripHTML(chapa),'http://'+self.host+'/elysian/'+chapa['href']))
         
         self.story.setMetadata('numChapters',len(self.chapterUrls))
-        
+
         asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
+        storylink = asoup.find('a', href=re.compile(r'viewstory.php\?sid='+
+                                                    self.story.getMetadata('storyId')+'($|[^\d])'))
+        # author's story list is paginated if there's a pagelinks div.
+        # Only need to look in it if the story wasn't on the first page.
+        pagelinks = asoup.find('div',{'id':'pagelinks'})
+        if pagelinks and storylink==None:
+            authpageslist = pagelinks.findAll('a',href=re.compile(r'action=storiesby'))
+            for page in authpageslist[1:]: # skip first, already checked above.
+                asoup = bs.BeautifulSoup(self._fetchUrl('http://'+self.host+'/elysian/'+page['href']))
+                storylink = asoup.find('a', href=re.compile(r'viewstory.php\?sid='+
+                                                            self.story.getMetadata('storyId')+'($|[^\d])'))
+                if storylink:
+                    break
+
+        if not storylink:
+            raise exceptions.FailedToDownload("Unable to find story metadata on author's page(s)")
         
-        # for metalist in asoup.findAll('div', {'class' : re.compile('listbox\s+')}):
-        #     a = metalist.find('a', href=re.compile(r'viewstory.php\?sid='))
-        #     if a != None:
-        #         if 'viewstory.php?sid='+self.story.getMetadata('storyId') in a['href']:
-        #             break
-
-        metalist = asoup.find('a', href=re.compile(r'viewstory.php\?sid='+
-                                                   self.story.getMetadata('storyId')+'($|[^\d])')).parent.parent
-
+        metalist = storylink.parent.parent
         # eFiction sites don't help us out a lot with their meta data
         # formating, so it's a little ugly.
 
@@ -229,37 +234,34 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
             if 'Summary' in label:
                 ## Everything until the next span class='label'
                 svalue = ""
-                while not (defaultGetattr(value,'class') == 'label' or "Chapters: " in stripHTML(value)):
+                while value and not (defaultGetattr(value,'class') == 'label' or "Chapters: " in stripHTML(value)):
                     svalue += str(value)
                     value = value.nextSibling
                 self.setDescription(url,svalue)
                 #self.story.setMetadata('description',stripHTML(svalue))
 
-            if 'Rated' in label:
-                self.story.setMetadata('rating', value[:len(value)-2])
-
             if 'Word count' in label:
                 self.story.setMetadata('numWords', value)
 
             if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'categories.php\?catid=\d+'))
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
                 for cat in cats:
                     self.story.addToList('category',cat.string)
 
             if 'Characters' in label:
-                for char in value.string.split(', '):
-                    if not 'None' in char:
-                        self.story.addToList('characters',char)
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
 
             if 'Genre' in label:
-                for genre in value.string.split(', '):
-                    if not 'None' in genre:
-                        self.story.addToList('genre',genre)
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
 
             if 'Warnings' in label:
-                for warning in value.string.split(', '):
-                    if not 'None' in warning:
-                        self.story.addToList('warnings',warning)
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
 
             if 'Completed' in label:
                 if 'Yes' in value:
@@ -284,20 +286,40 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
             # use BeautifulSoup HTML parser to make everything easier to find.
             seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
             # can't use ^viewstory...$ in case of higher rated stories with javascript href.
-            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
-            i=1
-            for a in storyas:
-                # skip 'report this' and 'TOC' links
-                if 'contact.php' not in a['href'] and 'index' not in a['href']:
-                    if ('viewstory.php?sid='+self.story.getMetadata('storyId')) in a['href']:
-                        self.setSeries(series_name, i)
-                        self.story.setMetadata('seriesUrl',series_url)
+            storylink = seriessoup.find('a', href=re.compile(r'viewstory.php\?sid='+
+                                                             self.story.getMetadata('storyId')+'($|[^\d])'))
+            if storylink and storylink.parent and storylink.parent['class'] != 'title': # in case of links inside story summaries.
+                storylink = None
+
+            offset = 0
+            # series story list is paginated if there's a pagelinks div.
+            # Only need to look in it if the story wasn't on the first page.
+            pagelinks = seriessoup.find('div',{'id':'pagelinks'})
+            if pagelinks and storylink==None:
+                authpageslist = pagelinks.findAll('a',href=re.compile(r'offset='))
+                for page in authpageslist[1:]: # skip first, already checked above.
+                    seriessoup = bs.BeautifulSoup(self._fetchUrl('http://'+self.host+'/elysian/'+page['href']))
+                    storylink = seriessoup.find('a', href=re.compile(r'viewstory.php\?sid='+
+                                                                     self.story.getMetadata('storyId')+'($|[^\d])'))
+                    if storylink and storylink.parent and storylink.parent['class'] != 'title': # in case of links inside story summaries.
+                        storylink = None
+                    if storylink:
+                        offset = int(page['href'].split('=')[-1]) # offset is last.
                         break
-                    i+=1
+
+            # for reasons I don't understand, searching for story
+            # links by regex wasn't working reliably.  It was missing
+            # the javascript links sometimes.  This is cleaner anyway.
+            for i, div in enumerate(seriessoup.findAll('div', {'class':'title'})):
+                a = div.find('a') # first a is story link.
+                # skip 'report this' and 'TOC' links
+                if a == storylink:
+                    self.setSeries(series_name, 1+i+offset)
+                    self.story.setMetadata('seriesUrl',series_url)
+                    break
             
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
+        except Exception, e:
+            print("Series parsing failed: %s"%e)
             
     # grab the text for an individual chapter.
     def getChapterText(self, url):

fanficdownloader/adapters/adapter_destinysgatewaycom.py

@@ -15,6 +15,7 @@
 # limitations under the License.
 #
 
+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -48,7 +49,7 @@ class DestinysGatewayComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -66,8 +67,8 @@ class DestinysGatewayComAdapter(BaseSiteAdapter):
         return 'www.destinysgateway.com'
 
     @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
         return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
@@ -131,7 +132,7 @@ class DestinysGatewayComAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))

fanficdownloader/adapters/adapter_dokugacom.py

@@ -48,7 +48,7 @@ class DokugaComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[3])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
 
         # www.dokuga.com has two 'sections', shown in URL as
         # 'fanfiction' and 'spark' that change how things should be
@@ -76,8 +76,8 @@ class DokugaComAdapter(BaseSiteAdapter):
         return 'www.dokuga.com'
 
     @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/fanfiction/story/1234/1 http://"+self.getSiteDomain()+"/spark/story/1234/1"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/fanfiction/story/1234/1 http://"+cls.getSiteDomain()+"/spark/story/1234/1"
 
     def getSiteURLPattern(self):
         return r"http://"+self.getSiteDomain()+"/(fanfiction|spark)?/story/\d+/?\d+?$"
@@ -164,7 +164,7 @@ class DokugaComAdapter(BaseSiteAdapter):
         aut.extract()
         
         a = a.string[:(len(a.string)-4)]
-        self.story.setMetadata('title',a)
+        self.story.setMetadata('title',stripHTML(a))
 
         # Find the chapters:
         chapters = soup.find('select').findAll('option')
@@ -205,7 +205,6 @@ class DokugaComAdapter(BaseSiteAdapter):
             
             a=div.text.split('Category: ')
             if len(a) == 2: self.story.addToList('category', a[1].split('&')[0])
-            self.story.addToList('category', 'Fanfiction')
         
             a=div.text.split('Created: ')
             if len(a) == 2: self.story.setMetadata('datePublished', makeDate(stripHTML(a[1].split('&')[0]), self.dateformat))

fanficdownloader/adapters/adapter_dotmoonnet.py

@@ -48,7 +48,7 @@ class DotMoonNetAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL. www.dotmoon.net/library_view.php?storyid=3
         self._setURL('http://' + self.getSiteDomain() + '/library_view.php?storyid='+self.story.getMetadata('storyId'))
@@ -66,8 +66,8 @@ class DotMoonNetAdapter(BaseSiteAdapter):
         return 'www.dotmoon.net'
 
     @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/library_view.php?storyid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/library_view.php?storyid=1234"
 
     def getSiteURLPattern(self):
         return re.escape("http://"+self.getSiteDomain()+"/library_view.php?storyid=")+r"\d+$"
@@ -141,7 +141,7 @@ class DotMoonNetAdapter(BaseSiteAdapter):
         
         ## Title
         a = body.find('b')
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url. http://www.dotmoon.net/board/index.php?action=profile;u=1'
         a = body.find('a', href=re.compile(r"index.php\?action=profile;u=\d+"))

fanficdownloader/adapters/adapter_dracoandginnycom.py

@@ -15,6 +15,7 @@
 # limitations under the License.
 #
 
+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -48,7 +49,7 @@ class DracoAndGinnyComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -66,8 +67,8 @@ class DracoAndGinnyComAdapter(BaseSiteAdapter):
         return 'www.dracoandginny.com'
 
     @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
         return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
@@ -170,7 +171,7 @@ class DracoAndGinnyComAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))

fanficdownloader/adapters/adapter_dramioneorg.py

@@ -15,6 +15,7 @@
 # limitations under the License.
 #
 
+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -48,7 +49,7 @@ class DramioneOrgAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -66,8 +67,8 @@ class DramioneOrgAdapter(BaseSiteAdapter):
         return 'dramione.org'
 
     @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
         return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
@@ -154,7 +155,7 @@ class DramioneOrgAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))

fanficdownloader/adapters/adapter_efictionestelielde.py

@@ -15,6 +15,7 @@
 # limitations under the License.
 #
 
+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -48,7 +49,7 @@ class EfictionEstelielDeAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -66,8 +67,8 @@ class EfictionEstelielDeAdapter(BaseSiteAdapter):
         return 'efiction.esteliel.de'
 
     @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
         return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
@@ -104,7 +105,7 @@ class EfictionEstelielDeAdapter(BaseSiteAdapter):
         pagetitle = soup.find('div',{'id':'pagetitle'})
         ## Title
         a = pagetitle.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = pagetitle.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))

fanficdownloader/adapters/adapter_efpfanficnet.py

@@ -15,6 +15,7 @@
 # limitations under the License.
 #
 
+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -48,7 +49,7 @@ class EFPFanFicNet(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -66,15 +67,16 @@ class EFPFanFicNet(BaseSiteAdapter):
         return 'www.efpfanfic.net'
 
     @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
         return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
 
     ## Login seems to be reasonably standard across eFiction sites.
     def needToLoginCheck(self, data):
-        if 'Fai il login e leggi la storia!' in data:
+        if( 'Fai il login e leggi la storia!' in data or
+            'Questa storia presenta contenuti non adatti ai minori' in data ):
             return True
         else:
             return False
@@ -166,7 +168,7 @@ class EFPFanFicNet(BaseSiteAdapter):
         logger.debug("Normalizing to URL: "+url)
         self._setURL(url)
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
 
         # eFiction sites don't help us out a lot with their meta data
         # formating, so it's a little ugly.
@@ -203,8 +205,8 @@ class EFPFanFicNet(BaseSiteAdapter):
         noteblock = storyblock.find('div', {'class':'notebloc'})
         #print("%s"%noteblock)
         notetext = ("%s" % noteblock).replace("<br />"," |")
-        # <div class="notebloc">Autore: <a href="viewuser.php?uid=243036">Cendrillon89</a> | Pubblicata: 23/10/12 | Aggiornata: 30/10/12 | Rating: Arancione | Genere: Drammatico, Sentimentale | Capitoli: 10 | Completa<br />␍
-        # Tipo di coppia: Het |  Personaggi: Akasuna no Sasori , Akatsuki, Nuovo Personaggio |   Note: OOC | Avvertimenti: Tematiche delicate<br />␍
+        # <div class="notebloc">Autore: <a href="viewuser.php?uid=243036">Cendrillon89</a> | Pubblicata: 23/10/12 | Aggiornata: 30/10/12 | Rating: Arancione | Genere: Drammatico, Sentimentale | Capitoli: 10 | Completa<br />
+        # Tipo di coppia: Het |  Personaggi: Akasuna no Sasori , Akatsuki, Nuovo Personaggio |   Note: OOC | Avvertimenti: Tematiche delicate<br />
         # Categoria: <a href="categories.php?catid=1&amp;parentcatid=1">Anime & Manga</a> > <a href="categories.php?catid=108&amp;parentcatid=108">Naruto</a> | Contesto: Naruto Shippuuden | Leggi le <a href="reviews.php?sid=1331275&amp;a=">3</a> recensioni</div>
 
         cats = noteblock.findAll('a',href=re.compile(r'browse.php\?type=categories'))

fanficdownloader/adapters/adapter_erosnsapphosycophanthexcom.py

@@ -15,6 +15,7 @@
 # limitations under the License.
 #
 
+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -48,7 +49,7 @@ class ErosnSapphoSycophantHexComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
 		
         
         # normalized story URL.
@@ -67,8 +68,8 @@ class ErosnSapphoSycophantHexComAdapter(BaseSiteAdapter):
         return 'erosnsappho.sycophanthex.com'
 
     @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
         return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
@@ -132,7 +133,7 @@ class ErosnSapphoSycophantHexComAdapter(BaseSiteAdapter):
         ## Title
         pt = soup.find('div', {'id' : 'pagetitle'})
         a = pt.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.text)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = pt.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))

fanficdownloader/adapters/adapter_fanfichu.py

@@ -0,0 +1,185 @@
+# coding=utf-8
+
+import re
+import urllib2
+import urlparse
+
+from .. import BeautifulSoup
+
+from base_adapter import BaseSiteAdapter, makeDate
+from .. import exceptions
+
+
+_SOURCE_CODE_ENCODING = 'utf-8'
+
+
+def getClass():
+    return FanficHuAdapter
+
+
+def _get_query_data(url):
+    components = urlparse.urlparse(url)
+    query_data = urlparse.parse_qs(components.query)
+    return dict((key, data[0]) for key, data in query_data.items())
+
+
+class FanficHuAdapter(BaseSiteAdapter):
+    SITE_ABBREVIATION = 'ffh'
+    SITE_DOMAIN = 'fanfic.hu'
+    SITE_LANGUAGE = 'Hungarian'
+
+    BASE_URL = 'http://' + SITE_DOMAIN + '/merengo/'
+    VIEW_STORY_URL_TEMPLATE = BASE_URL + 'viewstory.php?sid=%s'
+
+    DATE_FORMAT = '%m/%d/%Y'
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        query_data = urlparse.parse_qs(self.parsedUrl.query)
+        story_id = query_data['sid'][0]
+
+        self.story.setMetadata('storyId', story_id)
+        self._setURL(self.VIEW_STORY_URL_TEMPLATE % story_id)
+        self.story.setMetadata('siteabbrev', self.SITE_ABBREVIATION)
+        self.story.setMetadata('language', self.SITE_LANGUAGE)
+
+    def _customized_fetch_url(self, url, exception=None, parameters=None):
+        if exception:
+            try:
+                data = self._fetchUrl(url, parameters)
+            except urllib2.HTTPError:
+                raise exception(self.url)
+        # Just let self._fetchUrl throw the exception, don't catch and
+        # customize it.
+        else:
+            data = self._fetchUrl(url, parameters)
+
+        return BeautifulSoup.BeautifulSoup(data)
+
+    @staticmethod
+    def getSiteDomain():
+        return FanficHuAdapter.SITE_DOMAIN
+
+    @classmethod
+    def getSiteExampleURLs(cls):
+        return cls.VIEW_STORY_URL_TEMPLATE % 1234
+
+    def getSiteURLPattern(self):
+        return re.escape(self.VIEW_STORY_URL_TEMPLATE[:-2]) + r'\d+$'
+
+    def extractChapterUrlsAndMetadata(self):
+        soup = self._customized_fetch_url(self.url + '&i=1')
+
+        if soup.title.string.encode(_SOURCE_CODE_ENCODING).strip(' :') == 'írta':
+            raise exceptions.StoryDoesNotExist(self.url)
+
+        chapter_options = soup.find('form', action='viewstory.php').select('option')
+        # Remove redundant "Fejezetek" option
+        chapter_options.pop(0)
+
+        # If there is still more than one entry remove chapter overview entry
+        if len(chapter_options) > 1:
+            chapter_options.pop(0)
+
+        for option in chapter_options:
+            url = urlparse.urljoin(self.url, option['value'])
+            self.chapterUrls.append((option.string, url))
+
+        author_url = urlparse.urljoin(self.BASE_URL, soup.find('a', href=lambda href: href and href.startswith('viewuser.php?uid='))['href'])
+        soup = self._customized_fetch_url(author_url)
+
+        story_id = self.story.getMetadata('storyId')
+        for table in soup('table', {'class': 'mainnav'}):
+            title_anchor = table.find('span', {'class': 'storytitle'}).a
+            href = title_anchor['href']
+            if href.startswith('javascript:'):
+                href = href.rsplit(' ', 1)[1].strip("'")
+            query_data = _get_query_data(href)
+
+            if query_data['sid'] == story_id:
+                break
+        else:
+            # This should never happen, the story must be found on the author's
+            # page.
+            raise exceptions.FailedToDownload(self.url)
+
+        self.story.setMetadata('title', title_anchor.string)
+
+        rows = table('tr')
+
+        anchors = rows[0].div('a')
+        author_anchor = anchors[1]
+        query_data = _get_query_data(author_anchor['href'])
+        self.story.setMetadata('author', author_anchor.string)
+        self.story.setMetadata('authorId', query_data['uid'])
+        self.story.setMetadata('authorUrl', urlparse.urljoin(self.BASE_URL, author_anchor['href']))
+        self.story.setMetadata('reviews', anchors[3].string)
+
+        if self.getConfig('keep_summary_html'):
+            self.story.setMetadata('description', self.utf8FromSoup(author_url, rows[1].td))
+        else:
+            self.story.setMetadata('description', ''.join(rows[1].td(text=True)))
+
+        for row in rows[3:]:
+            index = 0
+            cells = row('td')
+
+            while index < len(cells):
+                cell = cells[index]
+                key = cell.b.string.encode(_SOURCE_CODE_ENCODING).strip(':')
+                try:
+                    value = cells[index+1].string.encode(_SOURCE_CODE_ENCODING)
+                except AttributeError:
+                    value = None
+
+                if key == 'Kategória':
+                    for anchor in cells[index+1]('a'):
+                        self.story.addToList('category', anchor.string)
+
+                elif key == 'Szereplõk':
+                    if cells[index+1].string:
+                        for name in cells[index+1].string.split(', '):
+                            self.story.addToList('character', name)
+
+                elif key == 'Korhatár':
+                    if value != 'nem korhatáros':
+                        self.story.setMetadata('rating', value)
+
+                elif key == 'Figyelmeztetések':
+                    for b_tag in cells[index+1]('b'):
+                        self.story.addToList('warnings', b_tag.string)
+
+                elif key == 'Jellemzõk':
+                    for genre in cells[index+1].string.split(', '):
+                        self.story.addToList('genre', genre)
+
+                elif key == 'Fejezetek':
+                    self.story.setMetadata('numChapters', int(value))
+
+                elif key == 'Megjelenés':
+                    self.story.setMetadata('datePublished', makeDate(value, self.DATE_FORMAT))
+
+                elif key == 'Frissítés':
+                    self.story.setMetadata('dateUpdated', makeDate(value, self.DATE_FORMAT))
+
+                elif key == 'Szavak':
+                    self.story.setMetadata('numWords', value)
+
+                elif key == 'Befejezett':
+                    self.story.setMetadata('status', 'Completed' if value == 'Nem' else 'In-Progress')
+
+                index += 2
+
+        if self.story.getMetadata('rating') == '18':
+            if not (self.is_adult or self.getConfig('is_adult')):
+                raise exceptions.AdultCheckRequired(self.url)
+
+    def getChapterText(self, url):
+        soup = self._customized_fetch_url(url)
+        story_cell = soup.find('form', action='viewstory.php').parent.parent
+
+        for div in story_cell('div'):
+            div.extract()
+
+        return self.utf8FromSoup(url, story_cell)

fanficdownloader/adapters/adapter_fanfictioncsodaidokhu.py

@@ -0,0 +1,218 @@
+# coding=utf-8
+
+import re
+import urllib2
+import urlparse
+
+from .. import BeautifulSoup
+
+from base_adapter import BaseSiteAdapter, makeDate
+from .. import exceptions
+
+
+_SOURCE_CODE_ENCODING = 'utf-8'
+
+
+def getClass():
+    return FanfictionCsodaidokHuAdapter
+
+
+def _get_query_data(url):
+    components = urlparse.urlparse(url)
+    query_data = urlparse.parse_qs(components.query)
+    return dict((key, data[0]) for key, data in query_data.items())
+
+
+# yields Tag _and_ NavigableString siblings from the given tag. The
+# BeautifulSoup findNextSiblings() method for some reasons only returns either
+# NavigableStrings _or_ Tag objects, not both.
+def _yield_next_siblings(tag):
+    sibling = tag.nextSibling
+    while sibling:
+        yield sibling
+        sibling = sibling.nextSibling
+
+
+class FanfictionCsodaidokHuAdapter(BaseSiteAdapter):
+    _SITE_DOMAIN = 'fanfiction.csodaidok.hu'
+    _BASE_URL = 'http://' + _SITE_DOMAIN + '/'
+    _VIEW_STORY_URL_TEMPLATE = _BASE_URL + 'viewstory.php?sid=%s'
+    _VIEW_CHAPTER_URL_TEMPLATE = _VIEW_STORY_URL_TEMPLATE + '&chapter=%s'
+
+    _STORY_DOES_NOT_EXIST_PAGE_TITLE = 'Cím:  Szerző:'
+    _DATE_FORMAT = '%Y.%m.%d'
+    _SITE_LANGUAGE = 'Hungarian'
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        query_data = urlparse.parse_qs(self.parsedUrl.query)
+        story_id = query_data['sid'][0]
+
+        self.story.setMetadata('storyId', story_id)
+        self._setURL(self._VIEW_STORY_URL_TEMPLATE % story_id)
+        self.story.setMetadata('siteabbrev', self._SITE_DOMAIN)
+        self.story.setMetadata('language', self._SITE_LANGUAGE)
+
+    def _customized_fetch_url(self, url, exception=None, parameters=None):
+        if exception:
+            try:
+                data = self._fetchUrl(url, parameters)
+            except urllib2.HTTPError:
+                raise exception(self.url)
+        # Just let self._fetchUrl throw the exception, don't catch and
+        # customize it.
+        else:
+            data = self._fetchUrl(url, parameters)
+
+        return BeautifulSoup.BeautifulSoup(data)
+
+    @staticmethod
+    def getSiteDomain():
+        return FanfictionCsodaidokHuAdapter._SITE_DOMAIN
+
+    @classmethod
+    def getSiteExampleURLs(cls):
+        return cls._VIEW_STORY_URL_TEMPLATE % 1234
+
+    def getSiteURLPattern(self):
+        return re.escape(self._VIEW_STORY_URL_TEMPLATE[:-2]) + r'\d+$'
+
+    def extractChapterUrlsAndMetadata(self):
+        soup = self._customized_fetch_url(self.url + '&chapter=1')
+
+        element = soup.find('div', id='pagetitle')
+        page_title = ''.join(element(text=True)).encode(_SOURCE_CODE_ENCODING)
+        if page_title == self._STORY_DOES_NOT_EXIST_PAGE_TITLE:
+            raise exceptions.StoryDoesNotExist(self.url)
+
+        author_url = urlparse.urljoin(self.url, element.a['href'])
+
+        story_id = self.story.getMetadata('storyId')
+        element = soup.find('select', {'name': 'chapter'})
+        if element:
+            for option in element('option'):
+                title = option.string
+                url = self._VIEW_CHAPTER_URL_TEMPLATE % (story_id, option['value'])
+                self.chapterUrls.append((title, url))
+
+        soup = self._customized_fetch_url(author_url)
+        story_id = self.story.getMetadata('storyId')
+
+        for listbox_div in soup('div', {'class': lambda klass: klass and 'listbox' in klass}):
+            a = listbox_div.div.a
+            if not a['href'].startswith('viewstory.php?sid='):
+                continue
+
+            query_data = _get_query_data(a['href'])
+            if query_data['sid'] == story_id:
+                break
+        else:
+            raise exceptions.FailedToDownload(self.url)
+
+        title = ''.join(a(text=True))
+        self.story.setMetadata('title', title)
+        if not self.chapterUrls:
+            self.chapterUrls.append((title, self.url))
+
+        element = a.findNextSibling('a')
+        self.story.setMetadata('author', element.string)
+        query_data = _get_query_data(element['href'])
+        self.story.setMetadata('authorId', query_data['uid'])
+        self.story.setMetadata('authorUrl', author_url)
+
+        element = element.findNextSibling('span')
+        rating = element.nextSibling.strip(' [')
+
+        if rating.encode(_SOURCE_CODE_ENCODING) != 'Korhatár nélkül':
+            self.story.setMetadata('rating', rating)
+
+        if rating == '18':
+            raise exceptions.AdultCheckRequired(self.url)
+
+        element = element.findNextSiblings('a')[1]
+        self.story.setMetadata('reviews', element.string)
+
+        sections = listbox_div('div', {'class': lambda klass: klass and klass in ['content', 'tail']})
+        for section in sections:
+            for element in section('span', {'class': 'classification'}):
+                key = element.string.encode(_SOURCE_CODE_ENCODING).strip(' :')
+                try:
+                    value = element.nextSibling.string.encode(_SOURCE_CODE_ENCODING).strip()
+                except AttributeError:
+                    value = None
+
+                if key == 'Tartalom':
+                    contents = []
+                    keep_summary_html = self.getConfig('keep_summary_html')
+
+                    for sibling in _yield_next_siblings(element):
+                        if isinstance(sibling, BeautifulSoup.Tag):
+                            if sibling.name == 'span' and sibling.get('class', None) == 'classification':
+                                break
+
+                            if keep_summary_html:
+                                contents.append(self.utf8FromSoup(author_url, sibling))
+                            else:
+                                contents.append(''.join(sibling(text=True)))
+                        else:
+                            contents.append(sibling)
+                    self.story.setMetadata('description', ''.join(contents))
+
+                elif key == 'Kategória':
+                    for sibling in element.findNextSiblings(['a', 'span']):
+                        if sibling.name == 'span':
+                            break
+
+                        self.story.addToList('category', sibling.string)
+
+                elif key == 'Szereplők':
+                    for name in value.split(', '):
+                        self.story.addToList('characters', name)
+
+                elif key == 'Műfaj':
+                    if value != 'Nincs':
+                        self.story.setMetadata('genre', value)
+
+                elif key == 'Figyelmeztetés':
+                    if value != 'Nincs':
+                        for warning in value.split(', '):
+                            self.story.addToList('warnings', warning)
+
+                elif key == 'Kihívás':
+                    if value != 'Nincs':
+                        self.story.setMetadata('challenge', value)
+
+                elif key == 'Sorozat':
+                    if value != 'Nincs':
+                        self.story.setMetadata('series', value)
+
+                elif key == 'Fejezetek':
+                    self.story.setMetadata('numChapters', int(value))
+
+                elif key == 'Befejezett':
+                    self.story.setMetadata('status', 'Completed' if value == 'Nem' else 'In-Progress')
+
+                elif key == 'Szavak száma':
+                    self.story.setMetadata('numWords', value)
+
+                elif key == 'Feltöltve':
+                    self.story.setMetadata('datePublished', makeDate(value, self._DATE_FORMAT))
+
+                elif key == 'Frissítve':
+                    self.story.setMetadata('dateUpdated', makeDate(value, self._DATE_FORMAT))
+
+    def getChapterText(self, url):
+        soup = self._customized_fetch_url(url)
+        contents = []
+
+        notes_div = soup.find('div', id='notes')
+        if notes_div:
+            contents.append(self.utf8FromSoup(url, notes_div))
+            story_div = notes_div.findNextSibling('div')
+        else:
+            element = soup.find('div', {'class': 'jumpmenu'})
+            story_div = element.findNextSibling('div')
+
+        contents.append(self.utf8FromSoup(url, story_div.span))
+        return ''.join(contents)

fanficdownloader/adapters/adapter_fanfictionnet.py

@@ -16,6 +16,7 @@
 #
 
 import time
+from datetime import datetime
 import logging
 logger = logging.getLogger(__name__)
 import re
@@ -43,7 +44,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
 
         # normalized story URL.
-        self._setURL("http://"+self.getSiteDomain()\
+        self._setURL("https://"+self.getSiteDomain()\
                          +"/s/"+self.story.getMetadata('storyId')+"/1/")
 
         # ffnet update emails have the latest chapter URL.
@@ -51,10 +52,14 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         # latest chapter yet and going back to chapter 1 to pull the
         # chapter list doesn't get the latest.  So save and use the
         # original URL given to pull chapter list & metadata.
+        # Not used by plugin because URL gets normalized first for
+        # eliminating duplicate story urls.
         self.origurl = url
-        if "http://m." in self.origurl:
+        if "https://m." in self.origurl:
             ## accept m(mobile)url, but use www.
-            self.origurl = self.origurl.replace("http://m.","http://www.")
+            self.origurl = self.origurl.replace("https://m.","https://www.")
+
+        self.opener.addheaders.append(('Referer',self.origurl))
 
     @staticmethod
     def getSiteDomain():
@@ -65,13 +70,29 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         return ['www.fanfiction.net','m.fanfiction.net']
 
     @classmethod
-    def getSiteExampleURLs(self):
-        return "http://www.fanfiction.net/s/1234/1/ http://www.fanfiction.net/s/1234/12/ http://www.fanfiction.net/s/1234/1/Story_Title http://m.fanfiction.net/s/1234/1/"
+    def getSiteExampleURLs(cls):
+        return "https://www.fanfiction.net/s/1234/1/ https://www.fanfiction.net/s/1234/12/ http://www.fanfiction.net/s/1234/1/Story_Title http://m.fanfiction.net/s/1234/1/"
 
     def getSiteURLPattern(self):
-        return r"http://(www|m)?\.fanfiction\.net/s/\d+(/\d+)?(/|/[^/]+)?/?$"
+        return r"https?://(www|m)?\.fanfiction\.net/s/\d+(/\d+)?(/|/[^/]+)?/?$"
 
-    def extractChapterUrlsAndMetadata(self):
+    def _fetchUrl(self,url,parameters=None,extrasleep=1.0):
+        # time.sleep(1.0) ## ffnet(and, I assume, fpcom) tends to fail
+        #                 ## more if hit too fast.  This is in
+        #                 ## additional to what ever the
+        #                 ## slow_down_sleep_time setting is.
+        return BaseSiteAdapter._fetchUrl(self,url,
+                                         parameters=parameters,
+                                         extrasleep=extrasleep)
+
+    def use_pagecache(self):
+        '''
+        adapters that will work with the page cache need to implement
+        this and change it to True.
+        '''
+        return True
+    
+    def doExtractChapterUrlsAndMetadata(self,get_cover=True):
 
         # fetch the chapter.  From that we will get almost all the
         # metadata and chapter list
@@ -97,32 +118,35 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         if "not found. Please check to see you are not using an outdated url." in data:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  'Chapter not found. Please check to see you are not using an outdated url.'" % url)
 
-        try:
-            # rather nasty way to check for a newer chapter.  ffnet has a
-            # tendency to send out update notices in email before all
-            # their servers are showing the update on the first chapter.
+        if self.getConfig('check_next_chapter'):
             try:
-                chapcount = len(soup.find('select', { 'name' : 'chapter' } ).findAll('option'))
-            # get chapter part of url.
+                ## ffnet used to have a tendency to send out update
+                ## notices in email before all their servers were
+                ## showing the update on the first chapter.  It
+                ## generates another server request and doesn't seem
+                ## to be needed lately, so now default it to off.
+                try:
+                    chapcount = len(soup.find('select', { 'name' : 'chapter' } ).findAll('option'))
+                # get chapter part of url.
+                except:
+                    chapcount = 1
+                chapter = url.split('/',)[5]
+                tryurl = "https://%s/s/%s/%d/"%(self.getSiteDomain(),
+                                                self.story.getMetadata('storyId'),
+                                                chapcount+1)
+                logger.debug('=Trying newer chapter: %s' % tryurl)
+                newdata = self._fetchUrl(tryurl)
+                if "not found. Please check to see you are not using an outdated url." \
+                        not in newdata:
+                    logger.debug('=======Found newer chapter: %s' % tryurl)
+                    soup = bs.BeautifulSoup(newdata)
             except:
-                chapcount = 1
-            chapter = url.split('/',)[5]
-            tryurl = "http://%s/s/%s/%d/"%(self.getSiteDomain(),
-                                           self.story.getMetadata('storyId'),
-                                           chapcount+1)
-            logger.debug('=Trying newer chapter: %s' % tryurl)
-            newdata = self._fetchUrl(tryurl)
-            if "not found. Please check to see you are not using an outdated url." \
-                    not in newdata:
-                logger.debug('=======Found newer chapter: %s' % tryurl)
-                soup = bs.BeautifulSoup(newdata)
-        except:
-            pass
-        
+                pass
+            
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"^/u/\d+"))
         self.story.setMetadata('authorId',a['href'].split('/')[2])
-        self.story.setMetadata('authorUrl','http://'+self.host+a['href'])
+        self.story.setMetadata('authorUrl','https://'+self.host+a['href'])
         self.story.setMetadata('author',a.string)
 
         ## Pull some additional data from html.
@@ -141,9 +165,9 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
             # of Book, Movie, etc.
             self.story.addToList('category',stripHTML(categories[1]))
         elif 'Crossover' in categories[0]['href']:
-            caturl = "http://%s%s"%(self.getSiteDomain(),categories[0]['href'])
+            caturl = "https://%s%s"%(self.getSiteDomain(),categories[0]['href'])
             catsoup = bs.BeautifulSoup(self._fetchUrl(caturl))
-            for a in catsoup.findAll('a',href=re.compile(r"^/crossovers/")):
+            for a in catsoup.findAll('a',href=re.compile(r"^/crossovers/.+?/\d+/")):
                 self.story.addToList('category',stripHTML(a))
             else:
                 # Fall back.  I ran across a story with a Crossver
@@ -156,7 +180,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
                 
                 
             
-        a = soup.find('a', href='http://www.fictionratings.com/')
+        a = soup.find('a', href=re.compile(r'https?://www\.fictionratings\.com/'))
         rating = a.string
         if 'Fiction' in rating: # if rating has 'Fiction ', strip that out for consistency with past.
             rating = rating[8:]
@@ -165,7 +189,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
 
         # after Rating, the same bit of text containing id:123456 contains
         # Complete--if completed.
-        gui_table1i = soup.find('table',{'cellpadding':'5'})
+        gui_table1i = soup.find('div',{'id':'content_wrapper_inner'})
 
         self.story.setMetadata('title', stripHTML(gui_table1i.find('b'))) # title appears to be only(or at least first) bold tag in gui_table1i
 
@@ -199,15 +223,23 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         for g in genrelist:
             #logger.debug("g:(%s)"%g)
             if g.strip() not in ffnetgenres:
-                logger.info("g not in ffnetgenres")
+                #logger.info("g not in ffnetgenres")
                 goodgenres=False
         if goodgenres:
             self.story.extendList('genre',genrelist)
             metalist=metalist[1:]
 
+        # Updated: <span data-xutime='1368059198'>5/8</span> - Published: <span data-xutime='1278984264'>7/12/2010</span>
+        # Published: <span data-xutime='1384358726'>8m ago</span>
+        dates = soup.findAll('span',{'data-xutime':re.compile(r'^\d+$')})
+        if len(dates) > 1 :
+            # updated get set to the same as published upstream if not found.
+            self.story.setMetadata('dateUpdated',datetime.fromtimestamp(float(dates[0]['data-xutime'])))
+        self.story.setMetadata('datePublished',datetime.fromtimestamp(float(dates[-1]['data-xutime'])))
+            
         donechars = False
         while len(metalist) > 0:
-            if  metalist[0].startswith('Chapters') or metalist[0].startswith('Status') or metalist[0].startswith('id:'):
+            if  metalist[0].startswith('Chapters') or metalist[0].startswith('Status') or metalist[0].startswith('id:') or metalist[0].startswith('Updated:') or metalist[0].startswith('Published:'):
                 pass
             elif  metalist[0].startswith('Reviews'):
                 self.story.setMetadata('reviews',metalist[0].split(':')[1].strip())
@@ -215,34 +247,35 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
                 self.story.setMetadata('favs',metalist[0].split(':')[1].strip())
             elif  metalist[0].startswith('Follows:'):
                 self.story.setMetadata('follows',metalist[0].split(':')[1].strip())
-            elif  metalist[0].startswith('Updated'):
-                self.story.setMetadata('dateUpdated',makeDate(metalist[0].split(':')[1].strip(), '%m-%d-%y'))
-            elif  metalist[0].startswith('Published'):
-                self.story.setMetadata('datePublished',makeDate(metalist[0].split(':')[1].strip(), '%m-%d-%y'))
             elif  metalist[0].startswith('Words'):
                 self.story.setMetadata('numWords',metalist[0].split(':')[1].strip())
             elif not donechars:
-                self.story.extendList('characters',metalist[0].split('&'))
+                # with 'pairing' support, pairings are bracketed w/o comma after
+                # [Caspian X, Lucy Pevensie] Edmund Pevensie, Peter Pevensie
+                self.story.extendList('characters',metalist[0].replace('[','').replace(']',',').split(','))
+
+                l = metalist[0]
+                while '[' in l:
+                    self.story.addToList('ships',l[l.index('[')+1:l.index(']')].replace(', ','/'))
+                    l = l[l.index(']')+1:]
+
                 donechars = True
             metalist=metalist[1:]
 
-        # next might be characters, otherwise Reviews, Updated, Published, Words
-        # if not ( metalist[0].startswith('Reviews') or metalist[0].startswith('Updated') or metalist[0].startswith('Published') or metalist[0].startswith('Words') or metalist[0].startswith('Chapters') ):
-        #     self.story.extendList('characters',metalist[0].split('&'))
-        
         if 'Status: Complete' in metatext:
             self.story.setMetadata('status', 'Completed')
         else:
             self.story.setMetadata('status', 'In-Progress')
 
-        # Try the larger image first.
-        try:
-            img = soup.find('img',{'class':'lazy cimage'})
-            self.setCoverImage(url,img['data-original'])
-        except:
-            img = soup.find('img',{'class':'cimage'})
-            if img:
-                self.setCoverImage(url,img['src'])
+        if get_cover:
+            # Try the larger image first.
+            try:
+                img = soup.find('img',{'class':'lazy cimage'})
+                self.setCoverImage(url,img['data-original'])
+            except:
+                img = soup.find('img',{'class':'cimage'})
+                if img:
+                    self.setCoverImage(url,img['src'])
             
         # Find the chapter selector 
         select = soup.find('select', { 'name' : 'chapter' } )
@@ -253,9 +286,9 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         else:
             allOptions = select.findAll('option')
             for o in allOptions:
-                url = u'http://%s/s/%s/%s/' % ( self.getSiteDomain(),
-                                            self.story.getMetadata('storyId'),
-                                            o['value'])
+                url = u'https://%s/s/%s/%s/' % ( self.getSiteDomain(),
+                                                 self.story.getMetadata('storyId'),
+                                                 o['value'])
                 # just in case there's tags, like <i> in chapter titles.
                 title = u"%s" % o
                 title = re.sub(r'<[^>]+>','',title)
@@ -266,12 +299,12 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         return
 
     def getChapterText(self, url):
+        # time.sleep(4.0) ## ffnet(and, I assume, fpcom) tends to fail
+        #                 ## more if hit too fast.  This is in
+        #                 ## additional to what ever the
+        #                 ## slow_down_sleep_time setting is.
         logger.debug('Getting chapter text from: %s' % url)
-        time.sleep(0.5) ## ffnet(and, I assume, fpcom) tends to fail
-                        ## more if hit too fast.  This is in
-                        ## additional to what ever the
-                        ## slow_down_sleep_time setting is.
-        data = self._fetchUrl(url)
+        data = self._fetchUrl(url,extrasleep=4.0)
 
         if "Please email this error message in full to <a href='mailto:support@fanfiction.com'>support@fanfiction.com</a>" in data:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  FanFiction.net Site Error!" % url)
@@ -281,10 +314,11 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         # don't care about anything before "<div class='storytextp"
         # (there's a space after storytextp, so no close quote(')) and
         # this kills any body tags.
-        if "<div class='storytextp" not in data:
+        divstr = "<div role='main'"
+        if divstr not in data:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
         else:
-            data = data[data.index("<div class='storytextp"):]
+            data = data[data.index(divstr):]
         data.replace("<body","<notbody").replace("<BODY","<NOTBODY")
         
         soup = bs.BeautifulSoup(data)

fanficdownloader/adapters/adapter_fanfiktionde.py

@@ -50,7 +50,7 @@ class FanFiktionDeAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/s/'+self.story.getMetadata('storyId') + '/1')
@@ -68,12 +68,19 @@ class FanFiktionDeAdapter(BaseSiteAdapter):
         return 'www.fanfiktion.de'
 
     @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/s/46ccbef30000616306614050"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/s/46ccbef30000616306614050 http://"+cls.getSiteDomain()+"/s/46ccbef30000616306614050/1 http://"+cls.getSiteDomain()+"/s/46ccbef30000616306614050/1/story-name"
 
     def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/s/")+r"\w+(/\d+)?$"
+        return re.escape("http://"+self.getSiteDomain()+"/s/")+r"\w+(/\d+)?"
         
+    def use_pagecache(self):
+        '''
+        adapters that will work with the page cache need to implement
+        this and change it to True.
+        '''
+        return True
+    
         ## Login seems to be reasonably standard across eFiction sites.
     def needToLoginCheck(self, data):
         if 'Diese Geschichte wurde als entwicklungsbeeintr' in data \
@@ -126,7 +133,7 @@ class FanFiktionDeAdapter(BaseSiteAdapter):
         if self.needToLoginCheck(data):
             # need to log in for this one.
             self.performLogin(url)
-            data = self._fetchUrl(url)
+            data = self._fetchUrl(url,usecache=False)
             
         if "Uhr ist diese Geschichte nur nach einer" in data:
             raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Auserhalb der Zeit von 23:00 Uhr bis 04:00 Uhr ist diese Geschichte nur nach einer erfolgreichen Altersverifikation zuganglich.")
@@ -138,15 +145,15 @@ class FanFiktionDeAdapter(BaseSiteAdapter):
         # Now go hunting for all the meta data and the chapter list.
         
         ## Title
-        a = soup.find('a', href=re.compile(r'/s/'+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        a = soup.find('a', href=re.compile(r'/s/'+self.story.getMetadata('storyId')+"/"))
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
-        head = soup.find('div', {'class' : 'story-metadata-left-top'})
+        head = soup.find('div', {'class' : 'story-left'})
         a = head.find('a')
         self.story.setMetadata('authorId',a['href'].split('/')[2])
         self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
+        self.story.setMetadata('author',stripHTML(a))
 
         # Find the chapters:
         for chapter in soup.find('select').findAll('option'):
@@ -156,14 +163,18 @@ class FanFiktionDeAdapter(BaseSiteAdapter):
         self.story.setMetadata('language','German')
 
         #find metadata on the story page
-        self.story.setMetadata('datePublished', makeDate(head.text.split('erstellt: ')[1].split('\n')[0], self.dateformat))
+        headtext = stripHTML(head)
+        self.story.setMetadata('datePublished', makeDate(headtext.split('erstellt: ')[1].split('\n')[0], self.dateformat))
         
-        self.story.setMetadata('dateUpdated', makeDate(head.text.split('letztes Update: ')[1].split('\n')[0], self.dateformat))
-        
-        for genre in head.text.split('   ')[3].split('/')[0].split(', '):
-            self.story.addToList('genre',genre)
+        self.story.setMetadata('dateUpdated', makeDate(headtext.split('aktualisiert: ')[1].split('\n')[0], self.dateformat))
+
+        # second colspan=3 td in head.
+        genres=stripHTML(head.findAll('td',{'colspan':'3'})[1])
+        self.story.extendList('genre',genres[:genres.index('(')].split(', '))
+        # for genre in head.text.split('   ')[3].split('/')[0].split(', '):
+        #     self.story.addToList('genre',genre)
             
-        if 'fertiggestellt' in head.text:
+        if 'fertiggestellt' in headtext:
             self.story.setMetadata('status', 'Completed')
         else:
             self.story.setMetadata('status', 'In Progress')
@@ -173,14 +184,14 @@ class FanFiktionDeAdapter(BaseSiteAdapter):
         tr=asoup.findAll('tr')
         for i in range(1,len(tr)):
             a = tr[i].find('a')
-            if a['href'] == '/s/'+self.story.getMetadata('storyId'):
+            if '/s/'+self.story.getMetadata('storyId')+'/1/' in a['href']:
                 break
         self.setDescription(url,a['onmouseover'].split("', '")[1])
         
         td = tr[i].findAll('td')
-        self.story.addToList('category',stripHTML(td[1]))
-        self.story.setMetadata('rating', td[4].string)
-        self.story.setMetadata('numWords', td[5].string)
+        self.story.addToList('category',stripHTML(td[2]))
+        self.story.setMetadata('rating', stripHTML(td[5]))
+        self.story.setMetadata('numWords', stripHTML(td[6]))
  
             
     # grab the text for an individual chapter.

fanficdownloader/adapters/adapter_fannation.py

@@ -0,0 +1,44 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2014 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Software: eFiction
+import re
+from base_efiction_adapter import BaseEfictionAdapter
+
+class FanNationAdapter(BaseEfictionAdapter):
+
+    @staticmethod
+    def getSiteDomain():
+        return 'fannation.shades-of-moonlight.com'
+
+    @classmethod
+    def getPathToArchive(self):
+        return '/archive'
+
+    @classmethod
+    def getSiteAbbrev(self):
+        return 'fannation'
+
+    def handleMetadataPair(self, key, value):
+        if key == 'Romance':
+            for val in re.split("\s*,\s*", value):
+                self.story.addToList('romance', val)
+        else:
+            super(FanNationAdapter, self).handleMetadataPair(key, value)
+
+def getClass():
+    return FanNationAdapter

fanficdownloader/adapters/adapter_ficbooknet.py

@@ -52,7 +52,7 @@ class FicBookNetAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/readfic/'+self.story.getMetadata('storyId'))
@@ -70,8 +70,8 @@ class FicBookNetAdapter(BaseSiteAdapter):
         return 'www.ficbook.net'
 
     @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/readfic/12345 http://"+self.getSiteDomain()+"/readfic/93626/246417#part_content"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/readfic/12345 http://"+cls.getSiteDomain()+"/readfic/93626/246417#part_content"
 
     def getSiteURLPattern(self):
         return re.escape("http://"+self.getSiteDomain()+"/readfic/")+r"\d+"

fanficdownloader/adapters/adapter_fictionalleyorg.py

@@ -45,7 +45,7 @@ class FictionAlleyOrgSiteAdapter(BaseSiteAdapter):
         if m:
             self.story.setMetadata('authorId',m.group('auth'))
             self.story.setMetadata('storyId',m.group('id'))
-            logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+            
             # normalized story URL.
             self._setURL(url)
         else:
@@ -58,8 +58,8 @@ class FictionAlleyOrgSiteAdapter(BaseSiteAdapter):
         return 'www.fictionalley.org'
 
     @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/authors/drt/DA.html http://"+self.getSiteDomain()+"/authors/drt/JOTP01a.html"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/authors/drt/DA.html http://"+cls.getSiteDomain()+"/authors/drt/JOTP01a.html"
 
     def getSiteURLPattern(self):
         # http://www.fictionalley.org/authors/drt/DA.html
@@ -111,7 +111,7 @@ class FictionAlleyOrgSiteAdapter(BaseSiteAdapter):
             url=self.url
             logger.debug("Normalizing to URL: "+url)
             ## title's right there...
-            self.story.setMetadata('title',storya.string)
+            self.story.setMetadata('title',stripHTML(storya))
             data = self._fetchUrl(url)
             soup = bs.BeautifulSoup(data)
             chapterlinklist = soup.findAll('a',{'class':'chapterlink'})
@@ -121,7 +121,7 @@ class FictionAlleyOrgSiteAdapter(BaseSiteAdapter):
             ## same way.
             chapsoup = bs.BeautifulSoup(chapterdata)
             storya = chapsoup.find('div',{'class':'breadcrumbs'}).findAll('a')[1]
-            self.story.setMetadata('title',storya.string)
+            self.story.setMetadata('title',stripHTML(storya))
             del chapsoup
 
         del chapterdata

fanficdownloader/adapters/adapter_fictionmaniatv.py

@@ -0,0 +1,178 @@
+import re
+import urllib2
+import urlparse
+
+from .. import BeautifulSoup
+from ..BeautifulSoup import NavigableString
+
+from base_adapter import BaseSiteAdapter, makeDate
+from .. import exceptions
+
+
+def getClass():
+    return FictionManiaTVAdapter
+
+
+def _get_query_data(url):
+    components = urlparse.urlparse(url)
+    query_data = urlparse.parse_qs(components.query)
+    return dict((key, data[0]) for key, data in query_data.items())
+
+# yields Tag _and_ NavigableString siblings from the given tag. The
+# BeautifulSoup findNextSiblings() method for some reasons only returns either
+# NavigableStrings _or_ Tag objects, not both.
+def _yield_next_siblings(tag):
+    sibling = tag.nextSibling
+    while sibling:
+        yield sibling
+        sibling = sibling.nextSibling
+
+
+class FictionManiaTVAdapter(BaseSiteAdapter):
+    SITE_ABBREVIATION = 'fmt'
+    SITE_DOMAIN = 'fictionmania.tv'
+
+    BASE_URL = 'http://' + SITE_DOMAIN + '/stories/'
+    READ_TEXT_STORY_URL_TEMPLATE = BASE_URL + 'readtextstory.html?storyID=%s'
+    DETAILS_URL_TEMPLATE = BASE_URL + 'details.html?storyID=%s'
+
+    DATETIME_FORMAT = '%m/%d/%Y'
+    ALTERNATIVE_DATETIME_FORMAT = '%m/%d/%y'
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        query_data = urlparse.parse_qs(self.parsedUrl.query)
+        story_id = query_data['storyID'][0]
+
+        self.story.setMetadata('storyId', story_id)
+        self._setURL(self.READ_TEXT_STORY_URL_TEMPLATE % story_id)
+        self.story.setMetadata('siteabbrev', self.SITE_ABBREVIATION)
+
+        # Always single chapters, probably should use the Anthology feature to
+        # merge chapters of a story
+        self.story.setMetadata('numChapters', 1)
+
+    def _customized_fetch_url(self, url, exception=None, parameters=None):
+        if exception:
+            try:
+                data = self._fetchUrl(url, parameters)
+            except urllib2.HTTPError:
+                raise exception(self.url)
+        # Just let self._fetchUrl throw the exception, don't catch and
+        # customize it.
+        else:
+            data = self._fetchUrl(url, parameters)
+
+        return BeautifulSoup.BeautifulSoup(data)
+
+    @staticmethod
+    def getSiteDomain():
+        return FictionManiaTVAdapter.SITE_DOMAIN
+
+    @classmethod
+    def getSiteExampleURLs(cls):
+        return cls.READ_TEXT_STORY_URL_TEMPLATE % 1234
+
+    def getSiteURLPattern(self):
+        return re.escape(self.BASE_URL) + '(readtextstory|details)\.html\?storyID=\d+$'
+
+    def extractChapterUrlsAndMetadata(self):
+        url = self.DETAILS_URL_TEMPLATE % self.story.getMetadata('storyId')
+        soup = self._customized_fetch_url(url)
+
+        keep_summary_html = self.getConfig('keep_summary_html')
+        for row in soup.find('table')('tr'):
+            cells = row('td')
+            key = cells[0].b.string.strip(':')
+            try:
+                value = cells[1].string
+            except AttributeError:
+                value = None
+
+            if key == 'Story Name-Title':
+                self.story.setMetadata('title', value)
+                self.chapterUrls.append((value, self.url))
+
+            elif key == 'File Name':
+                self.story.setMetadata('fileName', value)
+
+            elif key == 'File Size':
+                self.story.setMetadata('fileSize', value)
+
+            elif key == 'Author':
+                element = cells[1].a
+                self.story.setMetadata('author', element.string)
+                query_data = _get_query_data(element['href'])
+                self.story.setMetadata('authorId', query_data['word'])
+                self.story.setMetadata('authorUrl', urlparse.urljoin(url, element['href']))
+
+            elif key == 'Date Added':
+                try:
+                    date = makeDate(value, self.DATETIME_FORMAT)
+                except ValueError:
+                    date = makeDate(value, self.ALTERNATIVE_DATETIME_FORMAT)
+                self.story.setMetadata('datePublished', date)
+
+            elif key == 'Old Name':
+                self.story.setMetadata('oldName', value)
+
+            elif key == 'New Name':
+                self.story.setMetadata('newName', value)
+
+            elif key == 'Other Key Names':
+                for name in value.split(', '):
+                    self.story.addToList('characters', name)
+
+            # I have no clue how the rating system works, if you are reading
+            # transgender fanfiction, you are probably an adult.
+            elif key == 'Rating':
+                self.story.setMetadata('rating', value)
+
+            elif key == 'Complete':
+                self.story.setMetadata('status', 'Complete' if value == 'Complete' else 'In-Progress')
+
+            elif key == 'Categories':
+                for element in cells[1]('a'):
+                    self.story.addToList('category', element.string)
+
+            elif key == 'Key Words':
+                for element in cells[1]('a'):
+                    self.story.addToList('keyWords', element.string)
+
+            elif key == 'Main Characters Age':
+                element = cells[1].a
+                self.story.setMetadata('mainCharactersAge', element.string)
+
+            elif key == 'Synopsis':
+                element = cells[1]
+
+                # Replace td with div to avoid possible strange formatting in
+                # the ebook later on
+                element.name = 'div'
+
+                if keep_summary_html:
+                    self.story.setMetadata('description', unicode(element))
+                else:
+                    self.story.setMetadata('description', ''.join(element(text=True)))
+
+            elif key == 'Reads':
+                self.story.setMetadata('readings', value)
+
+    def getChapterText(self, url):
+        soup = self._customized_fetch_url(url)
+        element = soup.find('pre')
+        element.name = 'div'
+
+        # The story's content is contained in a <pre> tag, probably taken 1:1
+        # from the source text file. A simple replacement of all newline
+        # characters with a break line tag should take care of formatting.
+
+        # While wrapping in paragraphs would be possible, it's too much work,
+        # I'd rather display the story 1:1 like it was found in the pre tag.
+        content = unicode(element)
+        content = content.replace('\n', '<br />')
+
+        if self.getConfig('non_breaking_spaces'):
+            content = content.replace(' ', '&nbsp;')
+        return content

fanficdownloader/adapters/adapter_fictionpadcom.py

@@ -0,0 +1,194 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+import time
+import json
+
+from .. import BeautifulSoup as bs
+#from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+class FictionPadSiteAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+        self.story.setMetadata('siteabbrev','fpad')
+        self.dateformat = "%Y-%m-%dT%H:%M:%SZ"
+        self.is_adult=False
+        self.username = None
+        self.password = None
+        # get storyId from url--url validation guarantees query correct
+        m = re.match(self.getSiteURLPattern(),url)
+        if m:
+            self.story.setMetadata('storyId',m.group('id'))
+            
+            # normalized story URL.
+            self._setURL("https://"+self.getSiteDomain()
+                         +"/author/"+m.group('author')
+                         +"/stories/"+self.story.getMetadata('storyId'))
+        else:
+            raise exceptions.InvalidStoryURL(url,
+                                             self.getSiteDomain(),
+                                             self.getSiteExampleURLs())
+
+    @staticmethod
+    def getSiteDomain():
+        return 'fictionpad.com'
+
+    @classmethod
+    def getSiteExampleURLs(cls):
+        return "https://fictionpad.com/author/Author/stories/1234/Some-Title"
+    
+    def getSiteURLPattern(self):
+        # http://fictionpad.com/author/Serdd/stories/4275
+        return r"http(s)?://(www\.)?fictionpad\.com/author/(?P<author>[^/]+)/stories/(?P<id>\d+)"
+
+# <form method="post" action="/signin">
+#     <input name="authenticity_token" type="hidden" value="u+cfdXh46dRnwVnSlmE2B2BFmHgu760paqgBG6KQeos=" />
+#     <input type="hidden" name="remember" value="1">
+#     <strong class="help-start text-center">or with FictionPad</strong>
+#     <label class="control-label hidden-placeholder">Pseudonym or Email Address</label>
+#     <input name="login" class="input-block-level" type="text" placeholder="Pseudonym or Email Address" maxlength="50" required autofocus>
+#     <label class="control-label hidden-placeholder">Password</label>
+#     <input name="password" class="input-block-level" type="password" placeholder="Password" minlength="6" required>
+#     <button type="submit" class="btn btn-primary btn-block">Sign In</button>
+#     <p class="help-end">
+#         <a href="/passwordreset">Forgot your password?</a>
+#     </p>
+# </form>
+    def performLogin(self):
+        params = {}
+
+        if self.password:
+            params['login'] = self.username
+            params['password'] = self.password
+        else:
+            params['login'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['remember'] = '1'
+
+        loginUrl = 'http://' + self.getSiteDomain() + '/signin'
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['login']))
+
+        ## need to pull empty login page first to get authenticity_token
+        soup = bs.BeautifulSoup(self._fetchUrl(loginUrl))
+        params['authenticity_token']=soup.find('input', {'name':'authenticity_token'})['value']
+        
+        data = self._postUrl(loginUrl, params)
+
+        if "Invalid email/pseudonym and password combination." in data:
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['login']))
+            raise exceptions.FailedToLogin(loginUrl,params['login'])
+            
+
+    def extractChapterUrlsAndMetadata(self):
+        # fetch the chapter.  From that we will get almost all the
+        # metadata and chapter list
+
+        url=self.url
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+            if "This is a mature story.  Please sign in to read it." in data:
+                self.performLogin()
+                data = self._fetchUrl(url)
+            
+            find = "wordyarn.config.page = "
+            data = data[data.index(find)+len(find):]
+            data = data[:data.index("</script>")]
+            data = data[:data.rindex(";")]
+            data = data.replace('tables:','"tables":')
+            tables = json.loads(data)['tables']
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(url)
+            else:
+                raise e
+
+        # looks like only one author per story allowed.
+        author = tables['users'][0]
+        story = tables['stories'][0]
+        story_ver = tables['story_versions'][0]
+        print("story:%s"%story)
+        
+        self.story.setMetadata('authorId',author['id'])
+        self.story.setMetadata('author',author['display_name'])
+        self.story.setMetadata('authorUrl','https://'+self.host+'/author/'+author['display_name']+'/stories')
+
+        self.story.setMetadata('title',story_ver['title'])
+        self.setDescription(url,story_ver['description'])
+        
+        if not ('assets/story_versions/covers' in story_ver['profile_image_url@2x']):
+            self.setCoverImage(url,story_ver['profile_image_url@2x'])
+
+        self.story.setMetadata('datePublished',makeDate(story['published_at'], self.dateformat))
+        self.story.setMetadata('dateUpdated',makeDate(story['published_at'], self.dateformat))
+
+        self.story.setMetadata('followers',story['followers_count'])
+        self.story.setMetadata('comments',story['comments_count'])
+        self.story.setMetadata('views',story['views_count'])
+        self.story.setMetadata('likes',int(story['likes'])) # no idea why they floated these.
+        if 'dislikes' in story:
+            self.story.setMetadata('dislikes',int(story['dislikes']))
+
+        if story_ver['is_complete']:
+            self.story.setMetadata('status', 'Completed')
+        else:
+            self.story.setMetadata('status', 'In-Progress')
+
+        self.story.setMetadata('rating', story_ver['maturity_level'])
+        self.story.setMetadata('numWords', unicode(story_ver['word_count']))
+
+        for i in tables['fandoms']:
+            self.story.addToList('category',i['name'])
+        
+        for i in tables['genres']:
+            self.story.addToList('genre',i['name'])
+        
+        for i in tables['characters']:
+            self.story.addToList('characters',i['name'])
+
+        for c in tables['chapters']:
+            chtitle = "Chapter %d"%c['number']
+            if c['title']:
+                chtitle += " - %s"%c['title']
+            self.chapterUrls.append((chtitle,c['body_url']))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+            
+    def getChapterText(self, url):
+        logger.debug('Getting chapter text from: %s' % url)
+        if not url:
+            data = u"<em>This chapter has no text.</em>"
+        else:
+            data = self._fetchUrl(url)
+        soup = bs.BeautifulSoup(u"<div id='story'>"+data+u"</div>")
+        return self.utf8FromSoup(url,soup)
+    
+def getClass():
+    return FictionPadSiteAdapter
+

fanficdownloader/adapters/adapter_fictionpresscom.py

@@ -40,11 +40,11 @@ class FictionPressComSiteAdapter(FanFictionNetSiteAdapter):
         return ['www.fictionpress.com','m.fictionpress.com']
 
     @classmethod
-    def getSiteExampleURLs(self):
-        return "http://www.fictionpress.com/s/1234/1/ http://www.fictionpress.com/s/1234/12/ http://www.fictionpress.com/s/1234/1/Story_Title http://m.fictionpress.com/s/1234/1/"
+    def getSiteExampleURLs(cls):
+        return "https://www.fictionpress.com/s/1234/1/ https://www.fictionpress.com/s/1234/12/ http://www.fictionpress.com/s/1234/1/Story_Title http://m.fictionpress.com/s/1234/1/"
 
     def getSiteURLPattern(self):
-        return r"http://(www|m)?\.fictionpress\.com/s/\d+(/\d+)?(/|/[a-zA-Z0-9_-]+)?/?$"
+        return r"https?://(www|m)?\.fictionpress\.com/s/\d+(/\d+)?(/|/[a-zA-Z0-9_-]+)?/?$"
 
 def getClass():
     return FictionPressComSiteAdapter

fanficdownloader/adapters/adapter_ficwadcom.py

@@ -46,7 +46,7 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
         return 'ficwad.com'
 
     @classmethod
-    def getSiteExampleURLs(self):
+    def getSiteExampleURLs(cls):
         return "http://ficwad.com/story/1234"
 
     def getSiteURLPattern(self):
@@ -65,7 +65,7 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
         loginUrl = 'http://' + self.getSiteDomain() + '/account/login'
         logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['username']))
-        d = self._postUrl(loginUrl,params)
+        d = self._postUrl(loginUrl,params,usecache=False)
 
         if "Login attempt failed..." in d:
             logger.info("Failed to login to URL %s as %s" % (loginUrl,
@@ -75,6 +75,13 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
         else:
             return True        
     
+    def use_pagecache(self):
+        '''
+        adapters that will work with the page cache need to implement
+        this and change it to True.
+        '''
+        return True
+
     def extractChapterUrlsAndMetadata(self):
 
         # fetch the chapter.  From that we will get almost all the
@@ -85,16 +92,20 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
 
         # use BeautifulSoup HTML parser to make everything easier to find.
         try:
-            soup = bs.BeautifulSoup(self._fetchUrl(url))
+            data = self._fetchUrl(url)
+            # non-existent/removed story urls get thrown to the front page.
+            if "<h2>Welcome to FicWad</h2>" in data:
+                raise exceptions.StoryDoesNotExist(self.url)
+            soup = bs.BeautifulSoup(data)
         except urllib2.HTTPError, e:
             if e.code == 404:
                 raise exceptions.StoryDoesNotExist(self.url)
             else:
                 raise e
             
-        h3 = soup.find('h3')
-        storya = h3.find('a',href=re.compile("^/story/\d+$"))
-        if storya : # if there's a story link in the h3 header, this is a chapter page.
+        divstory = soup.find('div',id='story')
+        storya = divstory.find('a',href=re.compile("^/story/\d+$"))
+        if storya : # if there's a story link in the divstory header, this is a chapter page.
             # normalize story URL on chapter list.
             self.story.setMetadata('storyId',storya['href'].split('/',)[2])
             url = "http://"+self.getSiteDomain()+storya['href']
@@ -109,17 +120,17 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
                     raise e
 
         # if blocked, attempt login.
-        if soup.find("li",{"class":"blocked"}):
+        if soup.find("div",{"class":"blocked"}):
             if self.performLogin(url): # performLogin raises
                                        # FailedToLogin if it fails.
-                soup = bs.BeautifulSoup(self._fetchUrl(url))
+                soup = bs.BeautifulSoup(self._fetchUrl(url,usecache=False))
 
         # title - first h4 tag will be title.
-        titleh4 = soup.find('h4')
-        self.story.setMetadata('title', titleh4.a.string)
+        titleh4 = soup.find('div',{'class':'storylist'}).find('h4')
+        self.story.setMetadata('title', stripHTML(titleh4.a))
 
         # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"^/author/\d+"))
+        a = soup.find('span',{'class':'author'}).find('a', href=re.compile(r"^/author/\d+"))
         self.story.setMetadata('authorId',a['href'].split('/')[2])
         self.story.setMetadata('authorUrl','http://'+self.host+a['href'])
         self.story.setMetadata('author',a.string)
@@ -135,7 +146,7 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
 
         # warnings
         # <span class="req"><a href="/help/38" title="Medium Spoilers">[!!] </a> <a href="/help/38" title="Rape/Sexual Violence">[R] </a> <a href="/help/38" title="Violence">[V] </a> <a href="/help/38" title="Child/Underage Sex">[Y] </a></span>
-        spanreq = metap.find("span",{"class":"req"})
+        spanreq = metap.find("span",{"class":"story-warnings"})
         if spanreq: # can be no warnings.
             for a in spanreq.findAll("a"):
                 self.story.addToList('warnings',a['title'])
@@ -161,16 +172,16 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
                 if g:
                     self.story.addToList('characters',g)
         
-        m = re.match(r".*?Published: ([0-9/]+?) -.*?",metastr)
+        m = re.match(r".*?Published: ([0-9-]+?) -.*?",metastr)
         if m:
-            self.story.setMetadata('datePublished',makeDate(m.group(1), "%Y/%m/%d"))
+            self.story.setMetadata('datePublished',makeDate(m.group(1), "%Y-%m-%d"))
 
         # Updated can have more than one space after it. <shrug>
-        m = re.match(r".*?Updated: ([0-9/]+?) +-.*?",metastr) 
+        m = re.match(r".*?Updated: ([0-9-]+?) +-.*?",metastr) 
         if m:
-            self.story.setMetadata('dateUpdated',makeDate(m.group(1), "%Y/%m/%d"))
+            self.story.setMetadata('dateUpdated',makeDate(m.group(1), "%Y-%m-%d"))
 
-        m = re.match(r".*? - ([0-9/]+?) words.*?",metastr)
+        m = re.match(r".*? - ([0-9,]+?) words.*?",metastr)
         if m:
             self.story.setMetadata('numWords',m.group(1))
 
@@ -181,7 +192,7 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
 
         # get the chapter list first this time because that's how we
         # detect the need to login.
-        storylistul = soup.find('ul',{'id':'storylist'})
+        storylistul = soup.find('ul',{'class':'storylist'})
         if not storylistul:
             # no list found, so it's a one-chapter story.
             self.chapterUrls.append((self.story.getMetadata('title'),url))

fanficdownloader/adapters/adapter_fimfictionnet.py

@@ -21,7 +21,6 @@ logger = logging.getLogger(__name__)
 import re
 import urllib2
 import cookielib as cl
-from datetime import datetime
 import json
 
 from .. import BeautifulSoup as bs
@@ -42,6 +41,10 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
         self._setURL("http://"+self.getSiteDomain()+"/story/"+self.story.getMetadata('storyId')+"/")
         self.is_adult = False
         
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%d %b %Y"
+            
     @staticmethod
     def getSiteDomain():
         return 'www.fimfiction.net'
@@ -52,16 +55,22 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
         return ['www.fimfiction.net','mobile.fimfiction.net', 'www.fimfiction.com', 'mobile.fimfiction.com']
 
     @classmethod
-    def getSiteExampleURLs(self):
+    def getSiteExampleURLs(cls):
         return "http://www.fimfiction.net/story/1234/story-title-here http://www.fimfiction.net/story/1234/ http://www.fimfiction.com/story/1234/1/ http://mobile.fimfiction.net/story/1234/1/story-title-here/chapter-title-here"
 
     def getSiteURLPattern(self):
-        return r"http://(www|mobile)\.fimfiction\.(net|com)/story/\d+/?.*"
+        return r"https?://(www|mobile)\.fimfiction\.(net|com)/story/\d+/?.*"
         
-    def extractChapterUrlsAndMetadata(self):
+    def use_pagecache(self):
+        '''
+        adapters that will work with the page cache need to implement
+        this and change it to True.
+        '''
+        return True
+    
+    def doExtractChapterUrlsAndMetadata(self,get_cover=True):
         
         if self.is_adult or self.getConfig("is_adult"):
-            cookieproc = urllib2.HTTPCookieProcessor()
             cookie = cl.Cookie(version=0, name='view_mature', value='true',
                                port=None, port_specified=False,
                                domain=self.getSiteDomain(), domain_specified=False, domain_initial_dot=False,
@@ -73,16 +82,15 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
                                comment_url=None,
                                rest={'HttpOnly': None},
                                rfc2109=False)
-            cookieproc.cookiejar.set_cookie(cookie)
-            self.opener = urllib2.build_opener(cookieproc)
+            self.cookiejar.set_cookie(cookie)
         
         try:
-            apiResponse = urllib2.urlopen("http://www.fimfiction.net/api/story.php?story=%s" % (self.story.getMetadata("storyId"))).read()
+            apiResponse = self._fetchUrl("http://www.fimfiction.net/api/story.php?story=%s" % (self.story.getMetadata("storyId")))
             apiData = json.loads(apiResponse)
             
             # Unfortunately, we still need to load the story index
             # page to parse the characters.  And chapters, now, too.
-            data = self._fetchUrl(self.url)
+            data = self.do_fix_blockquotes(self._fetchUrl(self.url))
             soup = bs.BeautifulSoup(data)
         except urllib2.HTTPError, e:
             if e.code == 404:
@@ -93,10 +101,12 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
         if "Warning: mysql_fetch_array(): supplied argument is not a valid MySQL result resource" in data:
             raise exceptions.StoryDoesNotExist(self.url)
 
-        if "/images/missing_story.png" in data:
-            raise exceptions.StoryDoesNotExist(self.url)
+        # Can cause problems if a missing story is referenced in a comment.
+        # Shouldn't be needed anyway.
+        # if "/images/missing_story.png" in data:
+        #     raise exceptions.StoryDoesNotExist(self.url)
         
-        if "This story has been marked as having adult content." in data:
+        if "This story has been marked as having adult content. Please click below to confirm you are of legal age to view adult material in your country." in data:
             raise exceptions.AdultCheckRequired(self.url)
         
         if self.password:
@@ -104,7 +114,7 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
             params['password'] = self.password
             data = self._postUrl(self.url,params)
 
-        if "Enter the password the author set for this story to view it." in data:
+        if not (soup.find('form', {'id' : 'password_form'}) == None):
             if self.getConfig('fail_on_password'):
                 raise exceptions.FailedToDownload("%s requires story password and fail_on_password is true."%self.url)
             else:
@@ -117,7 +127,7 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
             
         ## Title
         a = soup.find('a', href=re.compile(r'^/story/'+self.story.getMetadata('storyId')))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # self.story.setMetadata("title", storyMetadata["title"])
         # if not storyMetadata["title"]:
@@ -154,14 +164,10 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
         self.story.setMetadata("rating", storyMetadata["content_rating_text"])
 
         ## Warnings aren't included in the API.
-        bottomli = soup.find('li',{'class':'bottom'})
-        if bottomli:
-            bottomspans = bottomli.findAll('span')
-            # the first span in bottom is the rating, obtained above.
-            if bottomspans and len(bottomspans) > 1:
-                for warning in bottomspans[1:]:
-                    self.story.addToList('warnings',warning.string)
-                
+        if soup.find('a',{'class':'story_category story_category_gore'}):
+            self.story.addToList('warnings',"Gore")
+        if soup.find('a',{'class':'story_category story_category_sex'}):
+            self.story.addToList('warnings',"Sex")
         
         for category in storyMetadata["categories"]:
             if storyMetadata["categories"][category]:
@@ -178,21 +184,50 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
             if coverurl.startswith('//'): # fix for img urls missing 'http:'
                 coverurl = "http:"+coverurl
 
-            self.setCoverImage(self.url,coverurl)
+            if get_cover:
+                self.setCoverImage(self.url,coverurl)
 
         # fimf has started including extra stuff inside the description div.
-        descdivstr = "%s"%soup.find("div", {"class":"description"})
-        hrstr="<hr />"
-        descdivstr = '<div class="description">'+descdivstr[descdivstr.index(hrstr)+len(hrstr):]
+        descdivstr = u"%s"%soup.find("div", {"class":"description"})
+        hrstr=u"<hr />"
+        descdivstr = u'<div class="description">'+descdivstr[descdivstr.index(hrstr)+len(hrstr):]
         self.setDescription(self.url,descdivstr)
-        
+
+        # Can't trust dates from API anymore I'm told.
         # Dates are in Unix time
         # Take the publish date from the first chapter posted
-        rawDatePublished = storyMetadata["chapters"][0]["date_modified"]
-        self.story.setMetadata("datePublished", datetime.fromtimestamp(rawDatePublished))
-        rawDateUpdated = storyMetadata["date_modified"]
-        self.story.setMetadata("dateUpdated", datetime.fromtimestamp(rawDateUpdated))
+        # rawDatePublished = storyMetadata["chapters"][0]["date_modified"]
+        # self.story.setMetadata("datePublished", datetime.fromtimestamp(rawDatePublished))
+        # rawDateUpdated = storyMetadata["date_modified"]
+        # self.story.setMetadata("dateUpdated", datetime.fromtimestamp(rawDateUpdated))
+        
+        oldestChapter = None
+        newestChapter = None
+        self.newestChapterNum = None # save for comparing during update.
+        # Scan all chapters to find the oldest and newest, on
+        # FiMFiction it's possible for authors to insert new chapters
+        # out-of-order or change the dates of earlier ones by editing
+        # them--That WILL break epub update.
+        for index, chapterDate in enumerate(soup.findAll('span', {'class':'date'})):
+            date=re.sub(r"(\d+)(st|nd|rd|th)",r"\1",chapterDate.contents[1].strip())
+            chapterDate = makeDate(date,self.dateformat)
+            if oldestChapter == None or chapterDate < oldestChapter:
+                oldestChapter = chapterDate
+            if newestChapter == None or chapterDate > newestChapter:
+                newestChapter = chapterDate
+                self.newestChapterNum = index
 
+        self.story.setMetadata("dateUpdated", newestChapter)
+        
+        pubdatetag = soup.find('span', {'class':'date_approved'})
+        if pubdatetag is None:
+            self.story.setMetadata("datePublished", oldestChapter)            
+        else:
+            pubdateraw = pubdatetag('span')[1].text
+            datestripped=re.sub(r"(\d+)(st|nd|rd|th)",r"\1",pubdateraw.strip())
+            pubDate = makeDate(datestripped,self.dateformat)
+            self.story.setMetadata("datePublished", pubDate)
+            
         chars = soup.find("div", {"class":"inner_data"})
         # fimfic stopped putting the char name on or around the char
         # icon now for some reason.  Pull it from the image name with
@@ -215,11 +250,63 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
                 if not isinstance(value,basestring):
                     value = unicode(value)
                 self.story.setMetadata(metakey, value)
+
+        #groups
+        if soup.find('button', {'id':'button-view-all-groups'}):
+            groupResponse = self._fetchUrl("http://www.fimfiction.net/ajax/groups/story_groups_list.php?story=%s" % (self.story.getMetadata("storyId")))
+            groupData = json.loads(groupResponse)
+            groupList = bs.BeautifulSoup(groupData["content"])
+        else:
+            groupList = soup.find('ul', {'id':'story-groups-list'})
+
+        if not (groupList == None):
+            for groupName in groupList.findAll('a'):
+                self.story.addToList("groupsUrl", 'http://'+self.host+groupName["href"]) 
+                self.story.addToList("groups",stripHTML(groupName).replace(',', ';'))
+
+        #sequels
+        sequelStoryHeader = soup.find('h1', {'class':'header-stories'}, text="Sequels")
+        if not sequelStoryHeader == None:
+            sequelContainer = sequelStoryHeader.parent.parent
+            for sequel in sequelContainer.findAll('a', {'class':'story_link'}):
+                self.story.addToList("sequelsUrl", 'http://'+self.host+sequel["href"]) 
+                self.story.addToList("sequels", stripHTML(sequel).replace(',', ';'))
+                
+        #The link to the prequel is embedded in the description text, so erring
+        #on the side of caution and wrapping this whole thing in a try block.
+        #If anything goes wrong this probably wasn't a valid prequel link.
+        try:
+            description = soup.find('div', {'class':'description'})
+            firstHR = description.find("hr")
+            nextSib = firstHR.nextSibling
+            if "This story is a sequel to" in nextSib.string:
+                link = nextSib.nextSibling
+                if link.name == "a":
+                    self.story.setMetadata("prequelUrl", 'http://'+self.host+link["href"])
+                    self.story.setMetadata("prequel", stripHTML(link))
+        except:
+            pass
+        
+    def hookForUpdates(self,chaptercount):
+        if self.oldchapters and len(self.oldchapters) > self.newestChapterNum:
+            print("Existing epub has %s chapters\nNewest chapter is %s.  Discarding old chapters from there on."%(len(self.oldchapters), self.newestChapterNum+1))
+            self.oldchapters = self.oldchapters[:self.newestChapterNum]
+        return len(self.oldchapters)
+
+    def do_fix_blockquotes(self,data):
+        if self.getConfig('fix_fimf_blockquotes'):
+            # <p class="double"><blockquote>
+            # </blockquote></p>
+            # include > in re groups so there's always something in the group.
+            data = re.sub(r'<p([^>]*>\s*)<blockquote([^>]*>)',r'<blockquote\2<p\1',data)
+            data = re.sub(r'</blockquote(>\s*)</p>',r'</p\1</blockquote>',data)
+        return data
         
-            
     def getChapterText(self, url):
         logger.debug('Getting chapter text from: %s' % url)
-        soup = bs.BeautifulSoup(self._fetchUrl(url),selfClosingTags=('br','hr')).find('div', {'class' : 'chapter_content'})
+
+        data = self.do_fix_blockquotes(self._fetchUrl(url))
+        soup = bs.BeautifulSoup(data,selfClosingTags=('br','hr')).find('div', {'class' : 'chapter_content'})
         if soup == None:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
         return self.utf8FromSoup(url,soup)

fanficdownloader/adapters/adapter_finestoriescom.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-# Copyright 2012 Fanficdownloader team
+# Copyright 2013 Fanficdownloader team
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -37,11 +37,6 @@ class FineStoriesComAdapter(BaseSiteAdapter):
     def __init__(self, config, url):
         BaseSiteAdapter.__init__(self, config, url)
 
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
         self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
         self.password = ""
         self.is_adult=False
@@ -50,7 +45,6 @@ class FineStoriesComAdapter(BaseSiteAdapter):
         self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2].split(':')[0])
         if 'storyInfo' in self.story.getMetadata('storyId'):
             self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/s/storyInfo.php?id='+self.story.getMetadata('storyId'))
@@ -68,8 +62,8 @@ class FineStoriesComAdapter(BaseSiteAdapter):
         return 'finestories.com'
 
     @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/s/1234 http://"+self.getSiteDomain()+"/s/1234:4010 http://"+self.getSiteDomain()+"/library/storyInfo.php?id=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/s/1234 http://"+cls.getSiteDomain()+"/s/1234:4010 http://"+cls.getSiteDomain()+"/library/storyInfo.php?id=1234"
 
     def getSiteURLPattern(self):
         return re.escape("http://"+self.getSiteDomain())+r"/(s|library)?/(storyInfo.php\?id=)?\d+(:\d+)?(;\d+)?$"
@@ -93,7 +87,7 @@ class FineStoriesComAdapter(BaseSiteAdapter):
             params['theusername'] = self.getConfig("username")
             params['thepassword'] = self.getConfig("password")
         params['rememberMe'] = '1'
-        params['page'] = 'http://finestories.com/'
+        params['page'] = 'http://'+self.getSiteDomain()+'/'
         params['submit'] = 'Login'
     
         loginUrl = 'http://' + self.getSiteDomain() + '/login.php'
@@ -142,7 +136,7 @@ class FineStoriesComAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('a', href=re.compile(r'/s/'+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.text)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"/a/\w+"))
@@ -262,8 +256,6 @@ class FineStoriesComAdapter(BaseSiteAdapter):
                 last[len(last)-1]=last[len(last)-1].append(next)
                 div.append(div1)
             
-            
-            
         # removing all the left-over stuff    
         for a in div.findAll('span'):
             a.extract()

fanficdownloader/adapters/adapter_grangerenchantedcom.py

@@ -15,6 +15,7 @@
 # limitations under the License.
 #
 
+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -48,7 +49,7 @@ class GrangerEnchantedCom(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         self.section=self.parsedUrl.path.split('/',)[1]
              
         # normalized story URL.
@@ -75,7 +76,7 @@ class GrangerEnchantedCom(BaseSiteAdapter):
         return ['grangerenchanted.com','malfoymanor.grangerenchanted.com']
 
     @classmethod
-    def getSiteExampleURLs(self):
+    def getSiteExampleURLs(cls):
         return "http://grangerenchanted.com/enchant/viewstory.php?sid=1234 http://malfoymanor.grangerenchanted.com/themanor/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
@@ -182,7 +183,7 @@ class GrangerEnchantedCom(BaseSiteAdapter):
         
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))

fanficdownloader/adapters/adapter_harrypotterfanfictioncom.py

@@ -15,6 +15,7 @@
 # limitations under the License.
 #
 
+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -42,7 +43,7 @@ class HarryPotterFanFictionComSiteAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only psid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?psid='+self.story.getMetadata('storyId'))
@@ -57,7 +58,7 @@ class HarryPotterFanFictionComSiteAdapter(BaseSiteAdapter):
         return ['www.harrypotterfanfiction.com','harrypotterfanfiction.com']
 
     @classmethod
-    def getSiteExampleURLs(self):
+    def getSiteExampleURLs(cls):
         return "http://www.harrypotterfanfiction.com/viewstory.php?psid=1234"
 
     def getSiteURLPattern(self):
@@ -92,7 +93,7 @@ class HarryPotterFanFictionComSiteAdapter(BaseSiteAdapter):
 
         ## Title
         a = soup.find('a', href=re.compile(r'\?psid='+self.story.getMetadata('storyId')))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         ## javascript:if (confirm('Please note. This story may contain adult themes. By clicking here you are stating that you are over 17. Click cancel if you do not meet this requirement.')) location = '?psid=290995'
         if "This story may contain adult themes." in a['href'] and not (self.is_adult or self.getConfig("is_adult")):
             raise exceptions.AdultCheckRequired(self.url)
@@ -131,55 +132,55 @@ class HarryPotterFanFictionComSiteAdapter(BaseSiteAdapter):
 
         ## Finding the metadata is a bit of a pain.  Most of the meta
         ## data is in a center.table without a bgcolor.
-        for center in soup.findAll('center'):
-            table = center.find('table',{'bgcolor':None})
-            if table:
-                metastr = stripHTML(str(table)).replace('\n',' ').replace('\t',' ')
-                # Rating: 12+ Story Reviews: 3
-                # Chapters: 3
-                # Characters: Andromeda, Ted, Bellatrix, R. Lestrange, Lucius, Narcissa, OC
-                # Genre(s): Fluff, Romance, Young Adult Era: OtherPairings: Other Pairing, Lucius/Narcissa
-                # Status: Completed
-                # First Published: 2010.09.02
-                # Last Published Chapter: 2010.09.28
-                # Last Updated: 2010.09.28
-                # Favorite Story Of: 1 users
-                # Warnings: Scenes of a Mild Sexual Nature
+        #for center in soup.findAll('center'):
+        table = soup.find('table',{'class':'storymaininfo'})
+        if table:
+            metastr = stripHTML(str(table)).replace('\n',' ').replace('\t',' ')
+            # Rating: 12+ Story Reviews: 3
+            # Chapters: 3
+            # Characters: Andromeda, Ted, Bellatrix, R. Lestrange, Lucius, Narcissa, OC
+            # Genre(s): Fluff, Romance, Young Adult Era: OtherPairings: Other Pairing, Lucius/Narcissa
+            # Status: Completed
+            # First Published: 2010.09.02
+            # Last Published Chapter: 2010.09.28
+            # Last Updated: 2010.09.28
+            # Favorite Story Of: 1 users
+            # Warnings: Scenes of a Mild Sexual Nature
 
-                m = re.match(r".*?Status: Completed.*?",metastr)
-                if m:
-                    self.story.setMetadata('status','Completed')
-                else:
-                    self.story.setMetadata('status','In-Progress')
+            m = re.match(r".*?Status: Completed.*?",metastr)
+            if m:
+                self.story.setMetadata('status','Completed')
+            else:
+                self.story.setMetadata('status','In-Progress')
 
-                m = re.match(r".*?Rating: (.+?) Story Reviews.*?",metastr)
-                if m:
-                    self.story.setMetadata('rating', m.group(1))
+            m = re.match(r".*?Rating: (.+?) Story Reviews.*?",metastr)
+            if m:
+                self.story.setMetadata('rating', m.group(1))
 
-                m = re.match(r".*?Genre\(s\): (.+?) Era.*?",metastr)
-                if m:
-                    for g in m.group(1).split(','):
-                        self.story.addToList('genre',g)
-        
-                m = re.match(r".*?Characters: (.+?) Genre.*?",metastr)
-                if m:
-                    for g in m.group(1).split(','):
-                        self.story.addToList('characters',g)
-        
-                m = re.match(r".*?Warnings: (.+).*?",metastr)
-                if m:
-                    for w in m.group(1).split(','):
-                        if w != 'Now Warnings':
-                            self.story.addToList('warnings',w)
-        
-                m = re.match(r".*?First Published: ([0-9\.]+).*?",metastr)
-                if m:
-                    self.story.setMetadata('datePublished',makeDate(m.group(1), "%Y.%m.%d"))
+            m = re.match(r".*?Genre\(s\): (.+?) Era.*?",metastr)
+            if m:
+                for g in m.group(1).split(','):
+                    self.story.addToList('genre',g)
+    
+            m = re.match(r".*?Characters: (.+?) Genre.*?",metastr)
+            if m:
+                for g in m.group(1).split(','):
+                    self.story.addToList('characters',g)
+    
+            m = re.match(r".*?Warnings: (.+).*?",metastr)
+            if m:
+                for w in m.group(1).split(','):
+                    if w != 'Now Warnings':
+                        self.story.addToList('warnings',w)
+    
+            m = re.match(r".*?First Published: ([0-9\.]+).*?",metastr)
+            if m:
+                self.story.setMetadata('datePublished',makeDate(m.group(1), "%Y.%m.%d"))
 
-                # Updated can have more than one space after it. <shrug>
-                m = re.match(r".*?Last Updated: ([0-9\.]+).*?",metastr)
-                if m:
-                    self.story.setMetadata('dateUpdated',makeDate(m.group(1), "%Y.%m.%d"))
+            # Updated can have more than one space after it. <shrug>
+            m = re.match(r".*?Last Updated: ([0-9\.]+).*?",metastr)
+            if m:
+                self.story.setMetadata('dateUpdated',makeDate(m.group(1), "%Y.%m.%d"))
 
     def getChapterText(self, url):
 

fanficdownloader/adapters/adapter_hennethannunnet.py

@@ -48,7 +48,7 @@ class HennethAnnunNetAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/stories/chapter.cfm?stid='+self.story.getMetadata('storyId'))
@@ -66,8 +66,8 @@ class HennethAnnunNetAdapter(BaseSiteAdapter):
         return 'www.henneth-annun.net'
 
     @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/stories/chapter.cfm?stid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/stories/chapter.cfm?stid=1234"
 
     def getSiteURLPattern(self):
         return "http://"+self.getSiteDomain()+"/stories/chapter(_view)?.cfm\?stid="+r"\d+$"
@@ -101,7 +101,7 @@ class HennethAnnunNetAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('h2', {'id':'page_heading'})
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
 
         # Find the chapters: chapter_view.cfm?stid=6663&spordinal=1" 
         for chapter in soup.findAll('a', href=re.compile(r'chapter_view.cfm\?stid='+self.story.getMetadata('storyId')+"&spordinal=\d+$")):

fanficdownloader/adapters/adapter_hlfictionnet.py

@@ -15,6 +15,7 @@
 # limitations under the License.
 #
 
+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -48,7 +49,7 @@ class HLFictionNetAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -66,8 +67,8 @@ class HLFictionNetAdapter(BaseSiteAdapter):
         return 'hlfiction.net'
 
     @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
         return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
@@ -106,7 +107,7 @@ class HLFictionNetAdapter(BaseSiteAdapter):
         self.story.setMetadata('author',aut.string)
         aut.extract()
         
-        self.story.setMetadata('title',a.string[:(len(a.string)-3)])
+        self.story.setMetadata('title',stripHTML(a)[:(len(a.string)-3)])
 
         # Find the chapters:
         chapters=soup.find('select')

fanficdownloader/adapters/adapter_hpfandomnet.py

@@ -15,6 +15,7 @@
 # limitations under the License.
 #
 
+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -52,7 +53,7 @@ class HPFandomNetAdapterAdapter(BaseSiteAdapter): # XXX
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         # XXX Most sites don't have the /eff part.  Replace all to remove it usually.
@@ -71,8 +72,8 @@ class HPFandomNetAdapterAdapter(BaseSiteAdapter): # XXX
         return 'www.hpfandom.net' # XXX
 
     @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/eff/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/eff/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
         return re.escape("http://"+self.getSiteDomain()+"/eff/viewstory.php?sid=")+r"\d+$"

fanficdownloader/adapters/adapter_hpfanficarchivecom.py

@@ -15,6 +15,7 @@
 # limitations under the License.
 #
 
+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -48,7 +49,7 @@ class HPFanficArchiveComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/stories/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -66,8 +67,8 @@ class HPFanficArchiveComAdapter(BaseSiteAdapter):
         return 'www.hpfanficarchive.com'
 
     @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/stories/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/stories/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
         return re.escape("http://"+self.getSiteDomain()+"/stories/viewstory.php?sid=")+r"\d+$"
@@ -100,7 +101,7 @@ class HPFanficArchiveComAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -128,23 +129,22 @@ class HPFanficArchiveComAdapter(BaseSiteAdapter):
         # <span class="label">Rated:</span> NC-17<br /> etc
         labels = soup.findAll('span',{'class':'label'})
         for labelspan in labels:
-            value = labelspan.nextSibling
+            val = labelspan.nextSibling
+            value = unicode('')
+            while val and not defaultGetattr(val,'class') == 'label':
+                value += unicode(val)
+                val = val.nextSibling
             label = labelspan.string
+            #print("label:%s\nvalue:%s"%(label,value))
 
             if 'Summary' in label:
-                ## Everything until the next span class='label'
-                svalue = ""
-                while not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
-                    value = value.nextSibling
-                self.setDescription(url,svalue)
-                #self.story.setMetadata('description',stripHTML(svalue))
+                self.setDescription(url,value)
 
             if 'Rated' in label:
-                self.story.setMetadata('rating', value)
+                self.story.setMetadata('rating', stripHTML(value))
 
             if 'Word count' in label:
-                self.story.setMetadata('numWords', value)
+                self.story.setMetadata('numWords', stripHTML(value))
 
             if 'Categories' in label:
                 cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
@@ -161,13 +161,18 @@ class HPFanficArchiveComAdapter(BaseSiteAdapter):
                 for genre in genres:
                     self.story.addToList('genre',genre.string)
 
+            if 'Pairing' in label:
+                ships = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=4'))
+                for ship in ships:
+                    self.story.addToList('ships',ship.string)
+
             if 'Warnings' in label:
                 warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
                 for warning in warnings:
                     self.story.addToList('warnings',warning.string)
 
             if 'Completed' in label:
-                if 'Yes' in value:
+                if 'Yes' in stripHTML(value):
                     self.story.setMetadata('status', 'Completed')
                 else:
                     self.story.setMetadata('status', 'In-Progress')

fanficdownloader/adapters/adapter_iketernalnet.py

@@ -15,6 +15,7 @@
 # limitations under the License.
 #
 
+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -48,7 +49,7 @@ class IkEternalNetAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -66,8 +67,8 @@ class IkEternalNetAdapter(BaseSiteAdapter):
         return 'www.ik-eternal.net'
 
     @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
         return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
@@ -183,7 +184,7 @@ class IkEternalNetAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))

fanficdownloader/adapters/adapter_imagineeficcom.py

@@ -15,6 +15,7 @@
 # limitations under the License.
 #
 
+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -48,7 +49,7 @@ class ImagineEFicComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -66,8 +67,8 @@ class ImagineEFicComAdapter(BaseSiteAdapter):
         return 'imagine.e-fic.com'
 
     @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
         return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
@@ -170,7 +171,7 @@ class ImagineEFicComAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))

fanficdownloader/adapters/adapter_indeathnet.py

@@ -46,12 +46,12 @@ class InDeathNetAdapter(BaseSiteAdapter):
         self.password = ""
         self.is_adult=False
         
-		
-		  # get storyId from url--url validation guarantees query correct
+        
+        # get storyId from url--url validation guarantees query correct
         m = re.match(self.getSiteURLPattern(),url)
         if m:
             self.story.setMetadata('storyId',m.group('id'))
-            logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+            
             # normalized story URL.
             self._setURL('http://www.' + self.getSiteDomain() + '/blog/archive/'+self.story.getMetadata('storyId')+'-'+m.group('name')+'/')
         else:
@@ -73,8 +73,8 @@ class InDeathNetAdapter(BaseSiteAdapter):
 
 
     @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/blog/archive/123-story-in-death/"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/blog/archive/123-story-in-death/"
 
     def getSiteURLPattern(self):
         # http://www.indeath.net/blog/archive/169-ransom-in-death/
@@ -87,6 +87,34 @@ class InDeathNetAdapter(BaseSiteAdapter):
         postdate = makeDate(d.group('day')+' '+ym.group('mon')+' '+ym.group('year'),self.dateformat)
         return postdate
 
+    def getAuthorData(self):
+        
+        mainUrl = self.url.replace("/archive","")
+        
+        try:
+            maindata = self._fetchUrl(mainUrl)
+            
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.meta)
+            else:
+                raise e
+
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        mainsoup = bs.BeautifulSoup(maindata)
+
+        # find first entry
+        e = mainsoup.find('div',{'class':"entry"})
+        
+        # get post author as author
+        d = e.find('div',{'class':"desc"})
+        a = d.find('strong')
+        self.story.setMetadata('author',a.contents[0].string.strip())
+        
+        # Don't seem to be able to get author pages anymore
+        self.story.setMetadata('authorUrl','http://www.indeath.net/')
+        self.story.setMetadata('authorId','0')
+        
     ## Getting the chapter list and the meta data, plus 'is adult' checking.
     def extractChapterUrlsAndMetadata(self):
 
@@ -109,21 +137,14 @@ class InDeathNetAdapter(BaseSiteAdapter):
         ## Title
         h = soup.find('a', id="blog_title")
         t = h.find('span')
-        self.story.setMetadata('title',t.contents[0].string.strip())
-		
+        self.story.setMetadata('title',stripHTML(t.contents[0]).strip())
+        
         s = t.find('div')        
         if s != None:
             self.setDescription(url,s)
-		
-        # Find authorid and URL from first link in Recent Entries (don't yet reference 'recent entries' - let's see if that is required)
-        a = soup.find('a', href=re.compile(r"http://www.indeath.net/user/\d+\-[a-z0-9]+/$"))		#http://www.indeath.net/user/9083-cyrex/
-        m = re.search('http://www.indeath.net/user/(?P<id>\d+)\-(?P<name>[a-z0-9]*)/$',a['href'])
-        self.story.setMetadata('authorId',m.group('id'))
-        self.story.setMetadata('authorUrl',a['href'])
-        self.story.setMetadata('author',m.group('name'))
-
-
 
+        # Get Author from main blog page since it's not reliably on the archive page
+        self.getAuthorData()
 
         # Find the chapters:
         chapters=soup.findAll('a', title="View entry", href=re.compile(r'http://www.indeath.net/blog/'+self.story.getMetadata('storyId')+"/entry\-(\d+)\-([^/]*)/$"))
@@ -152,21 +173,20 @@ class InDeathNetAdapter(BaseSiteAdapter):
             if len(chapters)==1:
                 self.chapterUrls.append((self.story.getMetadata('title'),chapter['href']))
             else:
-            	ct = stripHTML(chapter)
-            	tnew = re.match("(?i)"+self.story.getMetadata('title')+r" - (?P<newtitle>.*)$",ct)
-            	if tnew:
-            		chaptertitle = tnew.group('newtitle')
-            	else:
-            		chaptertitle = ct
+                ct = stripHTML(chapter)
+                tnew = re.match("(?i)"+self.story.getMetadata('title')+r" - (?P<newtitle>.*)$",ct)
+                if tnew:
+                    chaptertitle = tnew.group('newtitle')
+                else:
+                    chaptertitle = ct
                 self.chapterUrls.append((chaptertitle,chapter['href']))
-	
-		
+
         
         
     # grab the text for an individual chapter.
     def getChapterText(self, url):
         logger.debug('Getting chapter text from: %s' % url)
-		
+        
         #chapter=bs.BeautifulSoup('<div class="story"></div>')
         data = self._fetchUrl(url)
         soup = bs.BeautifulSoup(data,selfClosingTags=('br','hr','span','center'))

fanficdownloader/adapters/adapter_ksarchivecom.py

@@ -15,6 +15,7 @@
 # limitations under the License.
 #
 
+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -52,7 +53,7 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
@@ -75,8 +76,8 @@ class KSArchiveComAdapter(BaseSiteAdapter): # XXX
         return 'ksarchive.com' # XXX
 
     @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
         return "http://(www.)?"+re.escape(self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"

fanficdownloader/adapters/adapter_libraryofmoriacom.py

@@ -15,6 +15,7 @@
 # limitations under the License.
 #
 
+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -49,7 +50,7 @@ class LibraryOfMoriaComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/a/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -67,8 +68,8 @@ class LibraryOfMoriaComAdapter(BaseSiteAdapter):
         return 'www.libraryofmoria.com'
 
     @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/a/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/a/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
         return re.escape("http://"+self.getSiteDomain()+"/a/viewstory.php?sid=")+r"\d+$"
@@ -129,7 +130,7 @@ class LibraryOfMoriaComAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))

fanficdownloader/adapters/adapter_literotica.py

@@ -0,0 +1,258 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+import urlparse
+import time
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter, makeDate
+
+class LiteroticaSiteAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["utf8",
+                       "Windows-1252"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+
+        self.story.setMetadata('siteabbrev','litero')
+
+        # normalize to first chapter.  Not sure if they ever have more than 2 digits.
+        storyId = self.parsedUrl.path.split('/',)[2]
+        # replace later chapters with first chapter but don't remove numbers
+        # from the URL that disambiguate stories with the same title.
+        storyId = re.sub("-ch-?\d\d", "", storyId)
+        self.story.setMetadata('storyId', storyId)
+
+        ## accept m(mobile)url, but use www.
+        url = re.sub("^(www|german|spanish|french|dutch|italian|romanian|portuguese|other)\.i",
+                              "\1",
+                              url)
+
+        ## strip ?page=...
+        url = re.sub("\?page=.*$", "", url)
+
+        ## set url
+        self._setURL(url)
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = '%m/%d/%y'
+
+    @staticmethod
+    def getSiteDomain():
+        return 'literotica.com'
+
+    @classmethod
+    def getAcceptDomains(cls):
+        return ['www.literotica.com',
+                'www.i.literotica.com',
+                'german.literotica.com',
+                'german.i.literotica.com',
+                'spanish.literotica.com',
+                'spanish.i.literotica.com',
+                'french.literotica.com',
+                'french.i.literotica.com',
+                'dutch.literotica.com',
+                'dutch.i.literotica.com',
+                'italian.literotica.com',
+                'italian.i.literotica.com',
+                'romanian.literotica.com',
+                'romanian.i.literotica.com',
+                'portuguese.literotica.com',
+                'portuguese.i.literotica.com',
+                'other.literotica.com',
+                'other.i.literotica.com']
+
+    @classmethod
+    def getSiteExampleURLs(cls):
+        return "http://www.literotica.com/s/story-title https://www.literotica.com/s/story-title http://portuguese.literotica.com/s/story-title http://german.literotica.com/s/story-title"
+
+    def getSiteURLPattern(self):
+        return r"https?://(www|german|spanish|french|dutch|italian|romanian|portuguese|other)(\.i)?\.literotica\.com/s/([a-zA-Z0-9_-]+)"
+
+    def extractChapterUrlsAndMetadata(self):
+        """
+        NOTE: Some stories can have versions, 
+              e.g. /my-story-ch-05-version-10
+        NOTE: If two stories share the same title, a running index is added,
+              e.g.: /my-story-ch-02-1
+        Strategy:
+            * Go to author's page, search for the current story link,
+            * If it's in a tr.root-story => One-part story
+                * , get metadata and be done
+            * If it's in a tr.sl => Chapter in series
+                * Search up from there until we find a tr.ser-ttl (this is the
+                story)
+                * Gather metadata
+                * Search down from there for all tr.sl until the next
+                tr.ser-ttl, foreach
+                    * Chapter link is there
+        """
+
+        if not (self.is_adult or self.getConfig("is_adult")):
+            raise exceptions.AdultCheckRequired(self.url)
+
+        logger.debug("Chapter/Story URL: <%s> " % self.url)
+        try:
+            data1 = self._fetchUrl(self.url)
+            soup1 = bs.BeautifulSoup(data1)
+            #strip comments from soup
+            [comment.extract() for comment in soup1.findAll(text=lambda text:isinstance(text, bs.Comment))]
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        # author
+        a = soup1.find("span", "b-story-user-y")
+        self.story.setMetadata('authorId', urlparse.parse_qs(a.a['href'].split('?')[1])['uid'][0])
+        authorurl = a.a['href']
+        if authorurl.startswith('//'):
+            authorurl = self.parsedUrl.scheme+':'+authorurl
+        self.story.setMetadata('authorUrl', authorurl)
+        self.story.setMetadata('author', a.text)
+
+        # get the author page
+        try:
+            dataAuth = self._fetchUrl(authorurl)
+            soupAuth = bs.BeautifulSoup(dataAuth)
+            #strip comments from soup
+            [comment.extract() for comment in soupAuth.findAll(text=lambda text:isinstance(text, bs.Comment))]
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(authorurl)
+            else:
+                raise e
+
+        ## Find link to url in author's page
+        ## site has started using //domain.name/asdf urls remove https?: from front
+        storyLink = soupAuth.find('a', href=self.url[self.url.index(':')+1:])
+
+        if storyLink is not None:
+            urlTr = storyLink.parent.parent
+            if urlTr['class'] == "sl":
+                isSingleStory = False
+            else:
+                isSingleStory = True
+        else:
+            raise exceptions.FailedToDownload("Couldn't find story <%s> on author's page <%s>" % (url, authorurl))
+
+        if isSingleStory:
+            self.story.setMetadata('title', storyLink.text)
+            self.story.setMetadata('description', urlTr.findAll("td")[1].text)
+            self.story.addToList('eroticatags', urlTr.findAll("td")[2].text)
+            date = urlTr.findAll('td')[-1].text
+            self.story.setMetadata('datePublished', makeDate(date, self.dateformat))
+            self.story.setMetadata('dateUpdated',makeDate(date, self.dateformat))
+            self.chapterUrls = [(storyLink.text, self.url)]
+        else:
+            seriesTr = urlTr.previousSibling
+            while seriesTr['class'] != 'ser-ttl':
+                seriesTr = seriesTr.previousSibling
+            m = re.match("^(?P<title>.*?):\s(?P<numChapters>\d+)\sPart\sSeries$", seriesTr.find("strong").text)
+            self.story.setMetadata('title', m.group('title'))
+
+            ## Walk the chapters
+            chapterTr = seriesTr.nextSibling
+            self.chapterUrls = []
+            dates = []
+            descriptions = []
+            while chapterTr is not None and chapterTr['class'] == 'sl':
+                descriptions.append(chapterTr.findAll("td")[1].text)
+                chapterLink = chapterTr.find("td", "fc").find("a")
+                self.chapterUrls.append((chapterLink.text, "http:" + chapterLink["href"]))
+                self.story.addToList('eroticatags', chapterTr.findAll("td")[2].text)
+                dates.append(makeDate(chapterTr.findAll('td')[-1].text, self.dateformat))
+                chapterTr = chapterTr.nextSibling
+
+            ## Set description to joint chapter descriptions
+            self.story.setMetadata('description', " / ".join(descriptions))
+
+            ## Set the oldest date as publication date, the newest as update date
+            dates.sort()
+            self.story.setMetadata('datePublished', dates[0])
+            self.story.setMetadata('dateUpdated', dates[-1])
+
+            # normalize on first chapter URL.
+            self._setURL(self.chapterUrls[0][1])
+
+        self.story.setMetadata('numChapters', len(self.chapterUrls))
+                
+        # set storyId to 'title-author' to avoid duplicates
+        # self.story.setMetadata('storyId', 
+        #     re.sub("[^a-z0-9]", "", self.story.getMetadata('title').lower())
+        #     + "-"
+        #     + re.sub("[^a-z0-9]", "", self.story.getMetadata('author').lower()))
+
+        return
+
+    def getChapterText(self, url):
+        logger.debug('Getting chapter text from <%s>' % url)
+        data1 = self._fetchUrl(url)
+        soup1 = bs.BeautifulSoup(data1)
+
+        #strip comments from soup
+        [comment.extract() for comment in soup1.findAll(text=lambda text:isinstance(text, bs.Comment))]
+
+        # get story text
+        story1 = soup1.find('div', 'b-story-body-x').p
+        story1.name='div'
+        story1.append('<br />')
+        storytext = self.utf8FromSoup(url,story1)
+
+        # find num pages
+        pgs = int(soup1.find("span", "b-pager-caption-t r-d45").string.split(' ')[0])
+        logger.debug("pages: "+str(pgs))
+
+        # get all the pages
+        for i in xrange(2, pgs+1):
+            try:
+                logger.debug("fetching page "+str(i))
+                time.sleep(0.5)
+                data2 = self._fetchUrl(url, {'page': i})
+                soup2 = bs.BeautifulSoup(data2)
+                [comment.extract() for comment in soup2.findAll(text=lambda text:isinstance(text, bs.Comment))]
+                story2 = soup2.find('div', 'b-story-body-x').p
+                story2.name='div'
+                story2.append('<br />')
+                storytext += self.utf8FromSoup(url,story2)
+            except urllib2.HTTPError, e:
+                if e.code == 404:
+                    raise exceptions.StoryDoesNotExist(url)
+                else:
+                    raise e
+        return storytext
+
+
+def getClass():
+    return LiteroticaSiteAdapter
+
+
+

fanficdownloader/adapters/adapter_lotrfanfictioncom.py

@@ -0,0 +1,36 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2014 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Software: eFiction
+from base_efiction_adapter import BaseEfictionAdapter
+
+class TheLOTRFanFictionSiteAdapter(BaseEfictionAdapter):
+
+    @staticmethod
+    def getSiteDomain():
+        return 'lotrfanfiction.com'
+
+    @classmethod
+    def getSiteAbbrev(seluuf):
+        return 'lotrff'
+
+    @classmethod
+    def getDateFormat(self):
+        return "%d/%m/%y"
+
+def getClass():
+    return TheLOTRFanFictionSiteAdapter

fanficdownloader/adapters/adapter_lumossycophanthexcom.py

@@ -15,6 +15,7 @@
 # limitations under the License.
 #
 
+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -48,7 +49,7 @@ class LumosSycophantHexComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
 		
         
         # normalized story URL.
@@ -67,8 +68,8 @@ class LumosSycophantHexComAdapter(BaseSiteAdapter):
         return 'lumos.sycophanthex.com'
 
     @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
         return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
@@ -116,7 +117,7 @@ class LumosSycophantHexComAdapter(BaseSiteAdapter):
         ## Title
         pt = soup.find('div', {'id' : 'pagetitle'})
         a = pt.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.text)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = pt.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))

fanficdownloader/adapters/adapter_mediaminerorg.py

@@ -43,7 +43,7 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
         m = re.match(self.getSiteURLPattern(),url)
         if m:
             self.story.setMetadata('storyId',m.group('id'))
-            logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+            
             # normalized story URL.
             self._setURL('http://' + self.getSiteDomain() + '/fanfic/view_st.php/'+self.story.getMetadata('storyId'))
         else:
@@ -56,8 +56,8 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
         return 'www.mediaminer.org'
 
     @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/fanfic/view_st.php/123456 http://"+self.getSiteDomain()+"/fanfic/view_ch.php/1234123/123444#fic_c"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/fanfic/view_st.php/123456 http://"+cls.getSiteDomain()+"/fanfic/view_ch.php/1234123/123444#fic_c"
 
     def getSiteURLPattern(self):
         ##  http://www.mediaminer.org/fanfic/view_st.php/76882
@@ -107,10 +107,10 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
         for font in title.findAll('font'):
             font.extract() # removes 'font' tags from inside the td.        
         if title.has_key('colspan'):
-            titlet = title.text
+            titlet = stripHTML(title)
         else:
             ## No colspan, it's part chapter title--even if it's a one-shot.
-            titlet = ':'.join(title.text.split(':')[:-1]) # strip trailing 'Chapter X' or chapter title
+            titlet = ':'.join(stripHTML(title).split(':')[:-1]) # strip trailing 'Chapter X' or chapter title
         self.story.setMetadata('title',titlet)
         ## The story title is difficult to reliably parse from the
         ## story pages.  Getting it from the author page is, but costs

fanficdownloader/adapters/adapter_merlinficdtwinscouk.py

@@ -15,6 +15,7 @@
 # limitations under the License.
 #
 
+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -48,7 +49,7 @@ class MerlinFicDtwinsCoUk(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -66,8 +67,8 @@ class MerlinFicDtwinsCoUk(BaseSiteAdapter):
         return 'merlinfic.dtwins.co.uk'
 
     @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
         return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
@@ -170,7 +171,7 @@ class MerlinFicDtwinsCoUk(BaseSiteAdapter):
         
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))

fanficdownloader/adapters/adapter_midnightwhispersca.py

@@ -15,6 +15,7 @@
 # limitations under the License.
 #
 
+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -52,7 +53,7 @@ class MidnightwhispersCaAdapter(BaseSiteAdapter): # XXX
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
@@ -71,8 +72,8 @@ class MidnightwhispersCaAdapter(BaseSiteAdapter): # XXX
         return 'www.midnightwhispers.ca' # XXX
 
     @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
         return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"

fanficdownloader/adapters/adapter_mugglenetcom.py

@@ -15,6 +15,7 @@
 # limitations under the License.
 #
 
+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -52,7 +53,7 @@ class MuggleNetComAdapter(BaseSiteAdapter): # XXX
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -74,8 +75,8 @@ class MuggleNetComAdapter(BaseSiteAdapter): # XXX
         return ['fanfiction.mugglenet.com','fanfic.mugglenet.com']
 
     @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
         return re.escape("http://")+r"fanfic(tion)?\.mugglenet\.com"+re.escape("/viewstory.php?sid=")+r"\d+$"
@@ -193,7 +194,7 @@ class MuggleNetComAdapter(BaseSiteAdapter): # XXX
         
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))

fanficdownloader/adapters/adapter_nationallibrarynet.py

@@ -15,6 +15,7 @@
 # limitations under the License.
 #
 
+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -48,7 +49,7 @@ class NationalLibraryNetAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only storyid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?storyid='+self.story.getMetadata('storyId'))
@@ -69,9 +70,9 @@ class NationalLibraryNetAdapter(BaseSiteAdapter):
         return ['www.national-library.net','national-library.net']
 
     @classmethod
-    def getSiteExampleURLs(self):
+    def getSiteExampleURLs(cls):
         # ONLY the stories archived on or after June 17, 2006 and that are hosted on the website: 
-        return "http://"+self.getSiteDomain()+"/viewstory.php?storyid=1234"
+        return "http://"+cls.getSiteDomain()+"/viewstory.php?storyid=1234"
 
     def getSiteURLPattern(self):
         return re.escape("http://")+"(www\.)?"+re.escape(self.getSiteDomain()+"/viewstory.php?storyid=")+r"\d+$"
@@ -104,7 +105,7 @@ class NationalLibraryNetAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('h1')
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"authorresults.php\?author=\d+"))

fanficdownloader/adapters/adapter_ncisficcom.py

@@ -15,6 +15,7 @@
 # limitations under the License.
 #
 
+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -48,7 +49,7 @@ class NCISFicComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only storyid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?storyid='+self.story.getMetadata('storyId'))
@@ -69,8 +70,8 @@ class NCISFicComAdapter(BaseSiteAdapter):
         return ['www.ncisfic.com','ncisfic.com']
 
     @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?storyid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/viewstory.php?storyid=1234"
 
     def getSiteURLPattern(self):
         return re.escape("http://")+"(www\.)?"+re.escape(self.getSiteDomain()+"/viewstory.php?storyid=")+r"\d+$"
@@ -103,7 +104,7 @@ class NCISFicComAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('h1')
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"authorresults.php\?author=\d+"))

fanficdownloader/adapters/adapter_ncisfictionnet.py

@@ -37,11 +37,6 @@ class NCISFictionNetAdapter(BaseSiteAdapter):
     def __init__(self, config, url):
         BaseSiteAdapter.__init__(self, config, url)
 
-        # ncisfiction.net blocks IPs the default user-agent.  However,
-        # when asked, they said it was just general anti-spam, not
-        # targeted at us.  That lets me do this in good conscience:
-        self.opener.addheaders = [('User-agent', 'FFDL/1.6')]
-        
         self.decode = ["iso-8859-1",
                        "Windows-1252"] # 1252 is a superset of iso-8859-1.
                                # Most sites that claim to be
@@ -53,7 +48,7 @@ class NCISFictionNetAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL("http://"+self.getSiteDomain()\
@@ -80,8 +75,8 @@ class NCISFictionNetAdapter(BaseSiteAdapter):
         return ['www.ncisfiction.net','www.ncisfiction.com']
 
     @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/story.php?stid=01234 http://"+self.getSiteDomain()+"/chapters.php?stid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/story.php?stid=01234 http://"+cls.getSiteDomain()+"/chapters.php?stid=1234"
 
     def getSiteURLPattern(self):
         return r'http://www\.ncisfiction\.(net|com)/(chapters|story)?.php\?stid=\d+'

fanficdownloader/adapters/adapter_netraptororg.py

@@ -15,6 +15,7 @@
 # limitations under the License.
 #
 
+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -46,7 +47,7 @@ class NetRaptorOrgAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/fanfiction/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -64,8 +65,8 @@ class NetRaptorOrgAdapter(BaseSiteAdapter):
         return 'netraptor.org'
 
     @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/fanfiction/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/fanfiction/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
         return re.escape("http://"+self.getSiteDomain()+"/fanfiction/viewstory.php?sid=")+r"\d+$"
@@ -97,7 +98,7 @@ class NetRaptorOrgAdapter(BaseSiteAdapter):
         ## Title
         pagetitle = soup.find('div',{'id':'pagetitle'})
         a = pagetitle.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = pagetitle.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))

fanficdownloader/adapters/adapter_nfacommunitycom.py

@@ -15,6 +15,7 @@
 # limitations under the License.
 #
 
+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -52,7 +53,7 @@ class NfaCommunityComAdapter(BaseSiteAdapter): # XXX
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
@@ -75,8 +76,8 @@ class NfaCommunityComAdapter(BaseSiteAdapter): # XXX
         return 'nfacommunity.com' # XXX
 
     @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
         return "http://(www.)?"+re.escape(self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
@@ -156,7 +157,7 @@ class NfaCommunityComAdapter(BaseSiteAdapter): # XXX
         
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))

fanficdownloader/adapters/adapter_nhamagicalworldsus.py

@@ -15,6 +15,7 @@
 # limitations under the License.
 #
 
+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -48,7 +49,7 @@ class NHAMagicalWorldsUsAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
 		
         
         # normalized story URL.
@@ -59,7 +60,7 @@ class NHAMagicalWorldsUsAdapter(BaseSiteAdapter):
 
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = " %m/%d/%y"
+        self.dateformat = " %d/%m/%y"
             
     @staticmethod # must be @staticmethod, don't remove it.
     def getSiteDomain():
@@ -67,8 +68,8 @@ class NHAMagicalWorldsUsAdapter(BaseSiteAdapter):
         return 'nha.magical-worlds.us'
 
     @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
         return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
@@ -86,6 +87,28 @@ class NHAMagicalWorldsUsAdapter(BaseSiteAdapter):
             else:
                 raise e
             
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid & error in url.
+                addurl = addurl.replace("&","&")
+                url = self.url+'&index=1'+addurl
+                logger.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
         if "Access denied. This story has not been validated by the adminstrators of this site." in data:
             raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
             
@@ -115,7 +138,7 @@ class NHAMagicalWorldsUsAdapter(BaseSiteAdapter):
             a = info.find('a')
             if 'viewstory.php?sid='+self.story.getMetadata('storyId') == a['href'] or \
                     ('viewstory.php?sid='+self.story.getMetadata('storyId')+'&') in a['href']:
-                self.story.setMetadata('title',a.string)
+                self.story.setMetadata('title',stripHTML(a))
                 break
 		
 

fanficdownloader/adapters/adapter_nickandgregnet.py

@@ -15,6 +15,7 @@
 # limitations under the License.
 #
 
+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -48,7 +49,7 @@ class NickAndGregNetAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
@@ -67,8 +68,8 @@ class NickAndGregNetAdapter(BaseSiteAdapter):
         return 'www.nickandgreg.net'
 
     @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/desert_archive/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/desert_archive/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
         return re.escape("http://"+self.getSiteDomain()+"/desert_archive/viewstory.php?sid=")+r"\d+$"
@@ -101,7 +102,7 @@ class NickAndGregNetAdapter(BaseSiteAdapter):
 
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -166,10 +167,11 @@ class NickAndGregNetAdapter(BaseSiteAdapter):
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.find('table', {'class' : 'tblborder6'})
 
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
+        # wrap a div around it.
+        divsoup = bs.BeautifulStoneSoup('<div class="story"></div>',
+                                        selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        div = divsoup.find('div')
+        div.append(soup.find('table', {'class' : 'tblborder6'}))
+
         return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_nocturnallightnet.py

@@ -0,0 +1,177 @@
+import re
+import urllib2
+import urlparse
+
+from .. import BeautifulSoup
+
+from base_adapter import BaseSiteAdapter, makeDate
+from .. import exceptions
+
+
+def getClass():
+    return NocturnalLightNetAdapter
+
+
+# yields Tag _and_ NavigableString siblings from the given tag. The
+# BeautifulSoup findNextSiblings() method for some reasons only returns either
+# NavigableStrings _or_ Tag objects, not both.
+def _yield_next_siblings(tag):
+    sibling = tag.nextSibling
+    while sibling:
+        yield sibling
+        sibling = sibling.nextSibling
+
+
+class NocturnalLightNetAdapter(BaseSiteAdapter):
+    SITE_ABBREVIATION = 'nln'
+    SITE_DOMAIN = 'nocturnal-light.net'
+    BASE_URL = 'http://' + SITE_DOMAIN + '/fanfiction/'
+    STORY_URL_TEMPLATE = BASE_URL + 'story/%s'
+    AUTHORS_URL_TEMPLATE = BASE_URL + 'authors/%s'
+
+    DATETIME_FORMAT = '%m-%d-%y'
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        url_tokens = self.parsedUrl.path.split('/')
+        story_id = url_tokens[url_tokens.index('story') + 1]
+
+        self.story.setMetadata('storyId', story_id)
+        self._setURL(self.STORY_URL_TEMPLATE % story_id)
+        self.story.setMetadata('siteabbrev', self.SITE_ABBREVIATION)
+
+    def _customized_fetch_url(self, url, exception=None, parameters=None):
+        if exception:
+            try:
+                data = self._fetchUrl(url, parameters)
+            except urllib2.HTTPError:
+                raise exception(self.url)
+        # Just let self._fetchUrl throw the exception, don't catch and
+        # customize it.
+        else:
+            data = self._fetchUrl(url, parameters)
+
+        return BeautifulSoup.BeautifulSoup(data)
+
+    @staticmethod
+    def getSiteDomain():
+        return NocturnalLightNetAdapter.SITE_DOMAIN
+
+    @classmethod
+    def getSiteExampleURLs(cls):
+        return cls.STORY_URL_TEMPLATE % 1234
+
+    def getSiteURLPattern(self):
+        return re.escape(self.STORY_URL_TEMPLATE[:-2]) + r'\d+.*$'
+
+    def extractChapterUrlsAndMetadata(self):
+        soup = self._customized_fetch_url(self.url)
+
+        # Since no 404 error code we have to raise the exception ourselves.
+        # A title that is just 'by' indicates that there is no author name
+        # and no story title available.
+        if soup.title.string.strip() == 'by':
+            raise exceptions.StoryDoesNotExist(self.url)
+
+        # "storycontent" is found in a single-chapter story
+        author_anchor = soup.find('div', id=lambda id: id in ('main', 'storycontent')).h1.a
+        self.story.setMetadata('author', author_anchor.string)
+
+        url_tokens = author_anchor['href'].split('/')
+        author_id = url_tokens[url_tokens.index('authors')+1]
+        self.story.setMetadata('authorId', author_id)
+        self.story.setMetadata('authorUrl', self.AUTHORS_URL_TEMPLATE % author_id)
+
+        chapter_anchors = soup('a', href=lambda href: href and href.startswith('/fanfiction/story/'))
+        for chapter_anchor in chapter_anchors:
+            url = urlparse.urljoin(self.BASE_URL, chapter_anchor['href'])
+            self.chapterUrls.append((chapter_anchor.string, url))
+
+        author_url = urlparse.urljoin(self.BASE_URL, author_anchor['href'])
+        soup = self._customized_fetch_url(author_url)
+        story_id = self.story.getMetadata('storyId')
+        for listbox in soup('div', {'class': 'listbox'}):
+            url_tokens = listbox.a['href'].split('/')
+            # Found the div containing the story's metadata; break the loop and
+            # parse the element
+            if story_id == url_tokens[url_tokens.index('story')+1]:
+                break
+        else:
+            raise exceptions.FailedToDownload(self.url)
+
+        title = listbox.a.string
+        self.story.setMetadata('title', title)
+
+        # No chapter anchors found in the original story URL, so the story has
+        # only a single chapter.
+        if not chapter_anchors:
+            self.chapterUrls.append((title, self.url))
+
+        for b_tag in listbox('b'):
+            key = b_tag.string.strip(':')
+            try:
+                value = b_tag.nextSibling.string.replace('•', '').strip(': ')
+            # This can happen with some fancy markup in the summary. Just
+            # ignore this error and set value to None, the summary parsing
+            # takes care of this
+            except AttributeError:
+                value = None
+
+            if key == 'Summary':
+                contents = []
+                keep_summary_html = self.getConfig('keep_summary_html')
+
+                for sibling in _yield_next_siblings(b_tag):
+                    if isinstance(sibling, BeautifulSoup.Tag):
+                        if sibling.name == 'b' and sibling.findPreviousSibling().name == 'br':
+                            break
+
+                        if keep_summary_html:
+                            contents.append(self.utf8FromSoup(author_url, sibling))
+                        else:
+                            contents.append(''.join(sibling(text=True)))
+                    else:
+                        contents.append(sibling)
+
+                # Pop last break line tag
+                contents.pop()
+                self.story.setMetadata('description', ''.join(contents))
+
+            elif key == 'Category':
+                for sibling in b_tag.findNextSiblings(['a', 'b']):
+                    if sibling.name == 'b':
+                        break
+
+                    self.story.addToList('category', sibling.string)
+
+            elif key == 'Rating':
+                self.story.setMetadata('rating', value)
+
+            elif key == 'Chapters':
+                self.story.setMetadata('numChapters', int(value))
+
+                # Also parse reviews number which lies right after the chapters
+                # section
+                reviews_anchor = b_tag.findNextSibling('a')
+                reviews = reviews_anchor.string.split(' ')[1].strip('()')
+                self.story.setMetadata('reviews', reviews)
+
+            elif key == 'Completed':
+                self.story.setMetadata('status', 'Completed' if value == 'Yes' else 'In-Progress')
+
+            elif key == 'Date Added':
+                self.story.setMetadata('datePublished', makeDate(value, self.DATETIME_FORMAT))
+
+            elif key == 'Last Updated':
+                self.story.setMetadata('dateUpdated', makeDate(value, self.DATETIME_FORMAT))
+
+            elif key == 'Read':
+                self.story.setMetadata('readings', value.split()[0])
+
+        if self.story.getMetadata('rating') == 'NC-17' and not (self.is_adult or self.getConfig('is_adult')):
+            raise exceptions.AdultCheckRequired(self.url)
+
+    def getChapterText(self, url):
+        soup = self._customized_fetch_url(url)
+        return self.utf8FromSoup(url, soup.find('div', id='storytext'))

fanficdownloader/adapters/adapter_occlumencysycophanthexcom.py

@@ -15,6 +15,7 @@
 # limitations under the License.
 #
 
+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -48,7 +49,7 @@ class OcclumencySycophantHexComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
 		
         
         # normalized story URL.
@@ -67,8 +68,8 @@ class OcclumencySycophantHexComAdapter(BaseSiteAdapter):
         return 'occlumency.sycophanthex.com'
 
     @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
         return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
@@ -158,7 +159,7 @@ class OcclumencySycophantHexComAdapter(BaseSiteAdapter):
         for info in asoup.findAll('table', {'class' : 'border'}):
             a = info.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
             if a != None:
-                self.story.setMetadata('title',a.text)
+                self.story.setMetadata('title',stripHTML(a))
                 break
 		
 

fanficdownloader/adapters/adapter_onedirectionfanfictioncom.py

@@ -15,6 +15,7 @@
 # limitations under the License.
 #
 
+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -48,7 +49,7 @@ class OneDirectionFanfictionComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -71,8 +72,8 @@ class OneDirectionFanfictionComAdapter(BaseSiteAdapter):
 
 
     @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
         return re.escape("http://")+"(www\.)?"+re.escape(self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
@@ -158,7 +159,7 @@ class OneDirectionFanfictionComAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -192,7 +193,7 @@ class OneDirectionFanfictionComAdapter(BaseSiteAdapter):
             if 'Summary' in label:
                 ## Everything until the next span class='label'
                 svalue = ""
-                while not defaultGetattr(value,'class') == 'label':
+                while value and not defaultGetattr(value,'class') == 'label':
                     svalue += str(value)
                     value = value.nextSibling
                 self.setDescription(url,svalue)

fanficdownloader/adapters/adapter_phoenixsongnet.py

@@ -48,7 +48,7 @@ class PhoenixSongNetAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[3])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/fanfiction/story/' +self.story.getMetadata('storyId')+'/')
@@ -66,8 +66,8 @@ class PhoenixSongNetAdapter(BaseSiteAdapter):
         return 'www.phoenixsong.net'
 
     @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/fanfiction/story/1234/"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/fanfiction/story/1234/"
 
     def getSiteURLPattern(self):
         return re.escape("http://"+self.getSiteDomain()+"/fanfiction/story/")+r"\d+/?$"
@@ -136,7 +136,7 @@ class PhoenixSongNetAdapter(BaseSiteAdapter):
         ## Title
         b = soup.find('div', {'id' : 'nav25'})
         a = b.find('a', href=re.compile(r'fanfiction/story/'+self.story.getMetadata('storyId')+"/$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.  /fanfiction/stories.php?psid=125
         a = b.find('a', href=re.compile(r"/fanfiction/stories.php\?psid=\d+"))

fanficdownloader/adapters/adapter_pommedesangcom.py

@@ -15,6 +15,7 @@
 # limitations under the License.
 #
 
+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -48,7 +49,7 @@ class PommeDeSangComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # pommedesang.com has two 'sections', shown in URL as
         # 'efiction' and 'sds' that change how things should be
@@ -76,8 +77,8 @@ class PommeDeSangComAdapter(BaseSiteAdapter):
         return 'pommedesang.com'
 
     @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/efiction/viewstory.php?sid=1234 http://"+self.getSiteDomain()+"/sds/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/efiction/viewstory.php?sid=1234 http://"+cls.getSiteDomain()+"/sds/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
         return r"http://"+self.getSiteDomain()+"/(efiction|sds)?/viewstory.php\?sid=\d+$"
@@ -180,7 +181,7 @@ class PommeDeSangComAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('a', href=re.compile('viewstory.php\?sid=\d+'))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))

fanficdownloader/adapters/adapter_ponyfictionarchivenet.py

@@ -15,6 +15,7 @@
 # limitations under the License.
 #
 
+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -48,8 +49,7 @@ class PonyFictionArchiveNetAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
-        logger.info(self.parsedUrl.netloc)
+        
         # normalized story URL.
         if "explicit" in self.parsedUrl.netloc:
             self._setURL('http://explicit.' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -71,8 +71,8 @@ class PonyFictionArchiveNetAdapter(BaseSiteAdapter):
         return ['www.ponyfictionarchive.net','ponyfictionarchive.net','explicit.ponyfictionarchive.net']
 
     @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234 http://explicit."+self.getSiteDomain()+"/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234 http://explicit."+cls.getSiteDomain()+"/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
         return re.escape("http://")+"(www\.|explicit\.)?"+re.escape(self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
@@ -137,7 +137,7 @@ class PonyFictionArchiveNetAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))

fanficdownloader/adapters/adapter_portkeyorg.py

@@ -55,7 +55,7 @@ class PortkeyOrgAdapter(BaseSiteAdapter): # XXX
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/story/'+self.story.getMetadata('storyId'))
@@ -73,12 +73,19 @@ class PortkeyOrgAdapter(BaseSiteAdapter): # XXX
         return 'fanfiction.portkey.org' # XXX
 
     @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/story/1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/story/1234"
 
     def getSiteURLPattern(self):
         return re.escape("http://"+self.getSiteDomain()+"/story/")+r"\d+(/\d+)?$"
 
+    def use_pagecache(self):
+        '''
+        adapters that will work with the page cache need to implement
+        this and change it to True.
+        '''
+        return True
+    
     ## Getting the chapter list and the meta data, plus 'is adult' checking.
     def extractChapterUrlsAndMetadata(self):
 
@@ -88,7 +95,6 @@ class PortkeyOrgAdapter(BaseSiteAdapter): # XXX
         # portkey screws around with using a different URL to set the
         # cookie and it's a pain.  So... cheat!
         if self.is_adult or self.getConfig("is_adult"):
-            cookieproc = urllib2.HTTPCookieProcessor()
             cookie = cl.Cookie(version=0, name='verify17', value='1',
                                port=None, port_specified=False,
                                domain=self.getSiteDomain(), domain_specified=False, domain_initial_dot=False,
@@ -99,9 +105,8 @@ class PortkeyOrgAdapter(BaseSiteAdapter): # XXX
                                comment=None,
                                comment_url=None,
                                rest={'HttpOnly': None},
-                               rfc2109=False)
-            cookieproc.cookiejar.set_cookie(cookie)
-            self.opener = urllib2.build_opener(cookieproc)        
+                               rfc2109=False) 
+            self.cookiejar.set_cookie(cookie)
 
         try:
             data = self._fetchUrl(url)
@@ -251,8 +256,10 @@ class PortkeyOrgAdapter(BaseSiteAdapter): # XXX
 
         logger.debug('Getting chapter text from: %s' % url)
         data = self._fetchUrl(url)
-        soup = bs.BeautifulStoneSoup(data,
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+
+        data = data.replace("HTML>","div>")
+        
+        soup = bs.BeautifulSoup(data)
 
         #print("soup:%s"%soup)
         tag = soup.find('td', {'class' : 'story'})

fanficdownloader/adapters/adapter_potionsandsnitchesnet.py

@@ -15,6 +15,7 @@
 # limitations under the License.
 #
 
+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -41,7 +42,7 @@ class PotionsAndSnitchesNetSiteAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/fanfiction/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -56,7 +57,7 @@ class PotionsAndSnitchesNetSiteAdapter(BaseSiteAdapter):
         return ['www.potionsandsnitches.net','potionsandsnitches.net']
 
     @classmethod
-    def getSiteExampleURLs(self):
+    def getSiteExampleURLs(cls):
         return "http://www.potionsandsnitches.net/fanfiction/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
@@ -83,7 +84,7 @@ class PotionsAndSnitchesNetSiteAdapter(BaseSiteAdapter):
 
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
@@ -163,12 +164,11 @@ class PotionsAndSnitchesNetSiteAdapter(BaseSiteAdapter):
                     self.story.setMetadata('status', 'In-Progress')
 
             if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), "%d %b %Y"))
+                # limit date values, there's some extra chars.
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value[:12]), "%d %b %Y"))
             
             if 'Updated' in label:
-                # there's a stray [ at the end.
-                #value = value[0:-1]
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), "%d %b %Y"))
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value[:12]), "%d %b %Y"))
 
         try:
             # Find Series name from series URL.

fanficdownloader/adapters/adapter_potterficscom.py

@@ -54,7 +54,7 @@ class PotterFicsComAdapter(BaseSiteAdapter):
         m = re.match(self.getSiteURLPattern(),url)
         if m:
             self.story.setMetadata('storyId',m.group('id'))
-            logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+            
             # normalized story URL. gets rid of chapter if there, left with chapter index URL
             nurl = "http://"+self.getSiteDomain()+"/historias/"+self.story.getMetadata('storyId')
             self._setURL(nurl)
@@ -62,7 +62,7 @@ class PotterFicsComAdapter(BaseSiteAdapter):
             raise exceptions.InvalidStoryURL(url,
                                              self.getSiteDomain(),
                                              self.getSiteExampleURLs())
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','potficscom')
@@ -73,7 +73,7 @@ class PotterFicsComAdapter(BaseSiteAdapter):
         return 'www.potterfics.com'
 
     @classmethod
-    def getSiteExampleURLs(self):
+    def getSiteExampleURLs(cls):
         return "http://www.potterfics.com/historias/12345 http://www.potterfics.com/historias/12345/capitulo-1 "
 
     def getSiteURLPattern(self):
@@ -147,7 +147,7 @@ class PotterFicsComAdapter(BaseSiteAdapter):
         #find first metadata block
         mb = cell.div.findNextSibling('div')
         #Get meta...
-        self.story.setMetadata('title', mb.b.string)
+        self.story.setMetadata('title', stripHTML(mb.b))
         #strip out brackets on rating
         self.story.setMetadata('rating', mb.span.string[1:-1])
         #Completion status is denoted by the presence of this image:

fanficdownloader/adapters/adapter_potterheadsanonymouscom.py

@@ -15,6 +15,7 @@
 # limitations under the License.
 #
 
+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -48,7 +49,7 @@ class PotterHeadsAnonymousComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -66,8 +67,8 @@ class PotterHeadsAnonymousComAdapter(BaseSiteAdapter):
         return 'fanfic.potterheadsanonymous.com'
 
     @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
         return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
@@ -181,7 +182,7 @@ class PotterHeadsAnonymousComAdapter(BaseSiteAdapter):
         
         ## Title
         a = pagetitle.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = pagetitle.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))

fanficdownloader/adapters/adapter_pretendercentrecom.py

@@ -15,6 +15,7 @@
 # limitations under the License.
 #
 
+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -48,7 +49,7 @@ class PretenderCenterComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/missingpieces/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -70,8 +71,8 @@ class PretenderCenterComAdapter(BaseSiteAdapter):
         return ['www.pretendercentre.com','pretendercentre.com']
 
     @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/missingpieces/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/missingpieces/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
         return re.escape("http://")+"(www\.)?"+re.escape(self.getSiteDomain()+"/missingpieces/viewstory.php?sid=")+r"\d+$"
@@ -134,7 +135,7 @@ class PretenderCenterComAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.text)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))

fanficdownloader/adapters/adapter_psychficcom.py

@@ -15,6 +15,7 @@
 # limitations under the License.
 #
 
+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -48,7 +49,7 @@ class PsychFicComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -66,8 +67,8 @@ class PsychFicComAdapter(BaseSiteAdapter):
         return 'www.psychfic.com'
 
     @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
         return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
@@ -130,7 +131,7 @@ class PsychFicComAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))

fanficdownloader/adapters/adapter_qafficcom.py

@@ -15,6 +15,7 @@
 # limitations under the License.
 #
 
+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -48,7 +49,7 @@ class QafFicComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/atp/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -66,8 +67,8 @@ class QafFicComAdapter(BaseSiteAdapter):
         return 'www.qaf-fic.com'
 
     @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/atp/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/atp/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
         return re.escape("http://"+self.getSiteDomain()+"/atp/viewstory.php?sid=")+r"\d+$"
@@ -138,7 +139,7 @@ class QafFicComAdapter(BaseSiteAdapter):
         self.story.setMetadata('author',aut.string)
         aut.extract()
         
-        self.story.setMetadata('title',a.string[:(len(a.string)-3)])
+        self.story.setMetadata('title',stripHTML(a)[:(len(a.string)-3)])
 
         # Find the chapters:
         chapters=soup.find('select')

fanficdownloader/adapters/adapter_restrictedsectionorg.py

@@ -22,7 +22,6 @@ import re
 import urllib2
 import cookielib as cl
 from datetime import datetime
-import json
 
 from .. import BeautifulSoup as bs
 from ..htmlcleanup import stripHTML
@@ -52,10 +51,9 @@ class RestrictedSectionOrgSiteAdapter(BaseSiteAdapter):
         m = re.match(self.getSiteURLPattern(),url)
         if m:
             self.story.setMetadata('storyId',m.group('id'))
-            logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+            
             # normalized story URL.
             self._setURL('http://' + self.getSiteDomain() + '/' + m.group('filestory') + '.php?' + m.group('filestory') + '=' + self.story.getMetadata('storyId'))
-            logger.debug("storyUrl: (%s)"%self.story.getMetadata('storyUrl'))
         else:
             raise exceptions.InvalidStoryURL(url,
                                              self.getSiteDomain(),
@@ -71,8 +69,8 @@ class RestrictedSectionOrgSiteAdapter(BaseSiteAdapter):
         return 'www.restrictedsection.org'
 
     @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/story.php?story=1234 http://"+self.getSiteDomain()+"/file.php?file=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/story.php?story=1234 http://"+cls.getSiteDomain()+"/file.php?file=1234"
 
     def getSiteURLPattern(self):
         return re.escape("http://"+self.getSiteDomain())+r"/(?P<filestory>file|story).php\?(file|story)=(?P<id>\d+)$"
@@ -127,10 +125,12 @@ class RestrictedSectionOrgSiteAdapter(BaseSiteAdapter):
         
         self.story.setMetadata('authorId',ahref.split('=')[1])
         self.story.setMetadata('authorUrl','http://'+self.host+'/'+ahref)
-        self.story.setMetadata('author',a.string)
+        self.story.setMetadata('author',stripHTML(a))
 
         # title, remove byauthorname.
-        self.story.setMetadata('title',h2.text[:h2.text.index("by"+a.string)])
+        auth=stripHTML(a)
+        title=stripHTML(h2)
+        self.story.setMetadata('title',title[:title.index(" by "+auth)])
 
         dates = soup.findAll('span', {'class':'date'})
         if dates: # only for multi-chapter

fanficdownloader/adapters/adapter_samandjacknet.py

@@ -0,0 +1,342 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+# By virtue of being recent and requiring both is_adult and user/pass,
+# adapter_fanficcastletvnet.py is the best choice for learning to
+# write adapters--especially for sites that use the eFiction system.
+# Most sites that have ".../viewstory.php?sid=123" in the story URL
+# are eFiction.
+
+# For non-eFiction sites, it can be considerably more complex, but
+# this is still a good starting point.
+
+# In general an 'adapter' needs to do these five things:
+
+# - 'Register' correctly with the downloader
+# - Site Login (if needed)
+# - 'Are you adult?' check (if needed--some do one, some the other, some both)
+# - Grab the chapter list
+# - Grab the story meta-data (some (non-eFiction) adapters have to get it from the author page)
+# - Grab the chapter texts
+
+# Search for XXX comments--that's where things are most likely to need changing.
+
+# This function is called by the downloader in all adapter_*.py files
+# in this dir to register the adapter class.  So it needs to be
+# updated to reflect the class below it.  That, plus getSiteDomain()
+# take care of 'Registering'.
+def getClass():
+    return SamAndJackNetAdapter # XXX
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class SamAndJackNetAdapter(BaseSiteAdapter): # XXX
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        
+        
+        # normalized story URL.
+        # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
+        self._setURL('http://' + self.getSiteDomain() + '/fanfics/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','sjn') # XXX
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%b %d, %Y" # XXX
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'samandjack.net' # XXX
+
+    @classmethod
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/fanfics/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/fanfics/viewstory.php?sid=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Registered Users Only' in data \
+                or 'There is no such account on our website' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['cookiecheck'] = '1'
+        params['submit'] = 'Submit'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/fanfics/user.php?action=login'
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "Member Account" not in d : #Member Account
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+
+            # Furthermore, there's a couple sites now with more than
+            # one warning level for different ratings.  And they're
+            # fussy about it.  midnightwhispers has three: 10, 3 & 5.
+            # we'll try 5 first.
+            addurl = "&ageconsent=ok&warning=5" # XXX
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        # The actual text that is used to announce you need to be an
+        # adult varies from site to site.  Again, print data before
+        # the title search to troubleshoot.
+            
+        # Since the warning text can change by warning level, let's
+        # look for the warning pass url.  nfacommunity uses
+        # &warning= -- actually, so do other sites.  Must be an
+        # eFiction book.
+            
+        # viewstory.php?sid=1882&warning=4
+        # viewstory.php?sid=1654&ageconsent=ok&warning=5
+        #print data
+        #m = re.search(r"'viewstory.php\?sid=1882(&warning=4)'",data)
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid & error in url.
+                addurl = addurl.replace("&","&")
+                url = self.url+'&index=1'+addurl
+                logger.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+
+        pagetitle = soup.find('div',{'id':'pagetitle'})
+        ## Title
+        a = pagetitle.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',stripHTML(a))
+             
+        # Find authorid and URL from... author url.
+        # (fetch multiple authors)
+        alist = soup.findAll('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        for a in alist:
+            self.story.addToList('authorId',a['href'].split('=')[1])
+            self.story.addToList('authorUrl','http://'+self.host+'/fanfics/'+a['href'])
+            self.story.addToList('author',a.string)
+                             
+        # Reviews
+        reviewdata = soup.find('div', {'id' : 'sort'})
+        a = reviewdata.findAll('a', href=re.compile(r'reviews.php\?type=ST&(amp;)?item='+self.story.getMetadata('storyId')+"$"))[1] # second one.
+        self.story.setMetadata('reviews',stripHTML(a))              
+        
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/fanfics/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+                
+               
+        # <span class="label">Rated:</span> NC-17<br /> etc 
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string  
+            
+            if 'Summary' in label:
+                self.setDescription(url,value)                     
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                catstext = [cat.string for cat in cats]
+                for cat in catstext:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                charstext = [char.string for char in chars]
+                for char in charstext:
+                    self.story.addToList('characters',char.string)
+
+            ## Not all sites use Genre, but there's no harm to
+            ## leaving it in.  Check to make sure the type_id number
+            ## is correct, though--it's site specific.
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) # XXX
+                genrestext = [genre.string for genre in genres]
+                self.genre = ', '.join(genrestext)
+                for genre in genrestext:
+                    self.story.addToList('genre',genre.string)
+
+            ## Not all sites use Warnings, but there's no harm to
+            ## leaving it in.  Check to make sure the type_id number
+            ## is correct, though--it's site specific.
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
+                warningstext = [warning.string for warning in warnings]
+                self.warning = ', '.join(warningstext)
+                for warning in warningstext:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                value=value.replace(' | ','')
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+                         
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/fanfics/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_samdeanarchivenu.py

@@ -15,6 +15,7 @@
 # limitations under the License.
 #
 
+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -48,7 +49,7 @@ class SamDeanArchiveNuAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -70,8 +71,8 @@ class SamDeanArchiveNuAdapter(BaseSiteAdapter):
         return ['www.samdean.archive.nu','samdean.archive.nu']
 
     @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
         return re.escape("http://")+r"(www\.)?"+re.escape(self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
@@ -110,7 +111,7 @@ class SamDeanArchiveNuAdapter(BaseSiteAdapter):
         self.story.setMetadata('author',aut.string)
         aut.extract()
         
-        self.story.setMetadata('title',a.string[:(len(a.string)-3)])
+        self.story.setMetadata('title',stripHTML(a)[:(len(stripHTML(a))-3)])
 
         # Find the chapters:
         chapters=soup.find('select')

fanficdownloader/adapters/adapter_scarheadnet.py

@@ -15,6 +15,7 @@
 # limitations under the License.
 #
 
+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -48,7 +49,7 @@ class ScarHeadNetAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -66,8 +67,8 @@ class ScarHeadNetAdapter(BaseSiteAdapter):
         return 'scarhead.net'
 
     @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
         return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
@@ -181,7 +182,7 @@ class ScarHeadNetAdapter(BaseSiteAdapter):
         
         ## Title
         a = pagetitle.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = pagetitle.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))

fanficdownloader/adapters/adapter_scarvesandcoffeenet.py

@@ -15,6 +15,7 @@
 # limitations under the License.
 #
 
+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -48,7 +49,7 @@ class ScarvesAndCoffeeNetAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
@@ -66,8 +67,8 @@ class ScarvesAndCoffeeNetAdapter(BaseSiteAdapter):
         return 'www.scarvesandcoffee.net'
 
     @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
         return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
@@ -139,7 +140,7 @@ class ScarvesAndCoffeeNetAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('div',{"id":"pagetitle"}).find('a', href=re.compile(r"viewuser.php\?uid=\d+"))

fanficdownloader/adapters/adapter_sg1heliopoliscom.py

@@ -15,6 +15,7 @@
 # limitations under the License.
 #
 
+# Software: eFiction
 import time
 import logging
 logger = logging.getLogger(__name__)
@@ -48,7 +49,7 @@ class SG1HeliopolisComAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query is only sid=1234
         self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        logger.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+        
         self.section=self.parsedUrl.path.split('/',)[1]
         
         # normalized story URL.
@@ -75,8 +76,8 @@ class SG1HeliopolisComAdapter(BaseSiteAdapter):
         return 'sg1-heliopolis.com'
 
     @classmethod
-    def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/archive/viewstory.php?sid=1234 http://"+self.getSiteDomain()+"/adult/viewstory.php?sid=1234 http://"+self.getSiteDomain()+"/atlantis/viewstory.php?sid=1234"
+    def getSiteExampleURLs(cls):
+        return "http://"+cls.getSiteDomain()+"/archive/viewstory.php?sid=1234 http://"+cls.getSiteDomain()+"/adult/viewstory.php?sid=1234 http://"+cls.getSiteDomain()+"/atlantis/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
         return r"http://sg1-heliopolis.com/(archive|adult|atlantis)?/viewstory.php\?sid=\d+$"
@@ -140,7 +141,7 @@ class SG1HeliopolisComAdapter(BaseSiteAdapter):
         
         ## Title
         a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',a.string)
+        self.story.setMetadata('title',stripHTML(a))
         
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))

fanficdownloader/adapters/adapter_sheppardweircom.py

@@ -0,0 +1,317 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+# By virtue of being recent and requiring both is_adult and user/pass,
+# adapter_fanficcastletvnet.py is the best choice for learning to
+# write adapters--especially for sites that use the eFiction system.
+# Most sites that have ".../viewstory.php?sid=123" in the story URL
+# are eFiction.
+
+# For non-eFiction sites, it can be considerably more complex, but
+# this is still a good starting point.
+
+# In general an 'adapter' needs to do these five things:
+
+# - 'Register' correctly with the downloader
+# - Site Login (if needed)
+# - 'Are you adult?' check (if needed--some do one, some the other, some both)
+# - Grab the chapter list
+# - Grab the story meta-data (some (non-eFiction) adapters have to get it from the author page)
+# - Grab the chapter texts
+
+# Search for XXX comments--that's where things are most likely to need changing.
+
+# This function is called by the downloader in all adapter_*.py files
+# in this dir to register the adapter class.  So it needs to be
+# updated to reflect the class below it.  That, plus getSiteDomain()
+# take care of 'Registering'.
+def getClass():
+    return SheppardWeirComAdapter # XXX
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class SheppardWeirComAdapter(BaseSiteAdapter): # XXX
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        
+        
+        # normalized story URL.
+        # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
+        self._setURL('http://' + self.getSiteDomain() + '/fanfics/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','swf') # XXX
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%B %d, %Y" # XXX
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'sheppardweir.com' # XXX
+
+    @classmethod
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/fanfics/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain()+"/fanfics/viewstory.php?sid=")+r"\d+$"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Registered Users Only' in data \
+                or 'There is no such account on our website' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['cookiecheck'] = '1'
+        params['submit'] = 'Submit'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/fanfics/user.php?action=login'
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "Member Account" not in d : #Member Account
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&ageconsent=ok&warning=4" # XXX
+        else:
+            addurl=""
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+'&index=1'+addurl
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+
+        # The actual text that is used to announce you need to be an
+        # adult varies from site to site.  Again, print data before
+        # the title search to troubleshoot.
+        if "Age Consent Required" in data: # XXX 
+            raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        # print data
+
+        # Now go hunting for all the meta data and the chapter list.
+
+        pagetitle = soup.find('div',{'id':'pagetitle'})
+        ## Title
+        a = pagetitle.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
+        self.story.setMetadata('title',stripHTML(a))
+        
+        # Find authorid and URL from... author url.
+        # (fetch multiple authors)
+        alist = soup.findAll('a', href=re.compile(r"viewuser.php\?uid=\d+"))
+        for a in alist:
+            self.story.addToList('authorId',a['href'].split('=')[1])
+            self.story.addToList('authorUrl','http://'+self.host+'/fanfics/'+a['href'])
+            self.story.addToList('author',a.string)
+        
+        
+        # Reviews
+        reviewdata = soup.find('div', {'id' : 'sort'})
+        a = reviewdata.findAll('a', href=re.compile(r'reviews.php\?type=ST&(amp;)?item='+self.story.getMetadata('storyId')+"$"))[1] # second one.
+        self.story.setMetadata('reviews',stripHTML(a))  
+
+
+        # Find the chapters:
+        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
+            # just in case there's tags, like <i> in chapter titles.
+            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/fanfics/'+chapter['href']+addurl))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        # utility method
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+                
+        # Summary        
+        summarydata = unicode(soup.find('div',{'class':'content'}))
+        start='<span class="label">Summary: </span>'
+        end='</div>'
+        summarydata = summarydata[summarydata.index(start)+len(start):summarydata.rindex(end)]
+        self.setDescription(url,bs.BeautifulSoup(summarydata)) 
+        
+        # <span class="label">Rated:</span> NC-17<br /> etc 
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string                        
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                catstext = [cat.string for cat in cats]
+                for cat in catstext:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                charstext = [char.string for char in chars]
+                for char in charstext:
+                    self.story.addToList('characters',char.string)
+
+            ## Not all sites use Genre, but there's no harm to
+            ## leaving it in.  Check to make sure the type_id number
+            ## is correct, though--it's site specific.
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1')) # XXX
+                genrestext = [genre.string for genre in genres]
+                self.genre = ', '.join(genrestext)
+                for genre in genrestext:
+                    self.story.addToList('genre',genre.string)
+
+            ## Not all sites use Warnings, but there's no harm to
+            ## leaving it in.  Check to make sure the type_id number
+            ## is correct, though--it's site specific.
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
+                warningstext = [warning.string for warning in warnings]
+                self.warning = ', '.join(warningstext)
+                for warning in warningstext:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                value=value.replace(' - ','')
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+            
+            if 'Updated' in label:
+                # there's a stray [ at the end.
+                #value = value[0:-1]
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/fanfics/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)