Update CLI download zip.

Also don't retry 404 errors and bump versions.

app.yaml

@@ -1,6 +1,6 @@
 # ffd-retief-hrd fanfictiondownloader
 application: fanfictiondownloader
-version: 4-4-77
+version: 4-4-96
 runtime: python27
 api_version: 1
 threadsafe: true

calibre-plugin/__init__.py

@@ -17,6 +17,12 @@ if sys.version_info >= (2, 7):
     loghandler.setLevel(logging.DEBUG)
     logger.setLevel(logging.DEBUG)
 
+# pulls in translation files for _() strings
+try:
+    load_translations()
+except NameError:
+    pass # load_translations() added in calibre 1.9
+
 # The class that all Interface Action plugin wrappers must inherit from
 from calibre.customize import InterfaceActionBase
 
@@ -33,11 +39,11 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
     calibre utilities to run without needing to load the GUI libraries.
     '''
     name                = 'FanFictionDownLoader'
-    description         = 'UI plugin to download FanFiction stories from various sites.'
+    description         = _('UI plugin to download FanFiction stories from various sites.')
     supported_platforms = ['windows', 'osx', 'linux']
     author              = 'Jim Miller'
-    version             = (1, 7, 48)
-    minimum_calibre_version = (0, 8, 57)
+    version             = (1, 8, 15)
+    minimum_calibre_version = (1, 13, 0)
 
     #: This field defines the GUI plugin class that contains all the code
     #: that actually does something. Its format is module_path:class_name

calibre-plugin/config.py

@@ -22,13 +22,40 @@ from calibre.gui2.ui import get_gui
 from calibre.gui2 import dynamic, info_dialog
 from calibre.constants import numeric_version as calibre_version
 
+# pulls in translation files for _() strings
+try:
+    load_translations()
+except NameError:
+    pass # load_translations() added in calibre 1.9
+
+# There are a number of things used several times that shouldn't be
+# translated.  This is just a way to make that easier by keeping them
+# out of the _() strings.
+# I'm tempted to override _() to include them...
+no_trans = { 'pini':'personal.ini',
+             'imgset':'\n\n[epub]\ninclude_images:true\nkeep_summary_html:true\nmake_firstimage_cover:true\n\n',
+             'gcset':'generate_cover_settings',
+             'ccset':'custom_columns_settings',
+             'gc':'Generate Cover',
+             'rl':'Reading List',
+             'cp':'Count Pages',
+             'cmplt':'Completed',
+             'inprog':'In-Progress',
+             'lul':'Last Updated',
+             'lus':'lastupdate',
+             'is':'include_subject',
+             'isa':'is_adult',
+             'u':'username',
+             'p':'password',
+             }
+
 from calibre_plugins.fanfictiondownloader_plugin.prefs import prefs, PREFS_NAMESPACE
 from calibre_plugins.fanfictiondownloader_plugin.dialogs \
-    import (UPDATE, UPDATEALWAYS, OVERWRITE, collision_order, RejectListDialog,
+    import (UPDATE, UPDATEALWAYS, collision_order, save_collisions, RejectListDialog,
             EditTextDialog, RejectUrlEntry)
     
 from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.adapters \
-    import (getConfigSections, getNormalStoryURL)
+    import getConfigSections
 
 from calibre_plugins.fanfictiondownloader_plugin.common_utils \
     import ( KeyboardConfigDialog, PrefsViewerDialog )
@@ -125,15 +152,21 @@ class ConfigWidget(QWidget):
         self.l = QVBoxLayout()
         self.setLayout(self.l)
 
-        label = QLabel('<a href="http://code.google.com/p/fanficdownloader/wiki/FanFictionDownloaderSupportedsites">List of Supported Sites</a> -- <a href="http://code.google.com/p/fanficdownloader/wiki/FanFictionDownloaderFAQs">FAQs</a>')
+        label = QLabel('<a href="http://code.google.com/p/fanficdownloader/wiki/FanFictionDownloaderSupportedsites">'+_('List of Supported Sites')+'</a> -- <a href="http://code.google.com/p/fanficdownloader/wiki/FanFictionDownloaderFAQs">'+_('FAQs')+'</a>')
         label.setOpenExternalLinks(True)
         self.l.addWidget(label)
 
-        tab_widget = QTabWidget(self)
-        self.l.addWidget(tab_widget)
 
+        self.scroll_area = QScrollArea(self)
+        self.scroll_area.setFrameShape(QScrollArea.NoFrame)
+        self.scroll_area.setWidgetResizable(True)
+        self.l.addWidget(self.scroll_area)
+
+        tab_widget = QTabWidget(self)
+        self.scroll_area.setWidget(tab_widget)
+        
         self.basic_tab = BasicTab(self, plugin_action)
-        tab_widget.addTab(self.basic_tab, 'Basic')
+        tab_widget.addTab(self.basic_tab, _('Basic'))
 
         self.personalini_tab = PersonalIniTab(self, plugin_action)
         tab_widget.addTab(self.personalini_tab, 'personal.ini')
@@ -154,20 +187,20 @@ class ConfigWidget(QWidget):
             self.countpages_tab.setEnabled(False)
 
         self.std_columns_tab = StandardColumnsTab(self, plugin_action)
-        tab_widget.addTab(self.std_columns_tab, 'Standard Columns')
+        tab_widget.addTab(self.std_columns_tab, _('Standard Columns'))
 
         self.cust_columns_tab = CustomColumnsTab(self, plugin_action)
-        tab_widget.addTab(self.cust_columns_tab, 'Custom Columns')
+        tab_widget.addTab(self.cust_columns_tab, _('Custom Columns'))
 
         self.other_tab = OtherTab(self, plugin_action)
-        tab_widget.addTab(self.other_tab, 'Other')
+        tab_widget.addTab(self.other_tab, _('Other'))
 
 
     def save_settings(self):
 
         # basic
         prefs['fileform'] = unicode(self.basic_tab.fileform.currentText())
-        prefs['collision'] = unicode(self.basic_tab.collision.currentText())
+        prefs['collision'] = save_collisions[unicode(self.basic_tab.collision.currentText())]
         prefs['updatemeta'] = self.basic_tab.updatemeta.isChecked()
         prefs['updatecover'] = self.basic_tab.updatecover.isChecked()
         prefs['updateepubcover'] = self.basic_tab.updateepubcover.isChecked()
@@ -176,6 +209,7 @@ class ConfigWidget(QWidget):
         prefs['suppresstitlesort'] = self.basic_tab.suppresstitlesort.isChecked()
         prefs['mark'] = self.basic_tab.mark.isChecked()
         prefs['showmarked'] = self.basic_tab.showmarked.isChecked()
+        prefs['autoconvert'] = self.basic_tab.autoconvert.isChecked()
         prefs['urlsfromclip'] = self.basic_tab.urlsfromclip.isChecked()
         prefs['updatedefault'] = self.basic_tab.updatedefault.isChecked()
         prefs['deleteotherforms'] = self.basic_tab.deleteotherforms.isChecked()
@@ -215,6 +249,7 @@ class ConfigWidget(QWidget):
                 #print("gc_site_settings[%s]:%s"%(site,gc_site_settings[site]))
         prefs['gc_site_settings'] = gc_site_settings
         prefs['allow_gc_from_ini'] = self.generatecover_tab.allow_gc_from_ini.isChecked()
+        prefs['gc_polish_cover'] = self.generatecover_tab.gc_polish_cover.isChecked()
 
         # Count Pages tab
         countpagesstats = []
@@ -278,17 +313,17 @@ class BasicTab(QWidget):
         topl = QVBoxLayout()
         self.setLayout(topl)
 
-        label = QLabel('These settings control the basic features of the plugin--downloading FanFiction.')
+        label = QLabel(_('These settings control the basic features of the plugin--downloading FanFiction.'))
         label.setWordWrap(True)
         topl.addWidget(label)
 
-        defs_gb = groupbox = QGroupBox("Defaults Options on Download")
+        defs_gb = groupbox = QGroupBox(_("Defaults Options on Download"))
         self.l = QVBoxLayout()
         groupbox.setLayout(self.l)
 
-        tooltip = "On each download, FFDL offers an option to select the output format. <br />This sets what that option will default to."
+        tooltip = _("On each download, FFDL offers an option to select the output format. <br />This sets what that option will default to.")
         horz = QHBoxLayout()
-        label = QLabel('Default Output &Format:')
+        label = QLabel(_('Default Output &Format:'))
         label.setToolTip(tooltip)
         horz.addWidget(label)
         self.fileform = QComboBox(self)
@@ -303,15 +338,15 @@ class BasicTab(QWidget):
         horz.addWidget(self.fileform)
         self.l.addLayout(horz)
 
-        tooltip = "On each download, FFDL offers an option of what happens if that story already exists. <br />This sets what that option will default to."
+        tooltip = _("On each download, FFDL offers an option of what happens if that story already exists. <br />This sets what that option will default to.")
         horz = QHBoxLayout()
-        label = QLabel('Default If Story Already Exists?')
+        label = QLabel(_('Default If Story Already Exists?'))
         label.setToolTip(tooltip)
         horz.addWidget(label)
         self.collision = QComboBox(self)
         # add collision options
         self.set_collisions()
-        i = self.collision.findText(prefs['collision'])
+        i = self.collision.findText(save_collisions[prefs['collision']])
         if i > -1:
             self.collision.setCurrentIndex(i)
         self.collision.setToolTip(tooltip)
@@ -319,142 +354,148 @@ class BasicTab(QWidget):
         horz.addWidget(self.collision)
         self.l.addLayout(horz)
 
-        self.updatemeta = QCheckBox('Default Update Calibre &Metadata?',self)
-        self.updatemeta.setToolTip("On each download, FFDL offers an option to update Calibre's metadata (title, author, URL, tags, custom columns, etc) from the web site. <br />This sets whether that will default to on or off. <br />Columns set to 'New Only' in the column tabs will only be set for new books.")
+        self.updatemeta = QCheckBox(_('Default Update Calibre &Metadata?'),self)
+        self.updatemeta.setToolTip(_("On each download, FFDL offers an option to update Calibre's metadata (title, author, URL, tags, custom columns, etc) from the web site. <br />This sets whether that will default to on or off. <br />Columns set to 'New Only' in the column tabs will only be set for new books."))
         self.updatemeta.setChecked(prefs['updatemeta'])
         self.l.addWidget(self.updatemeta)
 
-        self.updateepubcover = QCheckBox('Default Update EPUB Cover when Updating EPUB?',self)
-        self.updateepubcover.setToolTip("On each download, FFDL offers an option to update the book cover image <i>inside</i> the EPUB from the web site when the EPUB is updated.<br />This sets whether that will default to on or off.")
+        self.updateepubcover = QCheckBox(_('Default Update EPUB Cover when Updating EPUB?'),self)
+        self.updateepubcover.setToolTip(_("On each download, FFDL offers an option to update the book cover image <i>inside</i> the EPUB from the web site when the EPUB is updated.<br />This sets whether that will default to on or off."))
         self.updateepubcover.setChecked(prefs['updateepubcover'])
         self.l.addWidget(self.updateepubcover)
 
-        self.smarten_punctuation = QCheckBox('Smarten Punctuation (EPUB only)',self)
-        self.smarten_punctuation.setToolTip("Run Smarten Punctuation from Calibre's Polish Book feature on each EPUB download and update.")
+        self.smarten_punctuation = QCheckBox(_('Smarten Punctuation (EPUB only)'),self)
+        self.smarten_punctuation.setToolTip(_("Run Smarten Punctuation from Calibre's Polish Book feature on each EPUB download and update."))
         self.smarten_punctuation.setChecked(prefs['smarten_punctuation'])
         if calibre_version >= (0, 9, 39):
             self.l.addWidget(self.smarten_punctuation)
 
-        cali_gb = groupbox = QGroupBox("Updating Calibre Options")
+        cali_gb = groupbox = QGroupBox(_("Updating Calibre Options"))
         self.l = QVBoxLayout()
         groupbox.setLayout(self.l)
 
-        self.deleteotherforms = QCheckBox('Delete other existing formats?',self)
-        self.deleteotherforms.setToolTip('Check this to automatically delete all other ebook formats when updating an existing book.\nHandy if you have both a Nook(epub) and Kindle(mobi), for example.')
+        self.deleteotherforms = QCheckBox(_('Delete other existing formats?'),self)
+        self.deleteotherforms.setToolTip(_('Check this to automatically delete all other ebook formats when updating an existing book.\nHandy if you have both a Nook(epub) and Kindle(mobi), for example.'))
         self.deleteotherforms.setChecked(prefs['deleteotherforms'])
         self.l.addWidget(self.deleteotherforms)
         
-        self.updatecover = QCheckBox('Update Calibre Cover when Updating Metadata?',self)
-        self.updatecover.setToolTip("Update calibre book cover image from EPUB when metadata is updated.  (EPUB only.)\nDoesn't go looking for new images on 'Update Calibre Metadata Only'.")
+        self.updatecover = QCheckBox(_('Update Calibre Cover when Updating Metadata?'),self)
+        self.updatecover.setToolTip(_("Update calibre book cover image from EPUB when metadata is updated.  (EPUB only.)\nDoesn't go looking for new images on 'Update Calibre Metadata Only'."))
         self.updatecover.setChecked(prefs['updatecover'])
         self.l.addWidget(self.updatecover)
 
-        self.keeptags = QCheckBox('Keep Existing Tags when Updating Metadata?',self)
-        self.keeptags.setToolTip("Existing tags will be kept and any new tags added.\nCompleted and In-Progress tags will be still be updated, if known.\nLast Updated tags will be updated if lastupdate in include_subject_tags.\n(If Tags is set to 'New Only' in the Standard Columns tab, this has no effect.)")
+        self.keeptags = QCheckBox(_('Keep Existing Tags when Updating Metadata?'),self)
+        self.keeptags.setToolTip(_("Existing tags will be kept and any new tags added.\n%(cmplt)s and %(inprog)s tags will be still be updated, if known.\n%(lul)s tags will be updated if %(lus)s in %(is)s.\n(If Tags is set to 'New Only' in the Standard Columns tab, this has no effect.)")%no_trans)
         self.keeptags.setChecked(prefs['keeptags'])
         self.l.addWidget(self.keeptags)
 
-        self.suppressauthorsort = QCheckBox('Force Author into Author Sort?',self)
-        self.suppressauthorsort.setToolTip("If checked, the author(s) as given will be used for the Author Sort, too.\nIf not checked, calibre will apply it's built in algorithm which makes 'Bob Smith' sort as 'Smith, Bob', etc.")
+        self.suppressauthorsort = QCheckBox(_('Force Author into Author Sort?'),self)
+        self.suppressauthorsort.setToolTip(_("If checked, the author(s) as given will be used for the Author Sort, too.\nIf not checked, calibre will apply it's built in algorithm which makes 'Bob Smith' sort as 'Smith, Bob', etc."))
         self.suppressauthorsort.setChecked(prefs['suppressauthorsort'])
         self.l.addWidget(self.suppressauthorsort)
 
-        self.suppresstitlesort = QCheckBox('Force Title into Title Sort?',self)
-        self.suppresstitlesort.setToolTip("If checked, the title as given will be used for the Title Sort, too.\nIf not checked, calibre will apply it's built in algorithm which makes 'The Title' sort as 'Title, The', etc.")
+        self.suppresstitlesort = QCheckBox(_('Force Title into Title Sort?'),self)
+        self.suppresstitlesort.setToolTip(_("If checked, the title as given will be used for the Title Sort, too.\nIf not checked, calibre will apply it's built in algorithm which makes 'The Title' sort as 'Title, The', etc."))
         self.suppresstitlesort.setChecked(prefs['suppresstitlesort'])
         self.l.addWidget(self.suppresstitlesort)
 
-        self.checkforseriesurlid = QCheckBox("Check for existing Series Anthology books?",self)
-        self.checkforseriesurlid.setToolTip("Check for existings Series Anthology books using each new story's series URL before downloading.\nOffer to skip downloading if a Series Anthology is found.")
+        self.checkforseriesurlid = QCheckBox(_("Check for existing Series Anthology books?"),self)
+        self.checkforseriesurlid.setToolTip(_("Check for existings Series Anthology books using each new story's series URL before downloading.\nOffer to skip downloading if a Series Anthology is found."))
         self.checkforseriesurlid.setChecked(prefs['checkforseriesurlid'])
         self.l.addWidget(self.checkforseriesurlid)
 
-        self.checkforurlchange = QCheckBox("Check for changed Story URL?",self)
-        self.checkforurlchange.setToolTip("Warn you if an update will change the URL of an existing book.")
+        self.checkforurlchange = QCheckBox(_("Check for changed Story URL?"),self)
+        self.checkforurlchange.setToolTip(_("Warn you if an update will change the URL of an existing book.\nfanfiction.net URLs will change from http to https silently."))
         self.checkforurlchange.setChecked(prefs['checkforurlchange'])
         self.l.addWidget(self.checkforurlchange)
 
-        self.lookforurlinhtml = QCheckBox("Search EPUB text for Story URL?",self)
-        self.lookforurlinhtml.setToolTip("Look for first valid story URL inside EPUB text if not found in metadata.\nSomewhat risky, could find wrong URL depending on EPUB content.\nAlso finds and corrects bad ffnet URLs from ficsaver.com files.")
+        self.lookforurlinhtml = QCheckBox(_("Search EPUB text for Story URL?"),self)
+        self.lookforurlinhtml.setToolTip(_("Look for first valid story URL inside EPUB text if not found in metadata.\nSomewhat risky, could find wrong URL depending on EPUB content.\nAlso finds and corrects bad ffnet URLs from ficsaver.com files."))
         self.lookforurlinhtml.setChecked(prefs['lookforurlinhtml'])
         self.l.addWidget(self.lookforurlinhtml)
 
-        self.mark = QCheckBox("Mark added/updated books when finished?",self)
-        self.mark.setToolTip("Mark added/updated books when finished.  Use with option below.\nYou can also manually search for 'marked:ffdl_success'.\n'marked:ffdl_failed' is also available, or search 'marked:ffdl' for both.")
+        self.mark = QCheckBox(_("Mark added/updated books when finished?"),self)
+        self.mark.setToolTip(_("Mark added/updated books when finished.  Use with option below.\nYou can also manually search for 'marked:ffdl_success'.\n'marked:ffdl_failed' is also available, or search 'marked:ffdl' for both."))
         self.mark.setChecked(prefs['mark'])
         self.l.addWidget(self.mark)
 
-        self.showmarked = QCheckBox("Show Marked books when finished?",self)
-        self.showmarked.setToolTip("Show Marked added/updated books only when finished.\nYou can also manually search for 'marked:ffdl_success'.\n'marked:ffdl_failed' is also available, or search 'marked:ffdl' for both.")
+        self.showmarked = QCheckBox(_("Show Marked books when finished?"),self)
+        self.showmarked.setToolTip(_("Show Marked added/updated books only when finished.\nYou can also manually search for 'marked:ffdl_success'.\n'marked:ffdl_failed' is also available, or search 'marked:ffdl' for both."))
         self.showmarked.setChecked(prefs['showmarked'])
         self.l.addWidget(self.showmarked)
 
-        gui_gb = groupbox = QGroupBox("GUI Options")
+        self.autoconvert = QCheckBox(_("Automatically Convert new/update books?"),self)
+        self.autoconvert.setToolTip(_("Automatically call calibre's Convert for new/update books.\nConverts to the current output format as chosen in calibre's\nPreferences->Behavior settings."))
+        self.autoconvert.setChecked(prefs['autoconvert'])
+        self.l.addWidget(self.autoconvert)
+
+        gui_gb = groupbox = QGroupBox(_("GUI Options"))
         self.l = QVBoxLayout()
         groupbox.setLayout(self.l)
 
-        self.urlsfromclip = QCheckBox('Take URLs from Clipboard?',self)
-        self.urlsfromclip.setToolTip('Prefill URLs from valid URLs in Clipboard when Adding New.')
+        self.urlsfromclip = QCheckBox(_('Take URLs from Clipboard?'),self)
+        self.urlsfromclip.setToolTip(_('Prefill URLs from valid URLs in Clipboard when Adding New.'))
         self.urlsfromclip.setChecked(prefs['urlsfromclip'])
         self.l.addWidget(self.urlsfromclip)
 
-        self.updatedefault = QCheckBox('Default to Update when books selected?',self)
-        self.updatedefault.setToolTip('The top FanFictionDownLoader plugin button will start Update if\n'+
-                                      'books are selected.  If unchecked, it will always bring up \'Add New\'.')
+        self.updatedefault = QCheckBox(_('Default to Update when books selected?'),self)
+        self.updatedefault.setToolTip(_('The top FanFictionDownLoader plugin button will start Update if\nbooks are selected.  If unchecked, it will always bring up \'Add New\'.'))
         self.updatedefault.setChecked(prefs['updatedefault'])
         self.l.addWidget(self.updatedefault)
 
-        self.adddialogstaysontop = QCheckBox("Keep 'Add New from URL(s)' dialog on top?",self)
-        self.adddialogstaysontop.setToolTip("Instructs the OS and Window Manager to keep the 'Add New from URL(s)'\ndialog on top of all other windows.  Useful for dragging URLs onto it.")
+        self.adddialogstaysontop = QCheckBox(_("Keep 'Add New from URL(s)' dialog on top?"),self)
+        self.adddialogstaysontop.setToolTip(_("Instructs the OS and Window Manager to keep the 'Add New from URL(s)'\ndialog on top of all other windows.  Useful for dragging URLs onto it."))
         self.adddialogstaysontop.setChecked(prefs['adddialogstaysontop'])
         self.l.addWidget(self.adddialogstaysontop)
 
-        misc_gb = groupbox = QGroupBox("Misc Options")
+        misc_gb = groupbox = QGroupBox(_("Misc Options"))
         self.l = QVBoxLayout()
         groupbox.setLayout(self.l)
 
         # this is a cheat to make it easier for users to realize there's a new include_images features.
-        self.includeimages = QCheckBox("Include images in EPUBs?",self)
-        self.includeimages.setToolTip("Download and include images in EPUB stories.  This is equivalent to adding:\n\n[epub]\ninclude_images:true\nkeep_summary_html:true\nmake_firstimage_cover:true\n\n ...to the top of personal.ini.  Your settings in personal.ini will override this.")
+        self.includeimages = QCheckBox(_("Include images in EPUBs?"),self)
+        self.includeimages.setToolTip(_("Download and include images in EPUB stories.  This is equivalent to adding:%(imgset)s ...to the top of %(pini)s.  Your settings in %(pini)s will override this.")%no_trans)
         self.includeimages.setChecked(prefs['includeimages'])
         self.l.addWidget(self.includeimages)
 
-        self.injectseries = QCheckBox("Inject calibre Series when none found?",self)
-        self.injectseries.setToolTip("If no series is found, inject the calibre series (if there is one) so it appears on the FFDL title page(not cover).")
+        self.injectseries = QCheckBox(_("Inject calibre Series when none found?"),self)
+        self.injectseries.setToolTip(_("If no series is found, inject the calibre series (if there is one) so it appears on the FFDL title page(not cover)."))
         self.injectseries.setChecked(prefs['injectseries'])
         self.l.addWidget(self.injectseries)
 
-        rej_gb = groupbox = QGroupBox("Reject List")
+        rej_gb = groupbox = QGroupBox(_("Reject List"))
         self.l = QVBoxLayout()
         groupbox.setLayout(self.l)
 
-        self.rejectlist = QPushButton('Edit Reject URL List', self)
-        self.rejectlist.setToolTip("Edit list of URLs FFDL will automatically Reject.")
+        self.rejectlist = QPushButton(_('Edit Reject URL List'), self)
+        self.rejectlist.setToolTip(_("Edit list of URLs FFDL will automatically Reject."))
         self.rejectlist.clicked.connect(self.show_rejectlist)
         self.l.addWidget(self.rejectlist)
         
-        self.reject_urls = QPushButton('Add Reject URLs', self)
-        self.reject_urls.setToolTip("Add additional URLs to Reject as text.")
+        self.reject_urls = QPushButton(_('Add Reject URLs'), self)
+        self.reject_urls.setToolTip(_("Add additional URLs to Reject as text."))
         self.reject_urls.clicked.connect(self.add_reject_urls)
         self.l.addWidget(self.reject_urls)
         
-        self.reject_reasons = QPushButton('Edit Reject Reasons List', self)
-        self.reject_reasons.setToolTip("Customize the Reasons presented when Rejecting URLs")
+        self.reject_reasons = QPushButton(_('Edit Reject Reasons List'), self)
+        self.reject_reasons.setToolTip(_("Customize the Reasons presented when Rejecting URLs"))
         self.reject_reasons.clicked.connect(self.show_reject_reasons)
         self.l.addWidget(self.reject_reasons)
 
         topl.addWidget(defs_gb)
 
         horz = QHBoxLayout()
-        topl.addLayout(horz)
-        horz.addWidget(cali_gb)
-        horz.addWidget(rej_gb)
-        
-        horz = QHBoxLayout()
-        topl.addLayout(horz)
-        horz.addWidget(gui_gb)
-        horz.addWidget(misc_gb)
 
+        horz.addWidget(cali_gb)
+
+        vert = QVBoxLayout()
+        vert.addWidget(gui_gb)
+        vert.addWidget(misc_gb)
+        vert.addWidget(rej_gb)
+        
+        horz.addLayout(vert)
+        
+        topl.addLayout(horz)
         topl.insertStretch(-1)
         
     def set_collisions(self):
@@ -475,7 +516,7 @@ class BasicTab(QWidget):
         d = RejectListDialog(self,
                              rejecturllist.get_list(),
                              rejectreasons=rejecturllist.get_reject_reasons(),
-                             header="Edit Reject URLs List",
+                             header=_("Edit Reject URLs List"),
                              show_delete=False,
                              show_all_reasons=False)
         d.exec_()
@@ -489,22 +530,22 @@ class BasicTab(QWidget):
         d = EditTextDialog(self,
                            prefs['rejectreasons'],
                            icon=self.windowIcon(),
-                           title="Reject Reasons",
-                           label="Customize Reject List Reasons",
-                           tooltip="Customize the Reasons presented when Rejecting URLs")
+                           title=_("Reject Reasons"),
+                           label=_("Customize Reject List Reasons"),
+                           tooltip=_("Customize the Reasons presented when Rejecting URLs"))
         d.exec_()
         if d.result() == d.Accepted:
             prefs['rejectreasons'] = d.get_plain_text()
                 
     def add_reject_urls(self):
         d = EditTextDialog(self,
-                           "http://example.com/story.php?sid=5,Reason why I rejected it\nhttp://example.com/story.php?sid=6,Title by Author - Reason why I rejected it",
+                           "http://example.com/story.php?sid=5,"+_("Reason why I rejected it")+"\nhttp://example.com/story.php?sid=6,"+_("Title by Author")+" - "+_("Reason why I rejected it"),
                            icon=self.windowIcon(),
-                           title="Add Reject URLs",
-                           label="Add Reject URLs. Use: <b>http://...,note</b> or <b>http://...,title by author - note</b><br>Invalid story URLs will be ignored.",
-                           tooltip="One URL per line:\n<b>http://...,note</b>\n<b>http://...,title by author - note</b>",
+                           title=_("Add Reject URLs"),
+                           label=_("Add Reject URLs. Use: <b>http://...,note</b> or <b>http://...,title by author - note</b><br>Invalid story URLs will be ignored."),
+                           tooltip=_("One URL per line:\n<b>http://...,note</b>\n<b>http://...,title by author - note</b>"),
                            rejectreasons=rejecturllist.get_reject_reasons(),
-                           reasonslabel='Add this reason to all URLs added:')
+                           reasonslabel=_('Add this reason to all URLs added:'))
         d.exec_()
         if d.result() == d.Accepted:
             rejecturllist.add_text(d.get_plain_text(),d.get_reason_text())
@@ -519,7 +560,7 @@ class PersonalIniTab(QWidget):
         self.l = QVBoxLayout()
         self.setLayout(self.l)
 
-        label = QLabel('These settings provide more detailed control over what metadata will be displayed inside the ebook as well as let you set is_adult and user/password for different sites.')
+        label = QLabel(_('These settings provide more detailed control over what metadata will be displayed inside the ebook as well as let you set %(isa)s and %(u)s/%(p)s for different sites.')%no_trans)
         label.setWordWrap(True)
         self.l.addWidget(label)
         self.l.addSpacing(5)
@@ -537,8 +578,8 @@ class PersonalIniTab(QWidget):
         self.ini.setText(prefs['personal.ini'])
         self.l.addWidget(self.ini)
 
-        self.defaults = QPushButton('View Defaults (plugin-defaults.ini)', self)
-        self.defaults.setToolTip("View all of the plugin's configurable settings\nand their default settings.")
+        self.defaults = QPushButton(_('View Defaults')+' (plugin-defaults.ini)', self)
+        self.defaults.setToolTip(_("View all of the plugin's configurable settings\nand their default settings."))
         self.defaults.clicked.connect(self.show_defaults)
         self.l.addWidget(self.defaults)
         
@@ -556,14 +597,14 @@ class ShowDefaultsIniDialog(QDialog):
         self.resize(600, 500)
         self.l = QVBoxLayout()
         self.setLayout(self.l)
-        self.label = QLabel("Plugin Defaults (plugin-defaults.ini) (Read-Only)")
-        self.label.setToolTip("These are all of the plugin's configurable options\nand their default settings.")
+        self.label = QLabel(_("Plugin Defaults (%s) (Read-Only)")%'plugin-defaults.ini')
+        self.label.setToolTip(_("These are all of the plugin's configurable options\nand their default settings."))
         self.setWindowTitle(_('Plugin Defaults'))
         self.setWindowIcon(icon)
         self.l.addWidget(self.label)
         
         self.ini = QTextEdit(self)
-        self.ini.setToolTip("These are all of the plugin's configurable options\nand their default settings.")
+        self.ini.setToolTip(_("These are all of the plugin's configurable options\nand their default settings."))
         try:
             self.ini.setFont(QFont("Courier",
                                    get_gui().font().pointSize()+1))
@@ -574,7 +615,7 @@ class ShowDefaultsIniDialog(QDialog):
         self.ini.setReadOnly(True)
         self.l.addWidget(self.ini)
         
-        self.ok_button = QPushButton('OK', self)
+        self.ok_button = QPushButton(_('OK'), self)
         self.ok_button.clicked.connect(self.hide)
         self.l.addWidget(self.ok_button)
         
@@ -594,45 +635,45 @@ class ReadingListTab(QWidget):
         except KeyError:
             reading_lists= []
             
-        label = QLabel('These settings provide integration with the Reading List Plugin.  Reading List can automatically send to devices and change custom columns.  You have to create and configure the lists in Reading List to be useful.')
+        label = QLabel(_('These settings provide integration with the %(rl)s Plugin.  %(rl)s can automatically send to devices and change custom columns.  You have to create and configure the lists in %(rl)s to be useful.')%no_trans)
         label.setWordWrap(True)
         self.l.addWidget(label)
         self.l.addSpacing(5)
         
-        self.addtolists = QCheckBox('Add new/updated stories to "Send to Device" Reading List(s).',self)
-        self.addtolists.setToolTip('Automatically add new/updated stories to these lists in the Reading List plugin.')
+        self.addtolists = QCheckBox(_('Add new/updated stories to "Send to Device" Reading List(s).'),self)
+        self.addtolists.setToolTip(_('Automatically add new/updated stories to these lists in the %(rl)s plugin.')%no_trans)
         self.addtolists.setChecked(prefs['addtolists'])
         self.l.addWidget(self.addtolists)
             
         horz = QHBoxLayout()
-        label = QLabel('"Send to Device" Reading Lists')
-        label.setToolTip("When enabled, new/updated stories will be automatically added to these lists.")
+        label = QLabel(_('"Send to Device" Reading Lists'))
+        label.setToolTip(_("When enabled, new/updated stories will be automatically added to these lists."))
         horz.addWidget(label)        
         self.send_lists_box = MultiCompleteLineEdit(self)
-        self.send_lists_box.setToolTip("When enabled, new/updated stories will be automatically added to these lists.")
+        self.send_lists_box.setToolTip(_("When enabled, new/updated stories will be automatically added to these lists."))
         self.send_lists_box.update_items_cache(reading_lists)
         self.send_lists_box.setText(prefs['send_lists'])
         horz.addWidget(self.send_lists_box)
         self.l.addLayout(horz)
         
-        self.addtoreadlists = QCheckBox('Add new/updated stories to "To Read" Reading List(s).',self)
-        self.addtoreadlists.setToolTip('Automatically add new/updated stories to these lists in the Reading List plugin.\nAlso offers menu option to remove stories from the "To Read" lists.')
+        self.addtoreadlists = QCheckBox(_('Add new/updated stories to "To Read" Reading List(s).'),self)
+        self.addtoreadlists.setToolTip(_('Automatically add new/updated stories to these lists in the %(rl)s plugin.\nAlso offers menu option to remove stories from the "To Read" lists.')%no_trans)
         self.addtoreadlists.setChecked(prefs['addtoreadlists'])
         self.l.addWidget(self.addtoreadlists)
             
         horz = QHBoxLayout()
-        label = QLabel('"To Read" Reading Lists')
-        label.setToolTip("When enabled, new/updated stories will be automatically added to these lists.")
+        label = QLabel(_('"To Read" Reading Lists'))
+        label.setToolTip(_("When enabled, new/updated stories will be automatically added to these lists."))
         horz.addWidget(label)        
         self.read_lists_box = MultiCompleteLineEdit(self)
-        self.read_lists_box.setToolTip("When enabled, new/updated stories will be automatically added to these lists.")
+        self.read_lists_box.setToolTip(_("When enabled, new/updated stories will be automatically added to these lists."))
         self.read_lists_box.update_items_cache(reading_lists)
         self.read_lists_box.setText(prefs['read_lists'])
         horz.addWidget(self.read_lists_box)
         self.l.addLayout(horz)
         
-        self.addtolistsonread = QCheckBox('Add stories back to "Send to Device" Reading List(s) when marked "Read".',self)
-        self.addtolistsonread.setToolTip('Menu option to remove from "To Read" lists will also add stories back to "Send to Device" Reading List(s)')
+        self.addtolistsonread = QCheckBox(_('Add stories back to "Send to Device" Reading List(s) when marked "Read".'),self)
+        self.addtolistsonread.setToolTip(_('Menu option to remove from "To Read" lists will also add stories back to "Send to Device" Reading List(s)'))
         self.addtolistsonread.setChecked(prefs['addtolistsonread'])
         self.l.addWidget(self.addtolistsonread)
             
@@ -654,7 +695,7 @@ class GenerateCoverTab(QWidget):
         except KeyError:
             gc_settings= []
             
-        label = QLabel('The Generate Cover plugin can create cover images for books using various metadata and configurations.  If you have GC installed, FFDL can run GC on new downloads and metadata updates.  Pick a GC setting by site or Default.')
+        label = QLabel(_('The %(gc)s plugin can create cover images for books using various metadata and configurations.  If you have GC installed, FFDL can run GC on new downloads and metadata updates.  Pick a GC setting by site or Default.')%no_trans)
         label.setWordWrap(True)
         self.l.addWidget(label)
         self.l.addSpacing(5)
@@ -672,14 +713,15 @@ class GenerateCoverTab(QWidget):
 
         sitelist = getConfigSections()
         sitelist.sort()
-        sitelist.insert(0,u"Default")
+        sitelist.insert(0,_("Default"))
         for site in sitelist:
             horz = QHBoxLayout()
             label = QLabel(site)
-            if site == u"Default":
-                s = "On Metadata update, run Generate Cover with this setting, if not selected for specific site."
+            if site == _("Default"):
+                s = _("On Metadata update, run %(gc)s with this setting, if not selected for specific site.")%no_trans
             else:
-                s = "On Metadata update, run Generate Cover with this setting for %s stories."%site
+                no_trans['site']=site # not ideal, but, meh.
+                s = _("On Metadata update, run %(gc)s with this setting for %(site)s stories.")%no_trans
 
             label.setToolTip(s)
             horz.addWidget(label)
@@ -688,7 +730,12 @@ class GenerateCoverTab(QWidget):
             dropdown.addItem('',QVariant('none'))
             for setting in gc_settings:
                 dropdown.addItem(setting,QVariant(setting))
-            self.gc_dropdowns[site] = dropdown
+            if site == _("Default"):
+                self.gc_dropdowns["Default"] = dropdown
+                if 'Default' in prefs['gc_site_settings']:
+                    dropdown.setCurrentIndex(dropdown.findData(QVariant(prefs['gc_site_settings']['Default'])))
+            else:
+                self.gc_dropdowns[site] = dropdown
             if site in prefs['gc_site_settings']:
                 dropdown.setCurrentIndex(dropdown.findData(QVariant(prefs['gc_site_settings'][site])))
 
@@ -697,16 +744,21 @@ class GenerateCoverTab(QWidget):
         
         self.sl.insertStretch(-1)
         
-        self.gcnewonly = QCheckBox("Run Generate Cover Only on New Books",self)
-        self.gcnewonly.setToolTip("Default is to run GC any time the calibre metadata is updated.")
+        self.gcnewonly = QCheckBox(_("Run %(gc)s Only on New Books")%no_trans,self)
+        self.gcnewonly.setToolTip(_("Default is to run GC any time the calibre metadata is updated."))
         self.gcnewonly.setChecked(prefs['gcnewonly'])
         self.l.addWidget(self.gcnewonly)
 
-        self.allow_gc_from_ini = QCheckBox('Allow generate_cover_settings from personal.ini to override',self)
-        self.allow_gc_from_ini.setToolTip("The personal.ini parameter generate_cover_settings allows you to choose a GC setting based on metadata rather than site, but it's much more complex.<br \>generate_cover_settings is ignored when this is off.")
+        self.allow_gc_from_ini = QCheckBox(_('Allow %(gcset)s from %(pini)s to override')%no_trans,self)
+        self.allow_gc_from_ini.setToolTip(_("The %(pini)s parameter %(gcset)s allows you to choose a GC setting based on metadata rather than site, but it's much more complex.<br \>%(gcset)s is ignored when this is off.")%no_trans)
         self.allow_gc_from_ini.setChecked(prefs['allow_gc_from_ini'])
         self.l.addWidget(self.allow_gc_from_ini)
             
+        self.gc_polish_cover = QCheckBox(_("Use calibre's Polish feature to inject/update the cover"),self)
+        self.gc_polish_cover.setToolTip(_("Calibre's Polish feature will be used to inject or update the generated cover into the ebook, EPUB only."))
+        self.gc_polish_cover.setChecked(prefs['gc_polish_cover'])
+        self.l.addWidget(self.gc_polish_cover)
+            
 class CountPagesTab(QWidget):
 
     def __init__(self, parent_dialog, plugin_action):
@@ -717,39 +769,41 @@ class CountPagesTab(QWidget):
         self.l = QVBoxLayout()
         self.setLayout(self.l)
 
-        label = QLabel('These settings provide integration with the Count Pages Plugin.  Count Pages can automatically update custom columns with page, word and reading level statistics.  You have to create and configure the columns in Count Pages first.')
+        label = QLabel(_('These settings provide integration with the %(cp)s Plugin.  %(cp)s can automatically update custom columns with page, word and reading level statistics.  You have to create and configure the columns in %(cp)s first.')%no_trans)
         label.setWordWrap(True)
         self.l.addWidget(label)
         self.l.addSpacing(5)
 
-        label = QLabel('If any of the settings below are checked, when stories are added or updated, the Count Pages Plugin will be called to update the checked statistics.')
+        label = QLabel(_('If any of the settings below are checked, when stories are added or updated, the %(cp)s Plugin will be called to update the checked statistics.')%no_trans)
         label.setWordWrap(True)
         self.l.addWidget(label)
         self.l.addSpacing(5)
-        
+
+        # the same for all settings.  Mostly.
+        tooltip = _('Which column and algorithm to use are configured in %(cp)s.')%no_trans
         # 'PageCount', 'WordCount', 'FleschReading', 'FleschGrade', 'GunningFog'
         self.pagecount = QCheckBox('Page Count',self)
-        self.pagecount.setToolTip('Which column and algorithm to use are configured in Count Pages.')
+        self.pagecount.setToolTip(tooltip)
         self.pagecount.setChecked('PageCount' in prefs['countpagesstats'])
         self.l.addWidget(self.pagecount)
             
         self.wordcount = QCheckBox('Word Count',self)
-        self.wordcount.setToolTip('Which column and algorithm to use are configured in Count Words.\nWill overwrite word count from FFDL metadata if set to update the same custom column.')
+        self.wordcount.setToolTip(tooltip+"\n"+_('Will overwrite word count from FFDL metadata if set to update the same custom column.'))
         self.wordcount.setChecked('WordCount' in prefs['countpagesstats'])
         self.l.addWidget(self.wordcount)
 
         self.fleschreading = QCheckBox('Flesch Reading Ease',self)
-        self.fleschreading.setToolTip('Which column and algorithm to use are configured in Count Pages.')
+        self.fleschreading.setToolTip(tooltip)
         self.fleschreading.setChecked('FleschReading' in prefs['countpagesstats'])
         self.l.addWidget(self.fleschreading)
         
         self.fleschgrade = QCheckBox('Flesch-Kincaid Grade Level',self)
-        self.fleschgrade.setToolTip('Which column and algorithm to use are configured in Count Pages.')
+        self.fleschgrade.setToolTip(tooltip)
         self.fleschgrade.setChecked('FleschGrade' in prefs['countpagesstats'])
         self.l.addWidget(self.fleschgrade)
         
         self.gunningfog = QCheckBox('Gunning Fog Index',self)
-        self.gunningfog.setToolTip('Which column and algorithm to use are configured in Count Pages.')
+        self.gunningfog.setToolTip(tooltip)
         self.gunningfog.setChecked('GunningFog' in prefs['countpagesstats'])
         self.l.addWidget(self.gunningfog)
         
@@ -765,26 +819,23 @@ class OtherTab(QWidget):
         self.l = QVBoxLayout()
         self.setLayout(self.l)
 
-        label = QLabel("These controls aren't plugin settings as such, but convenience buttons for setting Keyboard shortcuts and getting all the FanFictionDownLoader confirmation dialogs back again.")
+        label = QLabel(_("These controls aren't plugin settings as such, but convenience buttons for setting Keyboard shortcuts and getting all the FanFictionDownLoader confirmation dialogs back again."))
         label.setWordWrap(True)
         self.l.addWidget(label)
         self.l.addSpacing(5)
         
-        keyboard_shortcuts_button = QPushButton('Keyboard shortcuts...', self)
-        keyboard_shortcuts_button.setToolTip(_(
-                    'Edit the keyboard shortcuts associated with this plugin'))
+        keyboard_shortcuts_button = QPushButton(_('Keyboard shortcuts...'), self)
+        keyboard_shortcuts_button.setToolTip(_('Edit the keyboard shortcuts associated with this plugin'))
         keyboard_shortcuts_button.clicked.connect(parent_dialog.edit_shortcuts)
         self.l.addWidget(keyboard_shortcuts_button)
 
         reset_confirmation_button = QPushButton(_('Reset disabled &confirmation dialogs'), self)
-        reset_confirmation_button.setToolTip(_(
-                    'Reset all show me again dialogs for the FanFictionDownLoader plugin'))
+        reset_confirmation_button.setToolTip(_('Reset all show me again dialogs for the FanFictionDownLoader plugin'))
         reset_confirmation_button.clicked.connect(self.reset_dialogs)
         self.l.addWidget(reset_confirmation_button)
         
-        view_prefs_button = QPushButton('&View library preferences...', self)
-        view_prefs_button.setToolTip(_(
-                    'View data stored in the library database for this plugin'))
+        view_prefs_button = QPushButton(_('&View library preferences...'), self)
+        view_prefs_button.setToolTip(_('View data stored in the library database for this plugin'))
         view_prefs_button.clicked.connect(self.view_prefs)
         self.l.addWidget(view_prefs_button)
         
@@ -844,35 +895,35 @@ permitted_values['text'] = permitted_values['enumeration']
 permitted_values['comments'] = permitted_values['enumeration']
 
 titleLabels = {
-    'category':'Category',
-    'genre':'Genre',
-    'language':'Language',
-    'status':'Status',
-    'status-C':'Status:Completed',
-    'status-I':'Status:In-Progress',
-    'series':'Series',
-    'characters':'Characters',
-    'ships':'Relationships',
-    'datePublished':'Published',
-    'dateUpdated':'Updated',
-    'dateCreated':'Created',
-    'rating':'Rating',
-    'warnings':'Warnings',
-    'numChapters':'Chapters',
-    'numWords':'Words',
-    'site':'Site',
-    'storyId':'Story ID',
-    'authorId':'Author ID',
-    'extratags':'Extra Tags',
-    'title':'Title',
-    'storyUrl':'Story URL',
-    'description':'Description',
-    'author':'Author',
-    'authorUrl':'Author URL',
-    'formatname':'File Format',
-    'formatext':'File Extension',
-    'siteabbrev':'Site Abbrev',
-    'version':'FFDL Version'
+    'category':_('Category'),
+    'genre':_('Genre'),
+    'language':_('Language'),
+    'status':_('Status'),
+    'status-C':_('Status:%(cmplt)s')%no_trans,
+    'status-I':_('Status:%(inprog)s')%no_trans,
+    'series':_('Series'),
+    'characters':_('Characters'),
+    'ships':_('Relationships'),
+    'datePublished':_('Published'),
+    'dateUpdated':_('Updated'),
+    'dateCreated':_('Created'),
+    'rating':_('Rating'),
+    'warnings':_('Warnings'),
+    'numChapters':_('Chapters'),
+    'numWords':_('Words'),
+    'site':_('Site'),
+    'storyId':_('Story ID'),
+    'authorId':_('Author ID'),
+    'extratags':_('Extra Tags'),
+    'title':_('Title'),
+    'storyUrl':_('Story URL'),
+    'description':_('Description'),
+    'author':_('Author'),
+    'authorUrl':_('Author URL'),
+    'formatname':_('File Format'),
+    'formatext':_('File Extension'),
+    'siteabbrev':_('Site Abbrev'),
+    'version':_('FFDL Version')
     }
 
 class CustomColumnsTab(QWidget):
@@ -887,7 +938,7 @@ class CustomColumnsTab(QWidget):
         self.l = QVBoxLayout()
         self.setLayout(self.l)
 
-        label = QLabel("If you have custom columns defined, they will be listed below.  Choose a metadata value type to fill your columns automatically.")
+        label = QLabel(_("If you have custom columns defined, they will be listed below.  Choose a metadata value type to fill your columns automatically."))
         label.setWordWrap(True)
         self.l.addWidget(label)
         self.l.addSpacing(5)
@@ -912,7 +963,7 @@ class CustomColumnsTab(QWidget):
                 #     print("column['%s'] => %s"%(k,v))
                 horz = QHBoxLayout()
                 label = QLabel(column['name'])
-                label.setToolTip("Update this %s column(%s) with..."%(key,column['datatype']))
+                label.setToolTip(_("Update this %s column(%s) with...")%(key,column['datatype']))
                 horz.addWidget(label)
                 dropdown = QComboBox(self)
                 dropdown.addItem('',QVariant('none'))
@@ -922,13 +973,13 @@ class CustomColumnsTab(QWidget):
                 if key in prefs['custom_cols']:
                     dropdown.setCurrentIndex(dropdown.findData(QVariant(prefs['custom_cols'][key])))
                 if column['datatype'] == 'enumeration':
-                    dropdown.setToolTip("Metadata values valid for this type of column.\nValues that aren't valid for this enumeration column will be ignored.")
+                    dropdown.setToolTip(_("Metadata values valid for this type of column.")+"\n"+_("Values that aren't valid for this enumeration column will be ignored."))
                 else:
-                    dropdown.setToolTip("Metadata values valid for this type of column.")
+                    dropdown.setToolTip(_("Metadata values valid for this type of column."))
                 horz.addWidget(dropdown)
 
-                newonlycheck = QCheckBox("New Only",self)
-                newonlycheck.setToolTip("Write to %s(%s) only for new\nbooks, not updates to existing books."%(column['name'],key))
+                newonlycheck = QCheckBox(_("New Only"),self)
+                newonlycheck.setToolTip(_("Write to %s(%s) only for new\nbooks, not updates to existing books.")%(column['name'],key))
                 self.custcol_newonlycheck[key] = newonlycheck
                 if key in prefs['custom_cols_newonly']:
                     newonlycheck.setChecked(prefs['custom_cols_newonly'][key])
@@ -939,19 +990,19 @@ class CustomColumnsTab(QWidget):
         self.sl.insertStretch(-1)
 
         self.l.addSpacing(5)
-        self.allow_custcol_from_ini = QCheckBox('Allow custom_columns_settings from personal.ini to override',self)
-        self.allow_custcol_from_ini.setToolTip("The personal.ini parameter custom_columns_settings allows you to set custom columns to site specific values that aren't common to all sites.<br \>custom_columns_settings is ignored when this is off.")
+        self.allow_custcol_from_ini = QCheckBox(_('Allow %(ccset)s from %(pini)s to override')%no_trans,self)
+        self.allow_custcol_from_ini.setToolTip(_("The %(pini)s parameter %(ccset)s allows you to set custom columns to site specific values that aren't common to all sites.<br />%(ccset)s is ignored when this is off.")%no_trans)
         self.allow_custcol_from_ini.setChecked(prefs['allow_custcol_from_ini'])
         self.l.addWidget(self.allow_custcol_from_ini)
         
         self.l.addSpacing(5)
-        label = QLabel("Special column:")
+        label = QLabel(_("Special column:"))
         label.setWordWrap(True)
         self.l.addWidget(label)
         
         horz = QHBoxLayout()
-        label = QLabel("Update/Overwrite Error Column:")
-        tooltip="When an update or overwrite of an existing story fails, record the reason in this column.\n(Text and Long Text columns only.)"
+        label = QLabel(_("Update/Overwrite Error Column:"))
+        tooltip=_("When an update or overwrite of an existing story fails, record the reason in this column.\n(Text and Long Text columns only.)")
         label.setToolTip(tooltip)
         horz.addWidget(label)
         self.errorcol = QComboBox(self)
@@ -976,21 +1027,21 @@ class StandardColumnsTab(QWidget):
 
         columns=OrderedDict()
         
-        columns["title"]="Title"
-        columns["authors"]="Author(s)"
-        columns["publisher"]="Publisher"
-        columns["tags"]="Tags"
-        columns["languages"]="Languages"
-        columns["pubdate"]="Published Date"
-        columns["timestamp"]="Date"
-        columns["comments"]="Comments"
-        columns["series"]="Series"
-        columns["identifiers"]="Ids(url id only)"
+        columns["title"]=_("Title")
+        columns["authors"]=_("Author(s)")
+        columns["publisher"]=_("Publisher")
+        columns["tags"]=_("Tags")
+        columns["languages"]=_("Languages")
+        columns["pubdate"]=_("Published Date")
+        columns["timestamp"]=_("Date")
+        columns["comments"]=_("Comments")
+        columns["series"]=_("Series")
+        columns["identifiers"]=_("Ids(url id only)")
 
         self.l = QVBoxLayout()
         self.setLayout(self.l)
 
-        label = QLabel("The standard calibre metadata columns are listed below.  You may choose whether FFDL will fill each column automatically on updates or only for new books.")
+        label = QLabel(_("The standard calibre metadata columns are listed below.  You may choose whether FFDL will fill each column automatically on updates or only for new books."))
         label.setWordWrap(True)
         self.l.addWidget(label)
         self.l.addSpacing(5)
@@ -1003,8 +1054,8 @@ class StandardColumnsTab(QWidget):
             #label.setToolTip("Update this %s column(%s) with..."%(key,column['datatype']))
             horz.addWidget(label)
 
-            newonlycheck = QCheckBox("New Only",self)
-            newonlycheck.setToolTip("Write to %s only for new\nbooks, not updates to existing books."%column)
+            newonlycheck = QCheckBox(_("New Only"),self)
+            newonlycheck.setToolTip(_("Write to %s only for new\nbooks, not updates to existing books.")%column)
             self.stdcol_newonlycheck[key] = newonlycheck
             if key in prefs['std_cols_newonly']:
                 newonlycheck.setChecked(prefs['std_cols_newonly'][key])

calibre-plugin/dialogs.py

@@ -29,6 +29,12 @@ from PyQt4.Qt import (QDialog, QTableWidget, QVBoxLayout, QHBoxLayout, QGridLayo
 from calibre.gui2.dialogs.confirm_delete import confirm
 from calibre.gui2.complete2 import EditWithComplete
 
+# pulls in translation files for _() strings
+try:
+    load_translations()
+except NameError:
+    pass # load_translations() added in calibre 1.9
+
 from calibre_plugins.fanfictiondownloader_plugin.common_utils \
     import (ReadOnlyTableWidgetItem, ReadOnlyTextIconWidgetItem, SizePersistedDialog,
             ImageTitleLayout, get_icon)
@@ -36,13 +42,13 @@ from calibre_plugins.fanfictiondownloader_plugin.common_utils \
 from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.geturls import get_urls_from_html, get_urls_from_text
 from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.adapters import getNormalStoryURL
 
-SKIP=u'Skip'
-ADDNEW=u'Add New Book'
-UPDATE=u'Update EPUB if New Chapters'
-UPDATEALWAYS=u'Update EPUB Always'
-OVERWRITE=u'Overwrite if Newer'
-OVERWRITEALWAYS=u'Overwrite Always'
-CALIBREONLY=u'Update Calibre Metadata Only'
+SKIP=_('Skip')
+ADDNEW=_('Add New Book')
+UPDATE=_('Update EPUB if New Chapters')
+UPDATEALWAYS=_('Update EPUB Always')
+OVERWRITE=_('Overwrite if Newer')
+OVERWRITEALWAYS=_('Overwrite Always')
+CALIBREONLY=_('Update Calibre Metadata Only')
 collision_order=[SKIP,
                  ADDNEW,
                  UPDATE,
@@ -51,6 +57,32 @@ collision_order=[SKIP,
                  OVERWRITEALWAYS,
                  CALIBREONLY,]
 
+# best idea I've had for how to deal with config/pref saving the
+# collision name in english.
+SAVE_SKIP='Skip'
+SAVE_ADDNEW='Add New Book'
+SAVE_UPDATE='Update EPUB if New Chapters'
+SAVE_UPDATEALWAYS='Update EPUB Always'
+SAVE_OVERWRITE='Overwrite if Newer'
+SAVE_OVERWRITEALWAYS='Overwrite Always'
+SAVE_CALIBREONLY='Update Calibre Metadata Only'
+save_collisions={
+    SKIP:SAVE_SKIP,
+    ADDNEW:SAVE_ADDNEW,
+    UPDATE:SAVE_UPDATE,
+    UPDATEALWAYS:SAVE_UPDATEALWAYS,
+    OVERWRITE:SAVE_OVERWRITE,
+    OVERWRITEALWAYS:SAVE_OVERWRITEALWAYS,
+    CALIBREONLY:SAVE_CALIBREONLY,
+    SAVE_SKIP:SKIP,
+    SAVE_ADDNEW:ADDNEW,
+    SAVE_UPDATE:UPDATE,
+    SAVE_UPDATEALWAYS:UPDATEALWAYS,
+    SAVE_OVERWRITE:OVERWRITE,
+    SAVE_OVERWRITEALWAYS:OVERWRITEALWAYS,
+    SAVE_CALIBREONLY:CALIBREONLY,
+    }
+    
 anthology_collision_order=[UPDATE,
                            UPDATEALWAYS,
                            OVERWRITEALWAYS]
@@ -104,6 +136,7 @@ class RejectUrlEntry:
     def fullnote(self):
         retval = ""
         if self.title and self.auth:
+            # don't translate--ends up being saved and confuses regex above.
             retval = retval + "%s by %s"%(self.title,self.auth)
             if self.note:
                 retval = retval + " - "
@@ -192,7 +225,7 @@ class AddNewDialog(SizePersistedDialog):
         self.l = QVBoxLayout()
         self.setLayout(self.l)
 
-        self.setWindowTitle('FanFictionDownLoader')
+        self.setWindowTitle(_('FanFictionDownLoader'))
         self.setWindowIcon(icon)
 
         self.toplabel=QLabel("Toplabel")
@@ -209,7 +242,7 @@ class AddNewDialog(SizePersistedDialog):
         # elements to show again when doing *update* merge
         self.mergeupdateshow = []
 
-        self.groupbox = QGroupBox("Show Download Options")
+        self.groupbox = QGroupBox(_("Show Download Options"))
         self.groupbox.setCheckable(True)
         self.groupbox.setChecked(False)
         self.groupbox.setFlat(True)
@@ -228,7 +261,7 @@ class AddNewDialog(SizePersistedDialog):
         self.groupbox.toggled.connect(self.gbf.setVisible)
         
         horz = QHBoxLayout()
-        label = QLabel('Output &Format:')
+        label = QLabel(_('Output &Format:'))
         self.mergehide.append(label)
         
         self.fileform = QComboBox(self)
@@ -236,7 +269,7 @@ class AddNewDialog(SizePersistedDialog):
         self.fileform.addItem('mobi')
         self.fileform.addItem('html')
         self.fileform.addItem('txt')
-        self.fileform.setToolTip('Choose output format to create.  May set default from plugin configuration.')
+        self.fileform.setToolTip(_('Choose output format to create.  May set default from plugin configuration.'))
         self.fileform.activated.connect(self.set_collisions)
         
         horz.addWidget(label)
@@ -252,7 +285,7 @@ class AddNewDialog(SizePersistedDialog):
         self.collision.setToolTip("CollisionToolTip")
         # add collision options
         self.set_collisions()
-        i = self.collision.findText(prefs['collision'])
+        i = self.collision.findText(save_collisions[prefs['collision']])
         if i > -1:
             self.collision.setCurrentIndex(i)
         self.collisionlabel.setBuddy(self.collision)
@@ -264,15 +297,15 @@ class AddNewDialog(SizePersistedDialog):
         self.mergeupdateshow.append(self.collision)
 
         horz = QHBoxLayout()
-        self.updatemeta = QCheckBox('Update Calibre &Metadata?',self)
-        self.updatemeta.setToolTip("Update metadata for existing stories in Calibre from web site?\n(Columns set to 'New Only' in the column tabs will only be set for new books.)")
+        self.updatemeta = QCheckBox(_('Update Calibre &Metadata?'),self)
+        self.updatemeta.setToolTip(_("Update metadata for existing stories in Calibre from web site?\n(Columns set to 'New Only' in the column tabs will only be set for new books.)"))
         self.updatemeta.setChecked(prefs['updatemeta'])
         horz.addWidget(self.updatemeta)
         self.mergehide.append(self.updatemeta)
         self.mergeupdateshow.append(self.updatemeta)
 
-        self.updateepubcover = QCheckBox('Update EPUB Cover?',self)
-        self.updateepubcover.setToolTip('Update book cover image from site or defaults (if found) <i>inside</i> the EPUB when EPUB is updated.')
+        self.updateepubcover = QCheckBox(_('Update EPUB Cover?'),self)
+        self.updateepubcover.setToolTip(_('Update book cover image from site or defaults (if found) <i>inside</i> the EPUB when EPUB is updated.'))
         self.updateepubcover.setChecked(prefs['updateepubcover'])
         horz.addWidget(self.updateepubcover)
         self.mergehide.append(self.updateepubcover)
@@ -319,10 +352,10 @@ class AddNewDialog(SizePersistedDialog):
         self.groupbox.setVisible(not(self.merge and self.newmerge))
         
         if self.merge:
-            self.toplabel.setText('Story URL(s) for anthology, one per line:')
-            self.url.setToolTip('URLs for stories to include in the anthology, one per line.\nWill take URLs from clipboard, but only valid URLs.')
-            self.collisionlabel.setText('If Story Already Exists in Anthology?')
-            self.collision.setToolTip("What to do if there's already an existing story with the same URL in the anthology.")
+            self.toplabel.setText(_('Story URL(s) for anthology, one per line:'))
+            self.url.setToolTip(_('URLs for stories to include in the anthology, one per line.\nWill take URLs from clipboard, but only valid URLs.'))
+            self.collisionlabel.setText(_('If Story Already Exists in Anthology?'))
+            self.collision.setToolTip(_("What to do if there's already an existing story with the same URL in the anthology."))
             for widget in self.mergehide:
                 widget.setVisible(False)
             if not self.newmerge:
@@ -331,10 +364,10 @@ class AddNewDialog(SizePersistedDialog):
         else:
             for widget in self.mergehide:
                 widget.setVisible(True)
-            self.toplabel.setText('Story URL(s), one per line:')
-            self.url.setToolTip('URLs for stories, one per line.\nWill take URLs from clipboard, but only valid URLs.\nAdd [1,5] after the URL to limit the download to chapters 1-5.')
-            self.collisionlabel.setText('If Story Already Exists?')
-            self.collision.setToolTip("What to do if there's already an existing story with the same URL or title and author.")
+            self.toplabel.setText(_('Story URL(s), one per line:'))
+            self.url.setToolTip(_('URLs for stories, one per line.\nWill take URLs from clipboard, but only valid URLs.\nAdd [1,5] after the URL to limit the download to chapters 1-5.'))
+            self.collisionlabel.setText(_('If Story Already Exists?'))
+            self.collision.setToolTip(_("What to do if there's already an existing story with the same URL or title and author."))
 
         # Need to re-able after hiding/showing
         self.setAcceptDrops(True)
@@ -350,10 +383,10 @@ class AddNewDialog(SizePersistedDialog):
 
         # add collision options
         self.set_collisions()
-        
-        i = self.collision.findText(self.prefs['collision'])
+        i = self.collision.findText(save_collisions[self.prefs['collision']])
         if i > -1:
             self.collision.setCurrentIndex(i)
+            
         self.updatemeta.setChecked(self.prefs['updatemeta'])
             
         if not self.merge:
@@ -434,18 +467,18 @@ class CollectURLDialog(SizePersistedDialog):
         self.url.setText(url_text)
         self.l.addWidget(self.url,1,1,1,2)
    
-        self.indiv_button = QPushButton('For Individual Books', self)
-        self.indiv_button.setToolTip('Get URLs and go to dialog for individual story downloads.')
+        self.indiv_button = QPushButton(_('For Individual Books'), self)
+        self.indiv_button.setToolTip(_('Get URLs and go to dialog for individual story downloads.'))
         self.indiv_button.clicked.connect(self.indiv)
         self.l.addWidget(self.indiv_button,2,0)
 
-        self.merge_button = QPushButton('For Anthology Epub', self)
-        self.merge_button.setToolTip('Get URLs and go to dialog for Anthology download.\nRequires EpubMerge 1.3.1+ plugin.')
+        self.merge_button = QPushButton(_('For Anthology Epub'), self)
+        self.merge_button.setToolTip(_('Get URLs and go to dialog for Anthology download.\nRequires %s plugin.')%'EpubMerge 1.3.1+')
         self.merge_button.clicked.connect(self.merge)
         self.l.addWidget(self.merge_button,2,1)
         self.merge_button.setEnabled(epubmerge_plugin!=None)
 
-        self.cancel_button = QPushButton('Cancel', self)
+        self.cancel_button = QPushButton(_('Cancel'), self)
         self.cancel_button.clicked.connect(self.cancel)
         self.l.addWidget(self.cancel_button,2,2)
 
@@ -477,29 +510,29 @@ class UserPassDialog(QDialog):
         self.setLayout(self.l)
 
         if exception.passwdonly:
-            self.setWindowTitle('Password')
-            self.l.addWidget(QLabel("Author requires a password for this story(%s)."%exception.url),0,0,1,2)
+            self.setWindowTitle(_('Password'))
+            self.l.addWidget(QLabel(_("Author requires a password for this story(%s).")%exception.url),0,0,1,2)
             # user isn't used, but it's easier to still have it for
             # post processing.
             self.user = FakeLineEdit()
         else:
-            self.setWindowTitle('User/Password')
-            self.l.addWidget(QLabel("%s requires you to login to download this story."%site),0,0,1,2)
+            self.setWindowTitle(_('User/Password'))
+            self.l.addWidget(QLabel(_("%s requires you to login to download this story.")%site),0,0,1,2)
         
-            self.l.addWidget(QLabel("User:"),1,0)
+            self.l.addWidget(QLabel(_("User:")),1,0)
             self.user = QLineEdit(self)
             self.l.addWidget(self.user,1,1)
    
-        self.l.addWidget(QLabel("Password:"),2,0)
+        self.l.addWidget(QLabel(_("Password:")),2,0)
         self.passwd = QLineEdit(self)
         self.passwd.setEchoMode(QLineEdit.Password)
         self.l.addWidget(self.passwd,2,1)
    
-        self.ok_button = QPushButton('OK', self)
+        self.ok_button = QPushButton(_('OK'), self)
         self.ok_button.clicked.connect(self.ok)
         self.l.addWidget(self.ok_button,3,0)
 
-        self.cancel_button = QPushButton('Cancel', self)
+        self.cancel_button = QPushButton(_('Cancel'), self)
         self.cancel_button.clicked.connect(self.cancel)
         self.l.addWidget(self.cancel_button,3,1)
 
@@ -521,9 +554,9 @@ class LoopProgressDialog(QProgressDialog):
                  book_list,
                  foreach_function,
                  finish_function,
-                 init_label="Fetching metadata for stories...",
-                 win_title="Downloading metadata for stories",
-                 status_prefix="Fetched metadata for"):
+                 init_label=_("Fetching metadata for stories..."),
+                 win_title=_("Downloading metadata for stories"),
+                 status_prefix=_("Fetched metadata for")):
         QProgressDialog.__init__(self,
                                  init_label,
                                  QString(), 0, len(book_list), gui)
@@ -541,7 +574,7 @@ class LoopProgressDialog(QProgressDialog):
         self.exec_()
 
     def updateStatus(self):
-        self.setLabelText("%s %d of %d"%(self.status_prefix,self.i+1,len(self.book_list)))
+        self.setLabelText("%s %d / %d"%(self.status_prefix,self.i+1,len(self.book_list)))
         self.setValue(self.i+1)
         #print(self.labelText())
 
@@ -647,7 +680,7 @@ class UpdateExistingDialog(SizePersistedDialog):
         spacerItem = QtGui.QSpacerItem(20, 40, QtGui.QSizePolicy.Minimum, QtGui.QSizePolicy.Expanding)
         button_layout.addItem(spacerItem)
         self.remove_button = QtGui.QToolButton(self)
-        self.remove_button.setToolTip('Remove selected books from the list')
+        self.remove_button.setToolTip(_('Remove selected books from the list'))
         self.remove_button.setIcon(get_icon('list_remove.png'))
         self.remove_button.clicked.connect(self.remove_from_list)
         button_layout.addWidget(self.remove_button)
@@ -656,7 +689,7 @@ class UpdateExistingDialog(SizePersistedDialog):
 
         options_layout = QHBoxLayout()
 
-        groupbox = QGroupBox("Show Download Options")
+        groupbox = QGroupBox(_("Show Download Options"))
         groupbox.setCheckable(True)
         groupbox.setChecked(False)
         groupbox.setFlat(True)
@@ -673,7 +706,7 @@ class UpdateExistingDialog(SizePersistedDialog):
         gbf.setVisible(False)
         groupbox.toggled.connect(gbf.setVisible)
         
-        label = QLabel('Output &Format:')
+        label = QLabel(_('Output &Format:'))
         gbl.addWidget(label)
         self.fileform = QComboBox(self)
         self.fileform.addItem('epub')
@@ -681,30 +714,30 @@ class UpdateExistingDialog(SizePersistedDialog):
         self.fileform.addItem('html')
         self.fileform.addItem('txt')
         self.fileform.setCurrentIndex(self.fileform.findText(prefs['fileform']))
-        self.fileform.setToolTip('Choose output format to create.  May set default from plugin configuration.')
+        self.fileform.setToolTip(_('Choose output format to create.  May set default from plugin configuration.'))
         self.fileform.activated.connect(self.set_collisions)
         label.setBuddy(self.fileform)
         gbl.addWidget(self.fileform)
         
-        label = QLabel('Update Mode:')
+        label = QLabel(_('Update Mode:'))
         gbl.addWidget(label)
         self.collision = QComboBox(self)
-        self.collision.setToolTip("What sort of update to perform.  May set default from plugin configuration.")
+        self.collision.setToolTip(_("What sort of update to perform.  May set default from plugin configuration."))
         # add collision options
         self.set_collisions()
-        i = self.collision.findText(prefs['collision'])
+        i = self.collision.findText(save_collisions[prefs['collision']])
         if i > -1:
             self.collision.setCurrentIndex(i)
         label.setBuddy(self.collision)
         gbl.addWidget(self.collision)
 
-        self.updatemeta = QCheckBox('Update Calibre &Metadata?',self)
-        self.updatemeta.setToolTip("Update metadata for existing stories in Calibre from web site?\n(Columns set to 'New Only' in the column tabs will only be set for new books.)")
+        self.updatemeta = QCheckBox(_('Update Calibre &Metadata?'),self)
+        self.updatemeta.setToolTip(_("Update metadata for existing stories in Calibre from web site?\n(Columns set to 'New Only' in the column tabs will only be set for new books.)"))
         self.updatemeta.setChecked(prefs['updatemeta'])
         gbl.addWidget(self.updatemeta)
                 
-        self.updateepubcover = QCheckBox('Update EPUB Cover?',self)
-        self.updateepubcover.setToolTip('Update book cover image from site or defaults (if found) <i>inside</i> the EPUB when EPUB is updated.')
+        self.updateepubcover = QCheckBox(_('Update EPUB Cover?'),self)
+        self.updateepubcover.setToolTip(_('Update book cover image from site or defaults (if found) <i>inside</i> the EPUB when EPUB is updated.'))
         self.updateepubcover.setChecked(prefs['updateepubcover'])
         gbl.addWidget(self.updateepubcover)
 
@@ -757,7 +790,7 @@ class StoryListTableWidget(QTableWidget):
         self.clear()
         self.setAlternatingRowColors(True)
         self.setRowCount(len(books))
-        header_labels = ['','Title', 'Author', 'URL', 'Comment']
+        header_labels = ['',_('Title'), _('Author'), 'URL', _('Comment')]
         self.setColumnCount(len(header_labels))
         self.setHorizontalHeaderLabels(header_labels)
         self.horizontalHeader().setStretchLastSection(True)
@@ -825,9 +858,9 @@ class StoryListTableWidget(QTableWidget):
         rows = self.selectionModel().selectedRows()
         if len(rows) == 0:
             return
-        message = '<p>Are you sure you want to remove this book from the list?'
+        message = '<p>'+_('Are you sure you want to remove this book from the list?')
         if len(rows) > 1:
-            message = '<p>Are you sure you want to remove the selected %d books from the list?'%len(rows)
+            message = '<p>'+_('Are you sure you want to remove the selected %d books from the list?')%len(rows)
         if not confirm(message,'fanfictiondownloader_delete_item', self):
             return
         first_sel_row = self.currentRow()
@@ -853,7 +886,7 @@ class RejectListTableWidget(QTableWidget):
         self.clear()
         self.setAlternatingRowColors(True)
         self.setRowCount(len(reject_list))
-        header_labels = ['URL', 'Title', 'Author', 'Note']
+        header_labels = ['URL', _('Title'), _('Author'), _('Note')]
         self.setColumnCount(len(header_labels))
         self.setHorizontalHeaderLabels(header_labels)
         self.horizontalHeader().setStretchLastSection(True)
@@ -895,7 +928,7 @@ class RejectListTableWidget(QTableWidget):
         note_cell.update_items_cache(items)
         note_cell.show_initial_value(rej.note)
         note_cell.set_separator(None)
-        note_cell.setToolTip('Select or Edit Reject Note.')
+        note_cell.setToolTip(_('Select or Edit Reject Note.'))
         self.setCellWidget(row, 3, note_cell)
         
     def remove_selected_rows(self):
@@ -903,9 +936,9 @@ class RejectListTableWidget(QTableWidget):
         rows = self.selectionModel().selectedRows()
         if len(rows) == 0:
             return
-        message = '<p>Are you sure you want to remove this URL from the list?'
+        message = '<p>'+_('Are you sure you want to remove this URL from the list?')
         if len(rows) > 1:
-            message = '<p>Are you sure you want to remove the %d selected URLs from the list?'%len(rows)
+            message = '<p>'+_('Are you sure you want to remove the %d selected URLs from the list?')%len(rows)
         if not confirm(message,'ffdl_rejectlist_delete_item_again', self):
             return
         first_sel_row = self.currentRow()
@@ -923,7 +956,7 @@ class RejectListTableWidget(QTableWidget):
 class RejectListDialog(SizePersistedDialog):
     def __init__(self, gui, reject_list,
                  rejectreasons=[],
-                 header="List of Books to Reject",
+                 header=_("List of Books to Reject"),
                  icon='rotate-right.png',
                  show_delete=True,
                  show_all_reasons=True,
@@ -936,7 +969,7 @@ class RejectListDialog(SizePersistedDialog):
         layout = QVBoxLayout(self)
         self.setLayout(layout)
         title_layout = ImageTitleLayout(self, icon, header,
-                                        '<i></i>FFDL will remember these URLs and display the note and offer to reject them if you try to download them again later.')
+                                        '<i></i>'+_('FFDL will remember these URLs and display the note and offer to reject them if you try to download them again later.'))
         layout.addLayout(title_layout)
         rejects_layout = QHBoxLayout()
         layout.addLayout(rejects_layout)
@@ -950,7 +983,7 @@ class RejectListDialog(SizePersistedDialog):
         button_layout.addItem(spacerItem)
         
         self.remove_button = QtGui.QToolButton(self)
-        self.remove_button.setToolTip('Remove selected URL(s) from the list')
+        self.remove_button.setToolTip(_('Remove selected URL(s) from the list'))
         self.remove_button.setIcon(get_icon('list_remove.png'))
         self.remove_button.clicked.connect(self.remove_from_list)
         button_layout.addWidget(self.remove_button)
@@ -968,11 +1001,11 @@ class RejectListDialog(SizePersistedDialog):
             self.reason_edit.update_items_cache(items)
             self.reason_edit.show_initial_value('')
             self.reason_edit.set_separator(None)
-            self.reason_edit.setToolTip("This will be added to whatever note you've set for each URL above.")
+            self.reason_edit.setToolTip(_("This will be added to whatever note you've set for each URL above."))
             
             horz = QHBoxLayout()
-            label = QLabel("Add this reason to all URLs added:")
-            label.setToolTip("This will be added to whatever note you've set for each URL above.")
+            label = QLabel(_("Add this reason to all URLs added:"))
+            label.setToolTip(_("This will be added to whatever note you've set for each URL above."))
             horz.addWidget(label)
             horz.addWidget(self.reason_edit)
             horz.insertStretch(-1)
@@ -981,8 +1014,8 @@ class RejectListDialog(SizePersistedDialog):
         options_layout = QHBoxLayout()
 
         if show_delete:
-            self.deletebooks = QCheckBox('Delete Books (including books without FanFiction URLs)?',self)
-            self.deletebooks.setToolTip("Delete the selected books after adding them to the Rejected URLs list.")
+            self.deletebooks = QCheckBox(_('Delete Books (including books without FanFiction URLs)?'),self)
+            self.deletebooks.setToolTip(_("Delete the selected books after adding them to the Rejected URLs list."))
             self.deletebooks.setChecked(True)
             options_layout.addWidget(self.deletebooks)
         

calibre-plugin/ffdl_plugin.py

@@ -13,7 +13,7 @@ logger = logging.getLogger(__name__)
 import time, os, copy, threading, re, platform, sys
 from StringIO import StringIO
 from functools import partial
-from datetime import datetime
+from datetime import datetime, time
 from string import Template
 import urllib
 import email
@@ -32,6 +32,7 @@ from calibre.ebooks.metadata.meta import get_metadata
 from calibre.gui2 import error_dialog, warning_dialog, question_dialog, info_dialog
 from calibre.gui2.dialogs.message_box import ViewLog
 from calibre.gui2.dialogs.confirm_delete import confirm
+from calibre.utils.config import prefs as calibre_prefs
 from calibre.utils.date import local_tz
 from calibre.library.comments import sanitize_comments_html
 from calibre.constants import config_dir as calibre_config_dir
@@ -39,6 +40,12 @@ from calibre.constants import config_dir as calibre_config_dir
 # The class that all interface action plugins must inherit from
 from calibre.gui2.actions import InterfaceAction
 
+# pulls in translation files for _() strings
+try:
+    load_translations()
+except NameError:
+    pass # load_translations() added in calibre 1.9
+
 from calibre_plugins.fanfictiondownloader_plugin.common_utils import (set_plugin_icon_resources, get_icon,
                                          create_menu_action_unique, get_library_uuid)
 
@@ -77,8 +84,8 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
     # keyboard shortcuts, so try to use an unusual/unused shortcut.
     # (text, icon_path, tooltip, keyboard shortcut)
     # icon_path isn't in the zip--icon loaded below.
-    action_spec = (name, None,
-                   'Download FanFiction stories from various web sites', ())
+    action_spec = (_('FanFictionDownLoader'), None,
+                   _('Download FanFiction stories from various web sites'), ())
     # None for keyboard shortcut doesn't allow shortcut.  () does, there just isn't one yet
 
     action_type = 'global'
@@ -108,7 +115,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         # are not found in the zip file will result in null QIcons.
         icon = get_icon('images/icon.png')
 
-        self.qaction.setText('FanFictionDL')
+        self.qaction.setText(_('FanFictionDL'))
         
         # The qaction is automatically created from the action_spec defined
         # above
@@ -231,78 +238,89 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
             self.menu.clear()
             self.actions_unique_map = {}
             self.menu_actions = []
-            self.add_action = self.create_menu_item_ex(self.menu, '&Add New from URL(s)', image='plus.png',
+            self.add_action = self.create_menu_item_ex(self.menu, _('&Add New from URL(s)'), image='plus.png',
                                                        unique_name='Add New FanFiction Book(s) from URL(s)',
-                                                       shortcut_name='Add New FanFiction Book(s) from URL(s)',
+                                                       shortcut_name=_('Add New FanFiction Book(s) from URL(s)'),
                                                        triggered=self.add_dialog )
 
-            self.update_action = self.create_menu_item_ex(self.menu, '&Update Existing FanFiction Book(s)', image='plusplus.png',
+            self.update_action = self.create_menu_item_ex(self.menu, _('&Update Existing FanFiction Book(s)'), image='plusplus.png',
+                                                          unique_name='&Update Existing FanFiction Book(s)',
                                                           triggered=self.update_dialog)
 
             if self.get_epubmerge_plugin():
                 self.menu.addSeparator()
-                self.get_list_url_action = self.create_menu_item_ex(self.menu, 'Get Story URLs to Download from Web Page', image='view.png',
+                self.get_list_url_action = self.create_menu_item_ex(self.menu, _('Get Story URLs to Download from Web Page'), image='view.png',
                                                                     unique_name='Get Story URLs from Web Page',
                                                                     triggered=self.get_urls_from_page_menu)
                 
-                self.makeanth_action = self.create_menu_item_ex(self.menu, '&Make Anthology Epub Manually from URL(s)', image='plusplus.png',
+                self.makeanth_action = self.create_menu_item_ex(self.menu, _('&Make Anthology Epub Manually from URL(s)'), image='plusplus.png',
                                                                 unique_name='Make FanFiction Anthology Epub Manually from URL(s)',
-                                                                shortcut_name='Make FanFiction Anthology Epub Manually from URL(s)',
+                                                                shortcut_name=_('Make FanFiction Anthology Epub Manually from URL(s)'),
                                                                 triggered=partial(self.add_dialog,merge=True) )
                 
-                self.updateanth_action = self.create_menu_item_ex(self.menu, '&Update Anthology Epub', image='plusplus.png',
+                self.updateanth_action = self.create_menu_item_ex(self.menu, _('&Update Anthology Epub'), image='plusplus.png',
                                                                   unique_name='Update FanFiction Anthology Epub',
-                                                                  shortcut_name='Update FanFiction Anthology Epub',
+                                                                  shortcut_name=_('Update FanFiction Anthology Epub'),
                                                                   triggered=self.update_anthology)
 
             if 'Reading List' in self.gui.iactions and (prefs['addtolists'] or prefs['addtoreadlists']) :
+                
                 self.menu.addSeparator()
                 addmenutxt, rmmenutxt = None, None
                 if prefs['addtolists'] and prefs['addtoreadlists'] :
-                    addmenutxt = 'Add to "To Read" and "Send to Device" Lists'
+                    addmenutxt = _('Add to "To Read" and "Send to Device" Lists')
                     if prefs['addtolistsonread']:
-                        rmmenutxt = 'Remove from "To Read" and add to "Send to Device" Lists'
+                        rmmenutxt = _('Remove from "To Read" and add to "Send to Device" Lists')
                     else:
-                        rmmenutxt = 'Remove from "To Read" Lists'
+                        rmmenutxt = _('Remove from "To Read" Lists')
                 elif prefs['addtolists'] :
-                    addmenutxt = 'Add Selected to "Send to Device" Lists'
+                    addmenutxt = _('Add Selected to "Send to Device" Lists')
                 elif prefs['addtoreadlists']:
-                    addmenutxt = 'Add to "To Read" Lists'
-                    rmmenutxt = 'Remove from "To Read" Lists'
+                    addmenutxt = _('Add to "To Read" Lists')
+                    rmmenutxt = _('Remove from "To Read" Lists')
 
                 if addmenutxt:
-                    self.add_send_action = self.create_menu_item_ex(self.menu, addmenutxt, image='plusplus.png',
+                    self.add_send_action = self.create_menu_item_ex(self.menu, addmenutxt,
+                                                                    unique_name='Add to "To Read" and "Send to Device" Lists',
+                                                                    image='plusplus.png',
                                                                     triggered=partial(self.update_lists,add=True))
 
                 if rmmenutxt:
-                    self.add_remove_action = self.create_menu_item_ex(self.menu, rmmenutxt, image='minusminus.png',
+                    self.add_remove_action = self.create_menu_item_ex(self.menu, rmmenutxt,
+                                                                      unique_name='Remove from "To Read" and add to "Send to Device" Lists',
+                                                                      image='minusminus.png',
                                                                       triggered=partial(self.update_lists,add=False))
                 
             self.menu.addSeparator()
-            self.get_list_action = self.create_menu_item_ex(self.menu, 'Get URLs from Selected Books', image='bookmarks.png',
+            self.get_list_action = self.create_menu_item_ex(self.menu, _('Get URLs from Selected Books'),
+                                                            unique_name='Get URLs from Selected Books',
+                                                            image='bookmarks.png',
                                                             triggered=self.list_story_urls)
 
             if not self.get_epubmerge_plugin():
-                self.get_list_url_action = self.create_menu_item_ex(self.menu, 'Get Story URLs from Web Page', image='view.png',
+                self.get_list_url_action = self.create_menu_item_ex(self.menu, _('Get Story URLs from Web Page'),
+                                                                    unique_name='Get Story URLs from Web Page',
+                                                                    image='view.png',
                                                                     triggered=self.get_urls_from_page_menu)
 
-            self.reject_list_action = self.create_menu_item_ex(self.menu, 'Reject Selected Books', image='rotate-right.png',
+            self.reject_list_action = self.create_menu_item_ex(self.menu, _('Reject Selected Books'),
+                                                               unique_name='Reject Selected Books', image='rotate-right.png',
                                                                triggered=self.reject_list_urls)
 
             # print("platform.system():%s"%platform.system())
             # print("platform.mac_ver()[0]:%s"%platform.mac_ver()[0])
             if not self.check_macmenuhack(): # not platform.mac_ver()[0]: # Some macs crash on these menu items for unknown reasons.
                 self.menu.addSeparator()
-                self.config_action = self.create_menu_item_ex(self.menu, '&Configure Plugin',
+                self.config_action = self.create_menu_item_ex(self.menu, _('&Configure Plugin'),
                                                               image= 'config.png',
                                                               unique_name='Configure FanFictionDownLoader',
-                                                              shortcut_name='Configure FanFictionDownLoader',
+                                                              shortcut_name=_('Configure FanFictionDownLoader'),
                                                               triggered=partial(do_user_config,parent=self.gui))
             
-                self.about_action = self.create_menu_item_ex(self.menu, 'About Plugin',
+                self.about_action = self.create_menu_item_ex(self.menu, _('About Plugin'),
                                                              image= 'images/icon.png',
                                                              unique_name='About FanFictionDownLoader', 
-                                                             shortcut_name='About FanFictionDownLoader', 
+                                                             shortcut_name=_('About FanFictionDownLoader'), 
                                                              triggered=self.about)
             
             # Before we finalize, make sure we delete any actions for menus that are no longer displayed
@@ -373,7 +391,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
             except:
                 urltxt = ""
         
-        d = CollectURLDialog(self.gui,"Get Story URLs from Web Page",urltxt,self.get_epubmerge_plugin())
+        d = CollectURLDialog(self.gui,_("Get Story URLs from Web Page"),urltxt,self.get_epubmerge_plugin())
         d.exec_()
         if not d.status:
             return
@@ -418,9 +436,9 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                            book_list,
                            partial(self.get_list_story_urls_loop, db=self.gui.current_db),
                            self.get_list_story_urls_finish,
-                           init_label="Collecting URLs for stories...",
-                           win_title="Get URLs for stories",
-                           status_prefix="URL retrieved")
+                           init_label=_("Collecting URLs for stories..."),
+                           win_title=_("Get URLs for stories"),
+                           status_prefix=_("URL retrieved"))
             
     def get_list_story_urls_loop(self,book,db=None):
         if book['calibre_id']:
@@ -465,9 +483,9 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                            book_list,
                            partial(self.reject_list_urls_loop, db=self.gui.current_db),
                            self.reject_list_urls_finish,
-                           init_label="Collecting URLs for Reject List...",
-                           win_title="Get URLs for Reject List",
-                           status_prefix="URL retrieved")
+                           init_label=_("Collecting URLs for Reject List..."),
+                           win_title=_("Get URLs for Reject List"),
+                           status_prefix=_("URL retrieved"))
             
     def reject_list_urls_loop(self,book,db=None):
         self.get_list_story_urls_loop(book,db) # common with get_list_story_urls_loop
@@ -501,7 +519,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                 self.gui.iactions['Remove Books'].do_library_delete(d.get_reject_list_ids())
             
         else:
-            message="<p>Rejecting FFDL URLs: None of the books selected have FanFiction URLs.</p><p>Proceed to Remove?</p>"
+            message="<p>"+_("Rejecting FFDL URLs: None of the books selected have FanFiction URLs.")+"</p><p>"+_("Proceed to Remove?")+"</p>"
             if confirm(message,'fanfictiondownloader_reject_non_fanfiction', self.gui):
                 self.gui.iactions['Remove Books'].delete_books()
 
@@ -523,7 +541,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
 
     def update_anthology(self):
         if not self.get_epubmerge_plugin():
-            self.gui.status_bar.show_message(_('Cannot Make Anthologys without EpubMerge 1.3.0+'), 3000)
+            self.gui.status_bar.show_message(_('Cannot Make Anthologys without %s')%'EpubMerge 1.3.1+', 3000)
             return
             
         if not self.is_library_view():
@@ -559,7 +577,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
 
         if not filenames or len(filenames) != len (url_list):
             info_dialog(self.gui, _("Cannot Update Anthology"),
-                        _("<p>Cannot Update Anthology</p><p>Book isn't an FFDL Anthology or contains book(s) without valid FFDL URLs."),
+                        "<p>"+_("Cannot Update Anthology")+"</p><p>"+_("Book isn't an FFDL Anthology or contains book(s) without valid FFDL URLs."),
                         show=True,
                         show_copy_button=False)
             remove_dir(tdir)
@@ -611,14 +629,17 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
 
         if urlmapfile:
             text = '''
-            <p>There are %d stories in the current anthology that are <b>not</b> going kept if you go ahead.</p>
-            <p>Story URLs that will be removed:</p>
-            <ul>
-            <li>%s</li>
-            </ul>
-            <p>Update anyway?</p>
-            '''%(len(urlmapfile),"</li><li>".join(urlmapfile.keys()))
-            if not question_dialog(self.gui, 'Stories Removed',
+                 <p>%s</p>
+                 <p>%s</p>
+                 <ul>
+                 <li>%s</li>
+                 </ul>
+                 <p>%s</p>'''%(
+                _('There are %d stories in the current anthology that are <b>not</b> going to be kept if you go ahead.')%len(urlmapfile),
+                _('Story URLs that will be removed:'),
+                "</li><li>".join(urlmapfile.keys()),
+                _('Update anyway?'))
+            if not question_dialog(self.gui, _('Stories Removed'),
                                text, show_copy_button=False):
                 logger.debug("Canceling anthology update due to removed stories.")
                 return
@@ -640,18 +661,18 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         #print("update_dialog()")
         
         db = self.gui.current_db
-        book_list = map( self.make_book_id_only, id_list )
+        books = map( self.make_book_id_only, id_list )
 
-        for j, book in enumerate(book_list):
+        for j, book in enumerate(books):
             book['listorder'] = j
-            
+
         LoopProgressDialog(self.gui,
-                           book_list,
+                           books,
                            partial(self.populate_book_from_calibre_id, db=self.gui.current_db),
                            self.update_dialog_finish,
-                           init_label="Collecting stories for update...",
-                           win_title="Get stories for updates",
-                           status_prefix="URL retrieved")            
+                           init_label=_("Collecting stories for update..."),
+                           win_title=_("Get stories for updates"),
+                           status_prefix=_("URL retrieved"))            
         
         #books = self.convert_calibre_ids_to_books(db, book_ids)
         #print("update books:%s"%books)
@@ -660,7 +681,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         '''Present list to update and head to prep when done.'''
         
         d = UpdateExistingDialog(self.gui,
-                                 'Update Existing List',
+                                 _('Update Existing List'),
                                  prefs,
                                  self.qaction.icon(),
                                  book_list,
@@ -698,6 +719,9 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
             url_list = split_text_to_urls(books)
             books = self.convert_urls_to_books(url_list)
 
+        ## for tweak_fg_sleep
+        options['ffnetcount']=len(filter(lambda x : x['site']=='www.fanfiction.net', books))
+
         options['version'] = self.version
         logger.debug(self.version)
         
@@ -709,7 +733,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
             options['tdir']=tdir
 
         if 0 < len(filter(lambda x : x['good'], books)):
-            self.gui.status_bar.show_message(_('Started fetching metadata for %s stories.'%len(books)), 3000)
+            self.gui.status_bar.show_message(_('Started fetching metadata for %s stories.')%len(books), 3000)
             LoopProgressDialog(self.gui,
                                books,
                                partial(self.prep_download_loop, options = options, merge=merge),
@@ -740,23 +764,35 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         if not merge: # skip reject list when merging.
             if rejecturllist.check(url):
                 rejnote = rejecturllist.get_full_note(url)
-                if question_dialog(self.gui, 'Reject URL?',
-                                   '<h3>Reject URL?</h3>'+
-                                   '<p><b>%s</b> is on your Reject URL list:</p><p>"<b>%s</b>"</p>'%(url,rejnote)+
-                                   "<p>Click '<b>Yes</b>' to Reject.</p>"+
-                                   "<p>Click '<b>No</b>' to download anyway.</p>",
+                if question_dialog(self.gui, _('Reject URL?'),'''
+                          <h3>%s</h3>
+                          <p>%s</p>
+                          <p>"<b>%s</b>"</p>
+                          <p>%s</p>
+                          <p>%s</p>'''%(
+                        _('Reject URL?'),
+                        _('<b>%s</b> is on your Reject URL list:')%url,
+                        rejnote,
+                        _("Click '<b>Yes</b>' to Reject."),
+                        _("Click '<b>No</b>' to download anyway.")),
                                    show_copy_button=False):
-                    book['comment'] = "Story on Reject URLs list (%s)."%rejnote
+                    book['comment'] = _("Story on Reject URLs list (%s).")%rejnote
                     book['good']=False
                     book['icon']='rotate-right.png'
-                    book['status'] = 'Rejected'
+                    book['status'] = _('Rejected')
                     return
                 else:
-                    if question_dialog(self.gui, 'Remove Reject URL?',
-                                       "<h3>Remove URL from Reject List?</h3>"+
-                                       '<p><b>%s</b> is on your Reject URL list:</p><p>"<b>%s</b>"</p>'%(url,rejnote)+
-                                       "<p>Click '<b>Yes</b>' to remove it from the list,</p>"+
-                                       "<p>Click '<b>No</b>' to leave it on the list.</p>",
+                    if question_dialog(self.gui, _('Remove Reject URL?'),'''
+                              <h3>%s</h3>
+                              <p>%s</p>
+                              <p>"<b>%s</b>"</p>
+                              <p>%s</p>
+                              <p>%s</p>'''%(
+                            _("Remove URL from Reject List?"),
+                            _('<b>%s</b> is on your Reject URL list:')%url,
+                            rejnote,
+                            _("Click '<b>Yes</b>' to remove it from the list,"),
+                            _("Click '<b>No</b>' to leave it on the list.")),
                                        show_copy_button=False):
                         rejecturllist.remove(url)
             
@@ -773,7 +809,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
 
         # Dialogs should prevent this case now.
         if collision in (UPDATE,UPDATEALWAYS) and fileform != 'epub':
-            raise NotGoingToDownload("Cannot update non-epub format.")
+            raise NotGoingToDownload(_("Cannot update non-epub format."))
         
         if not book['good']:
             # book has already been flagged bad for whatever reason.
@@ -783,6 +819,18 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         
         options['personal.ini'] = get_ffdl_personalini()
         adapter = get_ffdl_adapter(url,fileform)
+        # reduce foreground sleep time for ffnet when few books.
+        if 'ffnetcount' in options and \
+                adapter.getConfig('tweak_fg_sleep') and \
+                adapter.getSiteDomain() == 'www.fanfiction.net':
+            minslp = float(adapter.getConfig('min_fg_sleep'))
+            maxslp = float(adapter.getConfig('max_fg_sleep'))
+            dwnlds = float(adapter.getConfig('max_fg_sleep_at_downloads'))
+            m = (maxslp-minslp) / (dwnlds-1)
+            b = minslp - m
+            slp = min(maxslp,m*float(options['ffnetcount'])+b)
+            #print("m:%s b:%s = %s"%(m,b,slp))
+            adapter.set_sleep(slp)
 
         ## three tries, that's enough if both user/pass & is_adult needed,
         ## or a couple tries of one or the other
@@ -798,8 +846,8 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                     adapter.password = userpass.passwd.text()
                 
             except exceptions.AdultCheckRequired:
-                if question_dialog(self.gui, 'Are You Adult?', '<p>'+
-                                   "%s requires that you be an adult.  Please confirm you are an adult in your locale:"%url,
+                if question_dialog(self.gui, _('Are You an Adult?'), '<p>'+
+                                   _("%s requires that you be an adult.  Please confirm you are an adult in your locale:")%url,
                                    show_copy_button=False):
                     adapter.is_adult=True
 
@@ -809,23 +857,27 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         series = story.getMetadata('series')
         if not merge and series and prefs['checkforseriesurlid']:
             # try to find *series anthology* by *seriesUrl* identifier url or uri first.
-            searchstr = 'identifiers:"~ur(i|l):=%s"'%story.getMetadata('seriesUrl').replace(":","|")
+            searchstr = 'identifiers:"~ur(i|l):~^%s$"'%re.sub(r'https?\\:','https?(\:|\|)',re.escape(story.getMetadata('seriesUrl')))
             identicalbooks = db.search_getting_ids(searchstr, None)
             # print("searchstr:%s"%searchstr)
             # print("identicalbooks:%s"%identicalbooks)
-            if len(identicalbooks) > 0 and question_dialog(self.gui, 'Skip Story?',
-                                                           '<h3>Skip Anthology Story?</h3>'+
-                                                           '<p>"<b>%s</b>" is in series "<b><a href="%s">%s</a></b>" that you have an anthology book for.</p>'%
-                                                           (story.getMetadata('title'),story.getMetadata('seriesUrl'),series[:series.index(' [')])+
-                                                           "<p>Click '<b>Yes</b>' to Skip.</p>"+
-                                                           "<p>Click '<b>No</b>' to download anyway.</p>",
+            if len(identicalbooks) > 0 and question_dialog(self.gui, _('Skip Story?'),'''
+                                                              <h3>%s</h3>
+                                                              <p>%s</p>
+                                                              <p>%s</p>
+                                                              <p>%s</p>
+                                                           '''%(
+                                                           _('Skip Anthology Story?'),
+                                                           _('"<b>%s</b>" is in series "<b><a href="%s">%s</a></b>" that you have an anthology book for.')%(story.getMetadata('title'),story.getMetadata('seriesUrl'),series[:series.index(' [')]),
+                                                           _("Click '<b>Yes</b>' to Skip."),
+                                                           _("Click '<b>No</b>' to download anyway.")),
                                                            show_copy_button=False):
-                book['comment'] = "Story in Series Anthology(%s)."%series
+                book['comment'] = _("Story in Series Anthology(%s).")%series
                 book['title'] = story.getMetadata('title')
                 book['author'] = [story.getMetadata('author')]
                 book['good']=False
                 book['icon']='rotate-right.png'
-                book['status'] = 'Skipped'
+                book['status'] = _('Skipped')
                 return
                 
 
@@ -853,7 +905,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         book['password'] = adapter.password
 
         book['icon'] = 'plus.png'
-        book['status'] = 'Add'
+        book['status'] = _('Add')
         if story.getMetadataRaw('datePublished'):
             book['pubdate'] = story.getMetadataRaw('datePublished').replace(tzinfo=local_tz)
         if story.getMetadataRaw('dateUpdated'):
@@ -866,7 +918,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         if not merge:# skip all the collision code when d/ling for merging.
             if collision in (CALIBREONLY):
                 book['icon'] = 'metadata.png'
-                book['status'] = 'Meta'
+                book['status'] = _('Meta')
             
             book_id = None
             
@@ -882,8 +934,10 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                 logger.debug("from URL(%s)"%url)
     
                 # try to find by identifier url or uri first.
-                searchstr = 'identifiers:"~ur(i|l):=%s"'%url.replace(":","|")
+                searchstr = 'identifiers:"~ur(i|l):~^%s$"'%re.sub(r'https?\:','https?(\:|\|)',url)
                 identicalbooks = db.search_getting_ids(searchstr, None)
+                # print("searchstr:%s"%searchstr)
+                # print("identicalbooks:%s"%identicalbooks)
                 if len(identicalbooks) < 1:
                     # find dups
                     authlist = story.getList("author", removeallentities=True)
@@ -904,10 +958,10 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                     logger.debug("existing found by identifier URL")
                     
                 if collision == SKIP and identicalbooks:
-                    raise NotGoingToDownload("Skipping duplicate story.","list_remove.png")
+                    raise NotGoingToDownload(_("Skipping duplicate story."),"list_remove.png")
     
                 if len(identicalbooks) > 1:
-                    raise NotGoingToDownload("More than one identical book by Identifer URL or title/author(s)--can't tell which book to update/overwrite.","minusminus.png")
+                    raise NotGoingToDownload(_("More than one identical book by Identifer URL or title/author(s)--can't tell which book to update/overwrite."),"minusminus.png")
     
                 ## changed: add new book when CALIBREONLY if none found.
                 if collision == CALIBREONLY and not identicalbooks:
@@ -918,47 +972,54 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                     book_id = identicalbooks.pop()
                     book['calibre_id'] = book_id
                     book['icon'] = 'edit-redo.png'
-                    book['status'] = 'Update'
+                    book['status'] = _('Update')
     
                 if book_id and mi: # book_id and mi only set if matched by title/author.
                     liburl = self.get_story_url(db,book_id)
-                    if book['url'] != liburl and prefs['checkforurlchange']:
-                        if collision in (OVERWRITE,OVERWRITEALWAYS):
-                            updat="overwrit"
-                        else:
-                            updat="updat"
-                        if not question_dialog(self.gui, 'Change Story URL?',
-                                               '<h3>Change Story URL?</h3>'+
-                                               '<p><b>%s</b> by <b>%s</b> is already in your library with a different source URL:</p>'%
-                                                   (mi.title,', '.join(mi.author))+
-                                               '<p>In library: <a href="%(liburl)s">%(liburl)s</a></p><p>New URL: <a href="%(newurl)s">%(newurl)s</a></p>'%
-                                                   {'liburl':liburl,'newurl':book['url']}+
-                                               "<p>Click '<b>Yes</b>' to %se book with new URL.</p>"%updat+
-                                               "<p>Click '<b>No</b>' to skip %sing this book.</p>"%updat,
+                    if book['url'] != liburl and prefs['checkforurlchange'] and \
+                        not (book['url'].replace('https','http') == liburl and 'fanfiction.net' in liburl):
+                        # special case for ffnet urls change to https.
+                        if not question_dialog(self.gui, _('Change Story URL?'),'''
+                                                  <h3>%s</h3>
+                                                  <p>%s</p>
+                                                  <p>%s</p>
+                                                  <p>%s</p>
+                                                  <p>%s</p>
+                                                  <p>%s</p>'''%(
+                                _('Change Story URL?'),
+                                _('<b>%s</b> by <b>%s</b> is already in your library with a different source URL:')%(mi.title,', '.join(mi.author)),
+                                _('In library: <a href="%(liburl)s">%(liburl)s</a>')%{'liburl':liburl},
+                                _('New URL: <a href="%(newurl)s">%(newurl)s</a>')%{'newurl':book['url']},
+                                _("Click '<b>Yes</b>' to update/overwrite book with new URL."),
+                                _("Click '<b>No</b>' to skip updating/overwriting this book.")),
                                                show_copy_button=False):
-                            if question_dialog(self.gui, 'Download as New Book?',
-                                               '<h3>Download as New Book?</h3>'+
-                                               '<p><b>%s</b> by <b>%s</b> is already in your library with a different source URL.</p>'%
-                                               (mi.title,', '.join(mi.author))+
-                                               '<p>You chose not to update the existing book.  Do you want to add a new book for this URL?</p>'+
-                                               '<p>New URL: <a href="%(newurl)s">%(newurl)s</a></p>'%
-                                               {'newurl':book['url']}+
-                                               "<p>Click '<b>Yes</b>' to a new book with new URL.</p>"+
-                                               "<p>Click '<b>No</b>' to skip URL.</p>",
+                            if question_dialog(self.gui, _('Download as New Book?'),'''
+                                                  <h3>%s</h3>
+                                                  <p>%s</p>
+                                                  <p>%s</p>
+                                                  <p>%s</p>
+                                                  <p>%s</p>
+                                                  <p>%s</p>'''%(
+                                    _('Download as New Book?'),
+                                    _('<b>%s</b> by <b>%s</b> is already in your library with a different source URL.')%(mi.title,', '.join(mi.author)),
+                                    _('You chose not to update the existing book.  Do you want to add a new book for this URL?'),
+                                    _('New URL: <a href="%(newurl)s">%(newurl)s</a>')%{'newurl':book['url']},
+                                    _("Click '<b>Yes</b>' to a new book with new URL."),
+                                    _("Click '<b>No</b>' to skip URL.")),
                                                show_copy_button=False):
                                 book_id = None
                                 mi = None
                                 book['calibre_id'] = None
                             else:
-                                book['comment'] = "Update declined by user due to differing story URL(%s)"%liburl
+                                book['comment'] = _("Update declined by user due to differing story URL(%s)")%liburl
                                 book['good']=False
                                 book['icon']='rotate-right.png'
-                                book['status'] = 'Different URL'
+                                book['status'] = _('Different URL')
                                 return
         
             if book_id != None and collision != ADDNEW:
                 if collision in (CALIBREONLY):
-                    book['comment'] = 'Metadata collected.'
+                    book['comment'] = _('Metadata collected.')
                     # don't need temp file created below.
                     return
                 
@@ -971,25 +1032,31 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                         (epuburl,chaptercount) = \
                             get_dcsource_chaptercount(StringIO(db.format(book_id,'EPUB',
                                                                          index_is_id=True)))
-                        urlchaptercount = int(story.getMetadata('numChapters'))
+                        urlchaptercount = int(story.getMetadata('numChapters').replace(',',''))
                         if chaptercount == urlchaptercount:
                             if collision == UPDATE:
-                                raise NotGoingToDownload("Already contains %d chapters."%chaptercount,'edit-undo.png')
+                                raise NotGoingToDownload(_("Already contains %d chapters.")%chaptercount,'edit-undo.png')
                             else:
                                 # UPDATEALWAYS
                                 skip_date_update = True
                         elif chaptercount > urlchaptercount:
-                            raise NotGoingToDownload("Existing epub contains %d chapters, web site only has %d. Use Overwrite to force update." % (chaptercount,urlchaptercount),'dialog_error.png')
+                            raise NotGoingToDownload(_("Existing epub contains %d chapters, web site only has %d. Use Overwrite to force update.") % (chaptercount,urlchaptercount),'dialog_error.png')
                         elif chaptercount == 0:
-                            raise NotGoingToDownload("FFDL doesn't recognize chapters in existing epub, epub is probably from a different source. Use Overwrite to force update.",'dialog_error.png')
+                            raise NotGoingToDownload(_("FFDL doesn't recognize chapters in existing epub, epub is probably from a different source. Use Overwrite to force update."),'dialog_error.png')
         
                 if collision == OVERWRITE and \
                         db.has_format(book_id,formmapping[fileform],index_is_id=True):
                     # check make sure incoming is newer.
-                    lastupdated=story.getMetadataRaw('dateUpdated').date()
-                    fileupdated=datetime.fromtimestamp(os.stat(db.format_abspath(book_id, formmapping[fileform], index_is_id=True))[8]).date()
-                    if fileupdated > lastupdated:
-                        raise NotGoingToDownload("Not Overwriting, web site is not newer.",'edit-undo.png')
+                    lastupdated=story.getMetadataRaw('dateUpdated')
+                    fileupdated=datetime.fromtimestamp(os.stat(db.format_abspath(book_id, formmapping[fileform], index_is_id=True))[8])
+
+                    # updated doesn't have time (or is midnight), use dates only.
+                    # updated does have time, use full timestamps.
+                    if (lastupdated.time() == time.min and fileupdated.date() > lastupdated.date()) or \
+                            (lastupdated.time() != time.min and fileupdated > lastupdated):
+                        raise NotGoingToDownload(_("Not Overwriting, web site is not newer."),'edit-undo.png')
+                    
+                    
         
                 # For update, provide a tmp file copy of the existing epub so
                 # it can't change underneath us.  Now also overwrite for logpage preserve.
@@ -1053,17 +1120,19 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
             ## No good stories to try to download, go straight to
             ## updating error col.
             msg = '''
-<p>None of the <b>%d</b> URLs/stories given can be/need to be downloaded.</p>
-<p>See log for details.</p>
-<p>Proceed with updating your library(Error Column, if configured)?</p>
-'''%len(book_list)
+                    <p>%s</p>
+                    <p>%s</p>
+                    <p>%s</p>'''%(
+                _('None of the <b>%d</b> URLs/stories given can be/need to be downloaded.')%len(book_list),
+                _('See log for details.'),
+                _('Proceed with updating your library(Error Column, if configured)?'))
     
-            htmllog='<html><body><table border="1"><tr><th>Status</th><th>Title</th><th>Author</th><th>Comment</th><th>URL</th></tr>'
+            htmllog='<html><body><table border="1"><tr><th>'+_('Status')+'</th><th>'+_('Title')+'</th><th>'+_('Author')+'</th><th>'+_('Comment')+'</th><th>URL</th></tr>'
             for book in book_list:
                 if 'status' in book:
                     status = book['status']
                 else:
-                    status = 'Bad'
+                    status = _('Bad')
                 htmllog = htmllog + '<tr><td>' + '</td><td>'.join([escapehtml(status),escapehtml(book['title']),escapehtml(", ".join(book['author'])),escapehtml(book['comment']),book['url']]) + '</td></tr>'
             
             htmllog = htmllog + '</table></body></html>'
@@ -1071,7 +1140,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
             payload = ([], book_list, options)
             self.gui.proceed_question(self.update_error_column,
                                       payload, htmllog,
-                                      'FFDL log', 'FFDL download ended', msg,
+                                      _('FFDL log'), _('FFDL download ended'), msg,
                                       show_copy_button=False)
             return
             
@@ -1079,13 +1148,14 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         cpus = self.gui.job_manager.server.pool_size
         args = ['calibre_plugins.fanfictiondownloader_plugin.jobs', 'do_download_worker',
                 (book_list, options, cpus)]
-        desc = 'Download FanFiction Book'
+        desc = _('Download FanFiction Book')
         job = self.gui.job_manager.run_job(
                 self.Dispatcher(partial(self.download_list_completed,options=options,merge=merge)),
                 func, args=args,
                 description=desc)
         
-        self.gui.status_bar.show_message('Starting %d FanFictionDownLoads'%len(book_list),3000)
+        self.gui.jobs_pointer.start()
+        self.gui.status_bar.show_message(_('Starting %d FanFictionDownLoads')%len(book_list),3000)
 
     def update_books_loop(self,book,db=None,
                      options={'fileform':'epub',
@@ -1115,12 +1185,12 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
             try:
                 self.update_metadata(db, book['calibre_id'], book, mi, options)
             except:
-                det_msg = "".join(traceback.format_exception(*sys.exc_info()))+"\nStory Details:\n%s"%pretty_book(book)
+                det_msg = "".join(traceback.format_exception(*sys.exc_info()))+"\n"+_("Story Details:")+pretty_book(book)
                 logger.error("Error Updating Metadata:\n%s"%det_msg)
                 error_dialog(self.gui,
-                             "Error Updating Metadata",
-                             "<p>An error has occurred while FFDL was updating calibre's metadata for <a href='%s'>%s</a>.</p>"%(book['url'],book['title'])+
-                             "The ebook has been updated, but the metadata has not.",
+                             _("Error Updating Metadata"),
+                             "<p>"+_("An error has occurred while FFDL was updating calibre's metadata for <a href='%s'>%s</a>.")%(book['url'],book['title'])+"</p>"+
+                             _("The ebook has been updated, but the metadata has not."),
                              det_msg=det_msg,
                              show=True)
 
@@ -1172,13 +1242,17 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                 # Sort by our marked column to display the books in order
                 self.gui.library_view.sort_by_named_field('marked', True)
                 
-        self.gui.status_bar.show_message(_('Finished Adding/Updating %d books.'%(len(update_list) + len(add_list))), 3000)
+        self.gui.status_bar.show_message(_('Finished Adding/Updating %d books.')%(len(update_list) + len(add_list)), 3000)
         remove_dir(options['tdir'])
         
         if 'Count Pages' in self.gui.iactions and len(prefs['countpagesstats']) and len(all_ids):
             cp_plugin = self.gui.iactions['Count Pages']
             cp_plugin.count_statistics(all_ids,prefs['countpagesstats'])
 
+        if prefs['autoconvert'] and options['collision'] != CALIBREONLY:
+            self.gui.status_bar.show_message(_('Starting auto conversion of %d books.')%(len(all_ids)), 3000)
+            self.gui.iactions['Convert Books'].auto_convert_auto_add(all_ids)
+
     def download_list_completed(self, job, options={},merge=False):
         if job.failed:
             self.gui.job_exception(job, dialog_title='Failed to Download Stories')
@@ -1203,24 +1277,29 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                             show_copy_button=False)
                 return
 
-            msg = '<p>FFDL found <b>%s</b> good and <b>%s</b> bad updates.</p>'%(len(good_list),len(bad_list))
+            msg = '<p>'+_('FFDL found <b>%s</b> good and <b>%s</b> bad updates.')%(len(good_list),len(bad_list))+'</p>'
             if len(bad_list) > 0:
-                msg = msg + '''<p>Are you sure you want to continue with creating/updating this Anthology?</p>
-<p>Any updates that failed will <b>not</b> be included in the Anthology.</p>
-<p>However, if there's an older version, it will still be included.</p>
-<p>See log for details.</p>
-'''
-            msg = msg + '<p>Proceed with updating this anthology and your library?</p>'
+                msg = msg + '''
+                            <p>%s</p>
+                            <p>%s</p>
+                            <p>%s</p>
+                            <p>%s</p>'''%(
+                    _('Are you sure you want to continue with creating/updating this Anthology?'),
+                    _('Any updates that failed will <b>not</b> be included in the Anthology.'),
+                    _("However, if there's an older version, it will still be included."),
+                    _('See log for details.'))
+
+            msg = msg + '<p>'+_('Proceed with updating this anthology and your library?')+ '</p>'
     
-            htmllog='<html><body><table border="1"><tr><th>Status</th><th>Title</th><th>Author</th><th>Comment</th><th>URL</th></tr>'
+            htmllog='<html><body><table border="1"><tr><th>'+_('Status')+'</th><th>'+_('Title')+'</th><th>'+_('Author')+'</th><th>'+_('Comment')+'</th><th>URL</th></tr>'
             for book in sorted(good_list+bad_list,key=lambda x : x['listorder']):
                 if 'status' in book:
                     status = book['status']
                 else:
                     if book in good_list:
-                        status = 'Good'
+                        status = _('Good')
                     else:
-                        status = 'Bad'
+                        status = _('Bad')
                 htmllog = htmllog + '<tr><td>' + '</td><td>'.join([escapehtml(status),escapehtml(book['title']),escapehtml(", ".join(book['author'])),escapehtml(book['comment']),book['url']]) + '</td></tr>'
             
             htmllog = htmllog + '</table></body></html>'
@@ -1234,12 +1313,15 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
             do_update_func = self.do_download_merge_update
         else:        
             msg = '''
-<p>FFDL found <b>%s</b> good and <b>%s</b> bad updates.</p>
-<p>See log for details.</p>
-<p>Proceed with updating your library?</p>
-'''%(len(good_list),len(bad_list))
+                  <p>%s</p>
+                  <p>%s</p>
+                  <p>%s</p>'''%(
+                _('FFDL found <b>%s</b> good and <b>%s</b> bad updates.')%(len(good_list),len(bad_list)),
+                _('See log for details.'),
+                _('Proceed with updating your library?')
+                )
     
-            htmllog='<html><body><table border="1"><tr><th>Status</th><th>Title</th><th>Author</th><th>Comment</th><th>URL</th></tr>'
+            htmllog='<html><body><table border="1"><tr><th>'+_('Status')+'</th><th>'+_('Title')+'</th><th>'+_('Author')+'</th><th>'+_('Comment')+'</th><th>URL</th></tr>'
             for book in good_list:
                 if 'status' in book:
                     status = book['status']
@@ -1260,7 +1342,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
             
         self.gui.proceed_question(do_update_func,
                                   payload, htmllog,
-                                  'FFDL log', 'FFDL download complete', msg,
+                                  _('FFDL log'), _('FFDL download complete'), msg,
                                   show_copy_button=False)
         
     def do_download_merge_update(self, payload):
@@ -1273,8 +1355,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         good_list = sorted(good_list,key=lambda x : x['listorder'])
         bad_list = sorted(bad_list,key=lambda x : x['listorder'])
         
-        self.gui.status_bar.show_message(_('Merging %s books.'%total_good))
-
+        self.gui.status_bar.show_message(_('Merging %s books.')%total_good)
         
         existingbook = None
         if 'mergebook' in options:
@@ -1321,9 +1402,9 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                                good_list+bad_list,
                                partial(self.update_books_loop, options=options, db=self.gui.current_db),
                                partial(self.update_books_finish, options=options),
-                               init_label="Updating calibre for FanFiction stories...",
-                               win_title="Update calibre for FanFiction stories",
-                               status_prefix="Updated")
+                               init_label=_("Updating calibre for FanFiction stories..."),
+                               win_title=_("Update calibre for FanFiction stories"),
+                               status_prefix=_("Updated"))
 
     def update_error_column(self,payload):
         '''Update custom error column if configured.'''
@@ -1331,22 +1412,23 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         custom_columns = self.gui.library_view.model().custom_columns
         if prefs['mark'] or (prefs['errorcol'] != '' and prefs['errorcol'] in custom_columns):
             self.previous = self.gui.library_view.currentIndex() # used by update_books_finish.
-            self.gui.status_bar.show_message(_('Adding/Updating %s BAD books.'%len(book_list)))
-            label = custom_columns[prefs['errorcol']]['label']
+            self.gui.status_bar.show_message(_('Adding/Updating %s BAD books.')%len(book_list))
+            if (prefs['errorcol'] != '' and prefs['errorcol'] in custom_columns):
+                label = custom_columns[prefs['errorcol']]['label']
+            else:
+                label = None
             LoopProgressDialog(self.gui,
                                book_list,
                                partial(self.update_error_column_loop, db=self.gui.current_db, label=label),
                                partial(self.update_books_finish, options=options),
-                               init_label="Updating calibre for BAD FanFiction stories...",
-                               win_title="Update calibre for BAD FanFiction stories",
-                               status_prefix="Updated")
+                               init_label=_("Updating calibre for BAD FanFiction stories..."),
+                               win_title=_("Update calibre for BAD FanFiction stories"),
+                               status_prefix=_("Updated"))
 
-    def update_error_column_loop(self,book,db=None,label='errorcol'):
-        if book['calibre_id']:
-            custom_columns = self.gui.library_view.model().custom_columns
-            if (prefs['errorcol'] != '' and prefs['errorcol'] in custom_columns):
-                logger.debug("add/update bad %s %s %s"%(book['title'],book['url'],book['comment']))
-                db.set_custom(book['calibre_id'], book['comment'], label=label, commit=True)
+    def update_error_column_loop(self,book,db=None,label=None):
+        if book['calibre_id'] and label:
+            logger.debug("add/update bad %s %s %s"%(book['title'],book['url'],book['comment']))
+            db.set_custom(book['calibre_id'], book['comment'], label=label, commit=True)
 
     def add_book_or_update_format(self,book,options,prefs,mi=None):
         db = self.gui.current_db
@@ -1366,17 +1448,27 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         if not db.add_format_with_hooks(book_id,
                                         options['fileform'],
                                         book['outfile'], index_is_id=True):
-            book['comment'] = "Adding format to book failed for some reason..."
+            book['comment'] = _("Adding format to book failed for some reason...")
             book['good']=False
             book['icon']='dialog_error.png'
-            book['status'] = 'Error'
+            book['status'] = _('Error')
 
         if prefs['deleteotherforms']:
             fmts = db.formats(book['calibre_id'], index_is_id=True).split(',')
             for fmt in fmts:
                 if fmt != formmapping[options['fileform']]:
-                    logger.debug("remove f:"+fmt)
+                    logger.debug("deleteotherforms remove f:"+fmt)
                     db.remove_format(book['calibre_id'], fmt, index_is_id=True)#, notify=False
+        elif prefs['autoconvert']:
+            ## 'Convert Book'.auto_convert_auto_add doesn't convert if
+            ## the format is already there.
+            fmt = calibre_prefs['output_format']
+            # delete if there, but not if the format we just made.
+            if fmt != formmapping[options['fileform']] and \
+                    db.has_format(book_id,fmt,index_is_id=True):
+                logger.debug("autoconvert remove f:"+fmt)
+                db.remove_format(book['calibre_id'], fmt, index_is_id=True)#, notify=False
+                
 
         return book_id
 
@@ -1484,8 +1576,15 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                         (custcol,flag) = map( lambda x: x.strip(), custcol.split(",") )
 
                     if meta not in book['all_metadata']:
-                        logger.debug("No value for %s, skipping custom column(%s) update."%(meta,custcol))
-                        continue
+                        # if double quoted, use as a literal value.
+                        if meta[0] == '"' and meta[-1] == '"':
+                            val = meta[1:-1]
+                            logger.debug("No metadata value for %s, setting custom column(%s) literally to %s."%(meta,custcol,val))
+                        else:
+                            logger.debug("No value for %s, skipping custom column(%s) update."%(meta,custcol))
+                            continue
+                    else:
+                        val = book['all_metadata'][meta]    
                     
                     if custcol not in custom_columns:
                         continue
@@ -1497,12 +1596,19 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                         if coldef['datatype'] in ('int','float'): # for favs, etc--site specific metadata.
                             if 'anthology_meta_list' in book and meta in book['anthology_meta_list']:
                                 # re-split list, strip commas, convert to floats, sum up.
-                                val = sum([ float(x.replace(",","")) for x in book['all_metadata'][meta].split(", ") ])
+                                val = sum([ float(x.replace(",","")) for x in val.split(", ") ])
                             else:
-                                val = unicode(book['all_metadata'][meta]).replace(",","")
+                                val = unicode(val).replace(",","")
                         else:
-                            val = book['all_metadata'][meta]
+                            val = val
                         if val != '':
+                            if coldef['datatype'] == 'bool':
+                                if val.lower() in ('t','true','1','yes','y'):
+                                    val = True
+                                elif val.lower() in ('f','false','0','no','n'):
+                                    val = False
+                                else:
+                                    val = None # for tri-state 'booleans'. Yes/No/Null
                             db.set_custom(book_id, val, label=label, commit=False)
 
                     if flag == 'a':
@@ -1516,8 +1622,8 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                         except:
                             pass
 
-                        if book['all_metadata'][meta]:
-                            vallist = [book['all_metadata'][meta]]
+                        if val:
+                            vallist = [val]
                             
                         db.set_custom(book_id, ", ".join(vallist), label=label, commit=False)
 
@@ -1548,6 +1654,8 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         db.commit()
 
         if 'Generate Cover' in self.gui.iactions and (book['added'] or not prefs['gcnewonly']):
+
+            #logger.debug("Do Generate Cover added:%s gcnewonly:%s"%(book['added'],prefs['gcnewonly']))
             
             # force a refresh if generating cover so complex composite
             # custom columns are current and correct
@@ -1579,15 +1687,39 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                 
             if not setting_name and book['all_metadata']['site'] in prefs['gc_site_settings']:
                 setting_name =  prefs['gc_site_settings'][book['all_metadata']['site']]
+                logger.debug("Generate Cover Setting from site(%s)"%setting_name)
 
             if not setting_name and 'Default' in prefs['gc_site_settings']:
                 setting_name =  prefs['gc_site_settings']['Default']
+                logger.debug("Generate Cover Setting from Default(%s)"%setting_name)
                 
             if setting_name:
                 logger.debug("Running Generate Cover with settings %s."%setting_name)
                 realmi = db.get_metadata(book_id, index_is_id=True)
                 gc_plugin.generate_cover_for_book(realmi,saved_setting_name=setting_name)
 
+                if prefs['gc_polish_cover'] and \
+                        options['fileform'] == "epub" and calibre_version >= (0, 9, 39):
+                    # set cover inside epub from calibre's polish feature
+                    from calibre.ebooks.oeb.polish.main import polish, ALL_OPTS
+                    from calibre.utils.logging import Log
+                    from collections import namedtuple
+
+                    # Couldn't find a better way to get the cover path.
+                    cover_path = os.path.join(db.library_path, db.path(book_id, index_is_id=True), 'cover.jpg')
+                    data = {'cover':cover_path}
+                    #print("cover_path:%s"%cover_path)
+                    opts = ALL_OPTS.copy()
+                    opts.update(data)
+                    O = namedtuple('Options', ' '.join(ALL_OPTS.iterkeys()))
+                    opts = O(**opts)
+                
+                    log = Log(level=Log.DEBUG)
+                    outfile = db.format_abspath(book_id, formmapping[options['fileform']], index_is_id=True)
+                    #print("polish cover outfile:%s"%outfile)
+                    polish({outfile:outfile}, opts, log, logger.info)
+                
+
     def get_clean_reading_lists(self,lists):
         if lists == None or lists.strip() == "" :
             return []
@@ -1599,7 +1731,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
             rl_plugin = self.gui.iactions['Reading List']
         except:
             if prefs['addtolists'] or prefs['addtoreadlists']:
-                message="<p>You configured FanFictionDownLoader to automatically update Reading Lists, but you don't have the Reading List plugin installed anymore?</p>"
+                message="<p>"+_("You configured FanFictionDownLoader to automatically update Reading Lists, but you don't have the %s plugin installed anymore?")%'Reading List'+"</p>"
                 confirm(message,'fanfictiondownloader_no_reading_list_plugin', self.gui)
             return
         
@@ -1611,7 +1743,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                 
             lists = self.get_clean_reading_lists(prefs['read_lists'])
             if len(lists) < 1 :
-                message="<p>You configured FanFictionDownLoader to automatically update \"To Read\" Reading Lists, but you don't have any lists set?</p>"
+                message="<p>"+_("You configured FanFictionDownLoader to automatically update \"To Read\" Reading Lists, but you don't have any lists set?")+"</p>"
                 confirm(message,'fanfictiondownloader_no_read_lists', self.gui)
             for l in lists:
                 if l in rl_plugin.get_list_names():
@@ -1621,13 +1753,13 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                                   display_warnings=False)
                 else:
                     if l != '':
-                        message="<p>You configured FanFictionDownLoader to automatically update Reading List '%s', but you don't have a list of that name?</p>"%l
+                        message="<p>"+_("You configured FanFictionDownLoader to automatically update Reading List '%s', but you don't have a list of that name?")%l+"</p>"
                         confirm(message,'fanfictiondownloader_no_reading_list_%s'%l, self.gui)
                         
         if prefs['addtolists'] and (add or (prefs['addtolistsonread'] and prefs['addtoreadlists']) ):
             lists = self.get_clean_reading_lists(prefs['send_lists'])
             if len(lists) < 1 :
-                message="<p>You configured FanFictionDownLoader to automatically update \"Send to Device\" Reading Lists, but you don't have any lists set?</p>"
+                message="<p>"+_("You configured FanFictionDownLoader to automatically update \"Send to Device\" Reading Lists, but you don't have any lists set?")+"</p>"
                 confirm(message,'fanfictiondownloader_no_send_lists', self.gui)
 
             for l in lists:
@@ -1639,7 +1771,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                                                 display_warnings=False)
                 else:
                     if l != '':
-                        message="<p>You configured FanFictionDownLoader to automatically update Reading List '%s', but you don't have a list of that name?</p>"%l
+                        message="<p>"+_("You configured FanFictionDownLoader to automatically update Reading List '%s', but you don't have a list of that name?")%l+"</p>"
                         confirm(message,'fanfictiondownloader_no_reading_list_%s'%l, self.gui)
 
     def make_mi_from_book(self,book):
@@ -1676,6 +1808,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         book['end'] = None
         book['comment'] = '' # note this is a comment on the d/l or update.
         book['url'] = ''
+        book['site'] = ''
         book['added'] = False
         return book
     
@@ -1746,19 +1879,21 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
 
     def set_book_url_and_comment(self,book,url):
         if not url:
-            book['comment'] = "No story URL found."
+            book['comment'] = _("No story URL found.")
             book['good'] = False
             book['icon'] = 'search_delete_saved.png'
-            book['status'] = 'Not Found'
+            book['status'] = _('Not Found')
         else:
             # get normalized url or None.
-            book['url'] = self.is_good_downloader_url(url)
-            if book['url'] == None:
+            urlsitetuple = adapters.getNormalStoryURLSite(url)
+            if urlsitetuple == None:
                 book['url'] = url
-                book['comment'] = "URL is not a valid story URL."
+                book['comment'] = _("URL is not a valid story URL.")
                 book['good'] = False
                 book['icon']='dialog_error.png'
-                book['status'] = 'Bad URL'
+                book['status'] = _('Bad URL')
+            else:
+                (book['url'],book['site'])=urlsitetuple
     
     def get_story_url(self, db, book_id=None, path=None):
         if book_id == None:
@@ -1888,8 +2023,8 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
             book['comments'] = existingbook['comments']
         else:
             book['title'] = deftitle = book_list[0]['title']
-            book['comments'] = "Anthology containing:\n" + \
-                "\n".join([ "%s by %s"%(b['title'],', '.join(b['author'])) for b in book_list ])
+            book['comments'] = _("Anthology containing:")+"\n" + \
+                "\n".join([ _("%s by %s")%(b['title'],', '.join(b['author'])) for b in book_list ])
             # book['all_metadata']['description']
         
             # if all same series, use series for name.  But only if all and not previous named
@@ -1909,7 +2044,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
             else:
                 # No setting, do fall back default.  Shouldn't happen,
                 # should always have a version in defaults.
-                book['title'] = book['title']+" Anthology"
+                book['title'] = book['title']+_(" Anthology")
         
         book['all_metadata']['title'] = book['title'] # because custom columns are set from all_metadata
         book['all_metadata']['author'] = ", ".join(book['author'])
@@ -1946,7 +2081,7 @@ def pretty_book(d, indent=0, spacer='     '):
     if isinstance(d, dict):
         for k in ('password','username'):
             if k in d and d[k]:
-                d[k]='<was set, removed for security>'
+                d[k]=_('(was set, removed for security)')
         return '\n'.join(['%s%s:\n%s' % (kindent, k, pretty_book(v, indent + 1, spacer)) 
                           for k, v in d.items()])
     return "%s%s"%(kindent, d)

calibre-plugin/jobs.py

@@ -93,16 +93,13 @@ def do_download_worker(book_list, options,
         logger.info('Logfile for book ID %s (%s)'%(book_id, job._book['title']))
         logger.info(job.details)
 
+
+
         if count >= total:
-            # All done!  Output some lists for convenience of some users.
-            logger.info("Successfully downloaded:")
-            for book in book_list:
-                if book['good']:
-                    logger.info("%s %s"%(book['title'],book['url']))
-            logger.info("\nUnsuccessful:")
-            for book in book_list:
-                if not book['good']:
-                    logger.info("%s %s"%(book['title'],book['url']))
+            logger.info("\nSuccessful:\n%s\n"%("\n".join([book['url'] for book in 
+                                                      filter(lambda x: x['good'], book_list) ] ) ) )
+            logger.info("\nUnsuccessful:\n%s\n"%("\n".join([book['url'] for book in 
+                                                        filter(lambda x: not x['good'], book_list) ] ) ) )
             break
 
     server.close()
@@ -182,7 +179,7 @@ def do_download_for_worker(book,options,notification=lambda x,y:x):
 
             # update now handled by pre-populating the old images and
             # chapters in the adapter rather than merging epubs.
-            urlchaptercount = int(story.getMetadata('numChapters'))
+            urlchaptercount = int(story.getMetadata('numChapters').replace(',',''))
             (url,
              chaptercount,
              adapter.oldchapters,

calibre-plugin/prefs.py

@@ -34,6 +34,7 @@ default_prefs['suppressauthorsort'] = False
 default_prefs['suppresstitlesort'] = False
 default_prefs['mark'] = False
 default_prefs['showmarked'] = False
+default_prefs['autoconvert'] = False
 default_prefs['urlsfromclip'] = True
 default_prefs['updatedefault'] = True
 default_prefs['fileform'] = 'epub'
@@ -56,6 +57,7 @@ default_prefs['addtolistsonread'] = False
 default_prefs['gcnewonly'] = False
 default_prefs['gc_site_settings'] = {}
 default_prefs['allow_gc_from_ini'] = True
+default_prefs['gc_polish_cover'] = False
 
 default_prefs['countpagesstats'] = []
 

defaults.ini

@@ -182,6 +182,11 @@ extratags: FanFiction
 ## useful if pulling large numbers of stories or if the site is slow.
 #slow_down_sleep_time:0.5
 
+## How long to wait for each HTTP connection to finish.  Longer times
+## are better for sites that are slow to respond.  Shorter times
+## prevent excessive wait when your network or the site is down.
+connect_timeout:60.0
+
 ## For use only with stand-alone CLI version--run a command on the
 ## generated file after it's produced.  All of the titlepage_entries
 ## values are available, plus output_filename.
@@ -220,6 +225,11 @@ extratags: FanFiction
 ## doesn't work on some devices either.)
 #replace_hr: false
 
+## Some sites/authors/stories use br tags instead of p tags for
+## paragraphs.  This feature uses some heuristics to find and replace
+## br paragraphs with p tags while preserving scene breaks.
+#replace_br_with_p: false
+
 ## If set false, the summary will have all html stripped.
 ## Both this and include_images must be true to get images in the
 ## summary.
@@ -468,7 +478,8 @@ remove_transparency: true
 ## confused and displays it on every page after that under the text
 ## for the rest of the chapter.  I doubt adding a div around the img
 ## will break any other readers, but in case it does, the fix can be
-## turned off.
+## turned off.  This setting is not used if replace_br_with_p is
+## true--replace_br_with_p also fixes the problem.
 nook_img_fix:true
 
 [mobi]
@@ -532,6 +543,13 @@ extratags: FanFiction,Testing,HTML
 ## personal.ini, not defaults.ini.
 #is_adult:true
 
+## archiveofourown.org stories allow chapters to be added out of
+## order.  So the newest chapter may not be the last one.  FFDL update
+## doesn't like that.  If do_update_hook is uncommented and set true,
+## the adapter will discard all existing chapters from the newest one
+## on when updating to enforce accurate chapters.
+#do_update_hook:false
+
 ## AO3 adapter defines a few extra metadata entries.
 ## If there's ever more than 4 series, add series04,series04Url etc.
 extra_valid_entries:fandoms,freeformtags,freefromtags,ao3categories,comments,kudos,hits,bookmarks,collections,series00,series01,series02,series03,series00Url,series01Url,series02Url,series03Url
@@ -555,12 +573,13 @@ include_in_freefromtags:freeformtags
 ## adds to include_subject_tags instead of replacing it.
 #extra_subject_tags:fandoms,freeformtags,ao3categories
 
-[ashwinder.sycophanthex.com]
-## Site dedicated to these categories/characters/ships
-extracategories:Harry Potter
-extracharacters:Severus Snape,Hermione Granger
-extraships:Severus Snape/Hermione Granger
+## AO3 chapters can include several different types of notes.  We've
+## traditional included them all in the chapter text, but this allows
+## you to customize which you include.  Copy this parameter to your
+## personal.ini and list the ones you don't want.
+#exclude_notes:authorheadnotes,chaptersummary,chapterheadnotes,chapterfootnotes,authorfootnotes
 
+[ashwinder.sycophanthex.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -568,6 +587,11 @@ extraships:Severus Snape/Hermione Granger
 #username:YourName
 #password:yourpassword
 
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+extracharacters:Severus Snape,Hermione Granger
+extraships:Severus Snape/Hermione Granger
+
 [asr3.slashzone.org]
 ## Site dedicated to these categories/characters/ships
 extracategories:The Sentinel
@@ -578,9 +602,6 @@ extracategories:The Sentinel
 #is_adult:true
 
 [bloodties-fans.com]
-## Site dedicated to these categories/characters/ships
-extracategories:Blood Ties
-
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -593,6 +614,9 @@ extracategories:Blood Ties
 ## personal.ini, not defaults.ini.
 #is_adult:true
 
+## Site dedicated to these categories/characters/ships
+extracategories:Blood Ties
+
 [buffynfaith.net]
 ## Site dedicated to these categories/characters/ships
 extracategories:Buffy: The Vampire Slayer
@@ -603,9 +627,6 @@ extracategories:Buffy: The Vampire Slayer
 #is_adult:true
 
 [castlefans.org]
-## Site dedicated to these categories/characters/ships
-extracategories:Castle
-
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -618,6 +639,9 @@ extracategories:Castle
 ## personal.ini, not defaults.ini.
 #is_adult:true
 
+## Site dedicated to these categories/characters/ships
+extracategories:Castle
+
 [chaos.sycophanthex.com]
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
@@ -630,10 +654,23 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 
 [dark-solace.org]
 ## Site dedicated to these categories/characters/ships
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
 extracategories:Buffy: The Vampire Slayer
 extracharacters:Buffy, Spike
 extraships:Spike/Buffy
 
+[dramione.org]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -646,24 +683,11 @@ extraships:Spike/Buffy
 ## personal.ini, not defaults.ini.
 #is_adult:true
 
-[dramione.org]
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter
 extracharacters:Draco Malfoy,Hermione Granger
 extraships:Draco Malfoy/Hermione Granger
 
-## Some sites require login (or login for some rated stories) The
-## program can prompt you, or you can save it in config.  In
-## commandline version, this should go in your personal.ini, not
-## defaults.ini.
-#username:YourName
-#password:yourpassword
-
-## Some sites also require the user to confirm they are adult for
-## adult content.  In commandline version, this should go in your
-## personal.ini, not defaults.ini.
-#is_adult:true
-
 ## some sites include images that we don't ever want becoming the
 ## cover image.  This lets you exclude them.
 cover_exclusion_regexp:/images/.*?ribbon.gif
@@ -706,22 +730,19 @@ cliches_label:Character Cliches
 extracategories:Lord of the Rings
 
 [erosnsappho.sycophanthex.com]
-## Site dedicated to these categories/characters/ships
-extracategories:Harry Potter
-
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
 ## some sites include images that we don't ever want becoming the
 ## cover image.  This lets you exclude them.
 cover_exclusion_regexp:/images/.*?ribbon.gif
 
 [fanfiction.mugglenet.com]
-## Site dedicated to these categories/characters/ships
-extracategories:Harry Potter
-
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
@@ -734,10 +755,10 @@ extracategories:Harry Potter
 #username:YourName
 #password:yourpassword
 
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
 [fanfic.potterheadsanonymous.com]
-## Site dedicated to these categories/characters/ships
-extracategories:Harry Potter
-
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
@@ -750,16 +771,19 @@ extracategories:Harry Potter
 #username:YourName
 #password:yourpassword
 
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
 [fanfiction.portkey.org]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter
 extraships:Harry Potter/Hermione Granger
 
-## Some sites do not require a login, but do require the user to
-## confirm they are adult for adult content.  In commandline version,
-## this should go in your personal.ini, not defaults.ini.
-#is_adult:true
-
 [fanfiction.tenhawkpresents.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -793,11 +817,37 @@ dislikes_label:Dislikes
 #username:YourName
 #password:yourpassword
 
-[grangerenchanted.com]
-## Site dedicated to these categories/characters/ships
-extracategories:Harry Potter
-extracharacters:Hermione Granger
+[storiesonline.net]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
 
+## Clear FanFiction from defaults, site is original fiction.
+extratags:
+
+extra_valid_entries:size,universe,universeUrl,universeHTML,codes,notice
+#extra_titlepage_entries:size,universeHTML,codes,notice
+
+size_label:Size
+universe_label:Universe
+universeUrl_label:Universe URL
+universeHTML_label:Universe
+codes_label:Codes
+notice_label:Notice
+
+## Assume entryUrl, apply to "<a class='%slink' href='%s'>%s</a>" to
+## make entryHTML.
+make_linkhtml_entries:universe
+
+## storiesonline.net stories can be in a series or a universe, but not
+## both.  By default, universe will be populated in 'series' with
+## index=0
+universe_as_series: true
+
+[grangerenchanted.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -810,6 +860,10 @@ extracharacters:Hermione Granger
 ## personal.ini, not defaults.ini.
 #is_adult:true
 
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+extracharacters:Hermione Granger
+
 ## Extra metadata that this adapter knows about.  See [dramione.org]
 ## for examples of how to use them.
 extra_valid_entries:read,reviews
@@ -846,18 +900,15 @@ extracharacters:Kirk,Spock
 extraships:Kirk/Spock
 
 [lumos.sycophanthex.com]
-## Site dedicated to these categories/characters/ships
-extracategories:Harry Potter
-
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
-[merlinfic.dtwins.co.uk]
 ## Site dedicated to these categories/characters/ships
-extracategories:Merlin
+extracategories:Harry Potter
 
+[merlinfic.dtwins.co.uk]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -870,6 +921,9 @@ extracategories:Merlin
 ## personal.ini, not defaults.ini.
 #is_adult:true
 
+## Site dedicated to these categories/characters/ships
+extracategories:Merlin
+
 [national-library.net]
 ## Site dedicated to these categories/characters/ships
 extracategories:West Wing
@@ -881,24 +935,32 @@ extracategories:NCIS
 [netraptor.org]
 
 [nfacommunity.com]
-## Site dedicated to these categories/characters/ships
-extracategories:NCIS
-
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+## Site dedicated to these categories/characters/ships
+extracategories:NCIS
+
 [nha.magical-worlds.us]
 ## Site dedicated to these categories/characters/ships
 extracategories:Buffy: The Vampire Slayer
 extracharacters:Willow
 
 [occlumency.sycophanthex.com]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter
 extracharacters:Severus Snape
 
+[onedirectionfanfiction.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -906,10 +968,15 @@ extracharacters:Severus Snape
 #username:YourName
 #password:yourpassword
 
-[onedirectionfanfiction.com]
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
 ## Site dedicated to these categories/characters/ships
 extracategories:One Direction
 
+[pommedesang.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -922,40 +989,27 @@ extracategories:One Direction
 ## personal.ini, not defaults.ini.
 #is_adult:true
 
-[pommedesang.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:Anita Blake Vampire Hunter 
 
-## Some sites require login (or login for some rated stories) The
-## program can prompt you, or you can save it in config.  In
-## commandline version, this should go in your personal.ini, not
-## defaults.ini.
-#username:YourName
-#password:yourpassword
-
-## Some sites also require the user to confirm they are adult for
-## adult content.  In commandline version, this should go in your
-## personal.ini, not defaults.ini.
+[ponyfictionarchive.net]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
-[ponyfictionarchive.net]
 ## Site dedicated to these categories/characters/ships
 extracategories:My Little Pony: Friendship is Magic
 
+[pretendercentre.com]
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
-[pretendercentre.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:The Pretender
 
-## Some sites do not require a login, but do require the user to
-## confirm they are adult for adult content.  In commandline version,
-## this should go in your personal.ini, not defaults.ini.
-#is_adult:true
-
 [samdean.archive.nu]
 ## Site dedicated to these categories/characters/ships
 extracategories:Supernatural
@@ -963,9 +1017,6 @@ extracharacters:Sam,Dean
 extraships:Sam/Dean
 
 [scarhead.net]
-## Site dedicated to these categories/characters/ships
-extracategories:Harry Potter
-
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -978,6 +1029,9 @@ extracategories:Harry Potter
 ## personal.ini, not defaults.ini.
 #is_adult:true
 
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
 [sg1-heliopolis.com]
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
@@ -1008,9 +1062,6 @@ extracharacters:Draco Malfoy,Harry Potter
 extraships:Harry Potter/Draco Malfoy
 
 [thehookupzone.net]
-## Site dedicated to these categories/characters/ships
-extracategories:Criminal Minds
-
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -1023,6 +1074,9 @@ extracategories:Criminal Minds
 ## personal.ini, not defaults.ini.
 #is_adult:true
 
+## Site dedicated to these categories/characters/ships
+extracategories:Criminal Minds
+
 [themasque.net]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -1037,9 +1091,6 @@ extracategories:Criminal Minds
 #is_adult:true
 
 [thequidditchpitch.org]
-## Site dedicated to these categories/characters/ships
-extracategories:Harry Potter
-
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -1052,50 +1103,71 @@ extracategories:Harry Potter
 ## personal.ini, not defaults.ini.
 #is_adult:true
 
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
 [tokra.fandomnet.com]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
 ## Site dedicated to these categories/characters/ships
 extracategories:Stargate: SG-1
 
-## Some sites do not require a login, but do require the user to
-## confirm they are adult for adult content.  In commandline version,
-## this should go in your personal.ini, not defaults.ini.
+[trekiverse.org]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
 #is_adult:true
 
-[www.adastrafanfic.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:Star Trek
 
+extra_valid_entries:awards
+awards_label:Awards
+
+cover_exclusion_regexp:art/.*Awards.jpg
+
+[www.adastrafanfic.com]
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+## Site dedicated to these categories/characters/ships
+extracategories:Star Trek
+
 [www.dracoandginny.com]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter
 extracharacters:Draco Malfoy,Ginny Weasley
 extraships:Draco Malfoy/Ginny Weasley
 
-## Some sites require login (or login for some rated stories) The
-## program can prompt you, or you can save it in config.  In
-## commandline version, this should go in your personal.ini, not
-## defaults.ini.
-#username:YourName
-#password:yourpassword
-
-## Some sites also require the user to confirm they are adult for
-## adult content.  In commandline version, this should go in your
-## personal.ini, not defaults.ini.
-#is_adult:true
-
 [www.thealphagate.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:Stargate: SG-1
 
 [www.checkmated.com]
-## Site dedicated to these categories/characters/ships
-extracategories:Harry Potter
-
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -1103,6 +1175,9 @@ extracategories:Harry Potter
 #username:YourName
 #password:yourpassword
 
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
 [www.destinysgateway.com]
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
@@ -1110,11 +1185,6 @@ extracategories:Harry Potter
 #is_adult:true
 
 [www.dokuga.com]
-## Site dedicated to these categories/characters/ships
-extracategories:InuYasha
-extracharacters:Sesshoumaru,Kagome
-extraships:Sesshoumaru/Kagome
-
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -1122,6 +1192,11 @@ extraships:Sesshoumaru/Kagome
 #username:YourName
 #password:yourpassword
 
+## Site dedicated to these categories/characters/ships
+extracategories:InuYasha
+extracharacters:Sesshoumaru,Kagome
+extraships:Sesshoumaru/Kagome
+
 [www.dotmoon.net]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -1151,10 +1226,33 @@ user_agent:
 ## Change this to false to use them anyway.
 never_make_cover: true
 
+## fanfiction.net is blocking people more aggressively.  If you
+## download fewer stories less often you can likely get by with
+## reducing this sleep.
+slow_down_sleep_time:4
+
 ## Extra metadata that this adapter knows about.  See [dramione.org]
 ## for examples of how to use them.
 extra_valid_entries:reviews,favs,follows
 
+## ffnet uses 'Pairings', not 'Relationship', stating they don't have
+## to be romantic pairings.
+ships_label:Pairings
+
+## Date formats used by FFDL.  Published and Update don't usually have
+## time, but they do now on ffnet.
+## See http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+## Note that ini format requires % to be escaped as %%.
+#dateCreated_format:%%Y-%%m-%%d %%H:%%M:%%S
+datePublished_format:%%Y-%%m-%%d %%H:%%M:%%S
+dateUpdated_format:%%Y-%%m-%%d %%H:%%M:%%S
+
+## ffnet used to have a tendency to send out update notices in email
+## before all their servers were showing the update on the first
+## chapter.  It generates another server request and doesn't seem to
+## be needed lately, so now default it to off.
+check_next_chapter:false
+
 [www.fanfiktion.de]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -1166,14 +1264,14 @@ extra_valid_entries:reviews,favs,follows
 [www.ficbook.net]
 
 [www.fictionalley.org]
-## Site dedicated to these categories/characters/ships
-extracategories:Harry Potter
-
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
 ## fictionally.org storyIds are not unique.  Combine with authorId.
 output_filename: ${title}-${siteabbrev}_${authorId}_${storyId}${formatext}
 
@@ -1199,19 +1297,6 @@ extra_valid_entries:reviews,favs,follows
 #password:yourpassword
 
 [www.fimfiction.net]
-## Site dedicated to these categories/characters/ships
-extracategories:My Little Pony: Friendship is Magic
-
-## Extra metadata that this adapter knows about.  See [dramione.org]
-## for examples of how to use them.
-extra_valid_entries:likes,dislikes,views,total_views,short_description,groups
-likes_label:Likes
-dislikes_label:Dislikes
-views_label:Highest Single Chapter Views
-total_views_label:Total Views
-short_description_label:Short Summary
-groups_label:Groups
-
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
@@ -1229,38 +1314,63 @@ groups_label:Groups
 ## when updating to enforce accurate chapters.
 #do_update_hook:false
 
-[www.harrypotterfanfiction.com]
 ## Site dedicated to these categories/characters/ships
-extracategories:Harry Potter
+extracategories:My Little Pony: Friendship is Magic
 
+## Extra metadata that this adapter knows about.  See [dramione.org]
+## for examples of how to use them.
+extra_valid_entries:likes,dislikes,views,total_views,short_description,groups,groupsUrl,groupsHTML,prequel,prequelUrl,prequelHTML,sequels,sequelsUrl,sequelsHTML
+likes_label:Likes
+dislikes_label:Dislikes
+views_label:Highest Single Chapter Views
+total_views_label:Total Views
+short_description_label:Short Summary
+groups_label:Groups
+groupsUrl_label:Groups URLs
+groupsHTML_label:Groups
+prequel_label:Prequel
+prequelUrl_label:Prequel URL
+prequelHTML_label:Prequel
+sequels_label:Sequels
+sequelsUrl_label:Sequel URLs
+sequelsHTML_label:Sequels
+
+keep_in_order_sequels:true
+keep_in_order_sequelsUrl:true
+keep_in_order_groups:true
+keep_in_order_groupsUrl:true
+
+## Assume entryUrl, apply to "<a class='%slink' href='%s'>%s</a>" to
+## make entryHTML.
+make_linkhtml_entries:prequel,sequels,groups
+
+[www.harrypotterfanfiction.com]
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
 [www.henneth-annun.net]
 ## Site dedicated to these categories/characters/ships
 extracategories:The Hobbit
 
 [www.hpfandom.net]
-## Site dedicated to these categories/characters/ships
-extracategories:Harry Potter
-
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
 [www.hpfanficarchive.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter
 
 [www.ik-eternal.net]
-## Site dedicated to these categories/characters/ships
-extracategories:InuYasha
-extracharacters:InuYasha,Kagome
-extraships:InuYasha/Kagome
-
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -1273,30 +1383,35 @@ extraships:InuYasha/Kagome
 ## personal.ini, not defaults.ini.
 #is_adult:true
 
+## Site dedicated to these categories/characters/ships
+extracategories:InuYasha
+extracharacters:InuYasha,Kagome
+extraships:InuYasha/Kagome
+
 [www.jlaunlimited.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:JLA
 
 [www.libraryofmoria.com]
-## Site dedicated to these categories/characters/ships
-extracategories:Lord of the Rings
-
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+## Site dedicated to these categories/characters/ships
+extracategories:Lord of the Rings
+
 [www.mediaminer.org]
 
 [www.midnightwhispers.ca]
-## Site dedicated to these categories/characters/ships
-extracategories:Queer as Folk
-
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+## Site dedicated to these categories/characters/ships
+extracategories:Queer as Folk
+
 ## some sites include images that we don't ever want becoming the
 ## cover image.  This lets you exclude them.
 cover_exclusion_regexp:/stories/999/images/.*?_trophy.png
@@ -1311,10 +1426,6 @@ extracategories:CSI
 extraships:Nick Stokes/Greg Sanders
 
 [www.phoenixsong.net]
-## Site dedicated to these categories/characters/ships
-extracategories:Harry Potter
-extraships:Harry Potter/Ginny Weasley
-
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -1325,9 +1436,13 @@ extraships:Harry Potter/Ginny Weasley
 ## phoenixsong.net, oddly, can have high rated chapters (login
 ## required) in the middle of a lower rated story.  Use this to force
 ## FFDL to always login to phoenixsong.net so those stories download
-## correctly.
+## correctly.  If you have a login, this is recommended.
 #force_login:true
 
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+extraships:Harry Potter/Ginny Weasley
+
 [www.potionsandsnitches.net]
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter
@@ -1350,17 +1465,15 @@ extracategories:Psych
 #is_adult:true
 
 [www.qaf-fic.com]
-## Site dedicated to these categories/characters/ships
-extracategories:Queer as Folk
-
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+## Site dedicated to these categories/characters/ships
+extracategories:Queer as Folk
+
 [www.restrictedsection.org]
-extracategories:Harry Potter
-extragenres:Erotica
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -1368,20 +1481,21 @@ extragenres:Erotica
 #username:YourName
 #password:yourpassword
 
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+extragenres:Erotica
+
 [www.scarvesandcoffee.net]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
 ## Site dedicated to these categories/characters/ships
 extracategories:Glee
 extracharacters:Kurt Hummel,Blaine Anderson
 
-## Some sites do not require a login, but do require the user to
-## confirm they are adult for adult content.  In commandline version,
-## this should go in your personal.ini, not defaults.ini.
-#is_adult:true
-
 [www.simplyundeniable.com]
-## Site dedicated to these categories/characters/ships
-extracategories:Harry Potter
-
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -1389,15 +1503,18 @@ extracategories:Harry Potter
 #username:YourName
 #password:yourpassword
 
-[www.sinful-desire.org]
 ## Site dedicated to these categories/characters/ships
-extracategories:Supernatural
+extracategories:Harry Potter
 
+[www.sinful-desire.org]
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+## Site dedicated to these categories/characters/ships
+extracategories:Supernatural
+
 [www.siye.co.uk]
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter
@@ -1405,25 +1522,26 @@ extracharacters:Harry Potter,Ginny Weasley
 extraships:Harry Potter/Ginny Weasley
 
 [www.squidge.org/peja]
-# www.squidge.org/peja calls it Fandom <shrug>
+## www.squidge.org/peja calls it Fandom <shrug>
 category_label:Fandom
-# Remove numWords -- www.squidge.org/peja word counts are inaccurate
+
+## Remove numWords -- www.squidge.org/peja word counts are inaccurate
 titlepage_entries: seriesHTML,category,genre,language,characters,ships,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,site,description
 
 [www.squidge.org/peja:txt]
-## Add URLs since there aren't links.
-# Remove numWords -- www.squidge.org/peja word counts are inaccurate
+## Add URLs since there aren't links and remove numWords --
+## www.squidge.org/peja word counts are inaccurate
 titlepage_entries: series,seriesUrl,category,genre,language,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,site,storyUrl, authorUrl, description
 
 [www.storiesofarda.com]
-## Site dedicated to these categories/characters/ships
-extracategories:Lord of the Rings
-
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+## Site dedicated to these categories/characters/ships
+extracategories:Lord of the Rings
+
 [www.thepetulantpoetess.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -1466,9 +1584,6 @@ collect_series: false
 extracategories:Twilight
 
 [www.twilighted.net]
-## Site dedicated to these categories/characters/ships
-extracategories:Twilight
-
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -1476,13 +1591,13 @@ extracategories:Twilight
 #username:YourName
 #password:yourpassword
 
+## Site dedicated to these categories/characters/ships
+extracategories:Twilight
+
 ## twilighted.net (ab)uses series as personal reading lists.
 collect_series: false
 
 [www.twiwrite.net]
-## Site dedicated to these categories/characters/ships
-extracategories:Twilight
-
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -1490,20 +1605,23 @@ extracategories:Twilight
 #username:YourName
 #password:yourpassword
 
+## Site dedicated to these categories/characters/ships
+extracategories:Twilight
+
 ## twiwrite.net (ab)uses series as personal reading lists.
 collect_series: false
 
 [www.walkingtheplank.org]
-## Site dedicated to these categories/characters/ships
-extracategories:Harry Potter
-extracharacters:Severus Snape,Harry Potter
-extraships:Severus Snape/Harry Potter
-
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+extracharacters:Severus Snape,Harry Potter
+extraships:Severus Snape/Harry Potter
+
 [www.whofic.com]
 
 [www.wizardtales.net]
@@ -1525,14 +1643,14 @@ extracategories:X-Men Movie
 extracharacters:Wolverine,Rogue
 
 [www.wraithbait.com]
-## Site dedicated to these categories/characters/ships
-extracategories:Stargate: Atlantis
-
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+## Site dedicated to these categories/characters/ships
+extracategories:Stargate: Atlantis
+
 [overrides]
 ## It may sometimes be useful to override all of the specific format,
 ## site and site:format sections in your private configuration.  For

fanficdownloader/HtmlTagStack.py

@@ -0,0 +1,57 @@
+# coding: utf-8
+
+import re
+import codecs
+
+stack = []
+
+def get_end_tag(tag):
+    if len(tag) > 0 and tag.find(u'<') > -1 and tag.rfind(u'>') > -1:
+        return re.sub(r'.*<([^\ >]+).*', r'</\1>', tag)
+    return u''
+
+def get_tag_name(tag):
+    if len(tag) > 0 and tag.find(u'<') > -1 and tag.rfind(u'>') > -1:
+        return re.sub(r'</*([^\ >]+).*', r'\1', tag)
+    return u''
+
+def push(tag):
+    if len(tag) > 0 and tag.find(u'<') > -1 and tag.rfind(u'>') > -1:
+        stack.append(tag)
+
+def pop():
+    if len(stack) > 0:
+        return stack.pop()
+    return u''
+
+def pop_end_tag():
+    return unicode(get_end_tag(pop()))
+
+def spool_end():
+    html = u''
+    for tag in reversed(stack):
+        html += get_end_tag(tag)
+    return html
+
+def spool_start():
+    html = u''
+    for item in stack:
+        html += item
+    return html
+
+def has_elements():
+    return len(stack) > 0
+
+def get_last():
+    # t = pop()
+    # push(t)
+    # return t
+    if len(stack) > 0:
+        return stack[len(stack)-1]
+    return u''
+
+def flush():
+    del stack[:]
+
+def get_stack():
+    return stack

fanficdownloader/adapters/__init__.py

@@ -120,6 +120,9 @@ import adapter_potterheadsanonymouscom
 import adapter_simplyundeniablecom
 import adapter_scarheadnet
 import adapter_fictionpadcom
+import adapter_storiesonlinenet
+import adapter_trekiverseorg
+import adapter_literotica
 
 ## This bit of complexity allows adapters to be added by just adding
 ## importing.  It eliminates the long if/else clauses we used to need
@@ -143,6 +146,13 @@ for x in imports():
             __domain_map[site]=cls
 
 def getNormalStoryURL(url):
+    r = getNormalStoryURLSite(url)
+    if r:
+        return r[0]
+    else:
+        return None
+
+def getNormalStoryURLSite(url):
     if not getNormalStoryURL.__dummyconfig:
         getNormalStoryURL.__dummyconfig = Configuration("test1.com","EPUB")
     # pulling up an adapter is pretty low over-head.  If
@@ -150,10 +160,11 @@ def getNormalStoryURL(url):
     try:
         adapter = getAdapter(getNormalStoryURL.__dummyconfig,url)
         url = adapter.url
+        site = adapter.getSiteDomain()
         del adapter
-        return url
+        return (url,site)
     except:
-        return None;
+        return None
 
 # kludgey function static/singleton
 getNormalStoryURL.__dummyconfig = None
@@ -190,7 +201,7 @@ def getConfigSectionFor(url):
 
 def getClassFor(url):
     ## fix up leading protocol.
-    fixedurl = re.sub(r"(?i)^[htps]+[:/]+","http://",url.strip())
+    fixedurl = re.sub(r"(?i)^[htp]+(s?)[:/]+",r"http\1://",url.strip())
     if not fixedurl.startswith("http"):
         fixedurl = "http://%s"%url
     ## remove any trailing '#' locations.

fanficdownloader/adapters/adapter_archiveofourownorg.py

@@ -77,14 +77,14 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'archiveofourown.org'
 
-    @classmethod
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/works/123456 http://"+self.getSiteDomain()+"/collections/Some_Archive/works/123456 http://"+self.getSiteDomain()+"/works/123456/chapters/78901"
 
     def getSiteURLPattern(self):
         # http://archiveofourown.org/collections/Smallville_Slash_Archive/works/159770
         # Discard leading zeros from story ID numbers--AO3 doesn't use them in it's own chapter URLs.
-        return re.escape("http://")+re.escape(self.getSiteDomain())+r"(/collections/[^/]+)?/works/0*(?P<id>\d+)"
+        return r"https?://"+re.escape(self.getSiteDomain())+r"(/collections/[^/]+)?/works/0*(?P<id>\d+)"
         
     ## Login
     def needToLoginCheck(self, data):
@@ -177,19 +177,28 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
                 self.story.addToList('authorUrl','http://'+self.host+a['href'])
                 self.story.addToList('author',a.text)
 
+        newestChapter = None
+        self.newestChapterNum = None # save for comparing during update.
+        # Scan all chapters to find the oldest and newest, on AO3 it's
+        # possible for authors to insert new chapters out-of-order or
+        # change the dates of earlier ones by editing them--That WILL
+        # break epub update.
         # Find the chapters:
         chapters=soup.findAll('a', href=re.compile(r'/works/'+self.story.getMetadata('storyId')+"/chapters/\d+$"))
         self.story.setMetadata('numChapters',len(chapters))
         logger.debug("numChapters: (%s)"%self.story.getMetadata('numChapters'))
-        for x in range(0,len(chapters)):
-            # just in case there's tags, like <i> in chapter titles.
-            chapter=chapters[x]
-            if len(chapters)==1:
-                self.chapterUrls.append((self.story.getMetadata('title'),'http://'+self.host+chapter['href']+addurl))
-            else:
+        if len(chapters)==1:
+            self.chapterUrls.append((self.story.getMetadata('title'),'http://'+self.host+chapters[0]['href']+addurl))
+        else:
+            for index, chapter in enumerate(chapters):
+                # strip just in case there's tags, like <i> in chapter titles.
                 self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+chapter['href']+addurl))
-
-
+                # (2013-09-21)
+                date = stripHTML(chapter.findNext('span'))[1:-1]
+                chapterDate = makeDate(date,self.dateformat)
+                if newestChapter == None or chapterDate > newestChapter:
+                    newestChapter = chapterDate
+                    self.newestChapterNum = index
 
         a = metasoup.find('blockquote',{'class':'userstuff'})
         if a != None:
@@ -299,32 +308,44 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
                     self.setSeries(series_name, series_index)
                     self.story.setMetadata('seriesUrl',series_url)
 
+    def hookForUpdates(self,chaptercount):
+        if self.oldchapters and len(self.oldchapters) > self.newestChapterNum:
+            print("Existing epub has %s chapters\nNewest chapter is %s.  Discarding old chapters from there on."%(len(self.oldchapters), self.newestChapterNum+1))
+            self.oldchapters = self.oldchapters[:self.newestChapterNum]
+        return len(self.oldchapters)
+            
     # grab the text for an individual chapter.
     def getChapterText(self, url):
         logger.debug('Getting chapter text from: %s' % url)
 		
-        chapter=bs.BeautifulSoup('<div class="story"></div>')
+        chapter=bs.BeautifulSoup('<div class="story"></div>').find('div')
         data = self._fetchUrl(url)
         soup = bs.BeautifulSoup(data,selfClosingTags=('br','hr'))
-		
-        headnotes = soup.find('div', {'class' : "preface group"}).find('div', {'class' : "notes module"})
-        if headnotes != None:
-            headnotes = headnotes.find('blockquote', {'class' : "userstuff"})
+
+        exclude_notes=self.getConfigList('exclude_notes')
+
+        if 'authorheadnotes' not in exclude_notes:
+            headnotes = soup.find('div', {'class' : "preface group"}).find('div', {'class' : "notes module"})
             if headnotes != None:
-                chapter.append("<b>Author's Note:</b>")
-                chapter.append(headnotes)
+                headnotes = headnotes.find('blockquote', {'class' : "userstuff"})
+                if headnotes != None:
+                    chapter.append("<b>Author's Note:</b>")
+                    chapter.append(headnotes)
         
-        chapsumm = soup.find('div', {'id' : "summary"})
-        if chapsumm != None:
-            chapsumm = chapsumm.find('blockquote')
-            chapter.append("<b>Summary for the Chapter:</b>")
-            chapter.append(chapsumm)
-        chapnotes = soup.find('div', {'id' : "notes"})
-        if chapnotes != None:
-            chapnotes = chapnotes.find('blockquote')
+        if 'chaptersummary' not in exclude_notes:
+            chapsumm = soup.find('div', {'id' : "summary"})
+            if chapsumm != None:
+                chapsumm = chapsumm.find('blockquote')
+                chapter.append("<b>Summary for the Chapter:</b>")
+                chapter.append(chapsumm)
+                
+        if 'chapterheadnotes' not in exclude_notes:
+            chapnotes = soup.find('div', {'id' : "notes"})
             if chapnotes != None:
-                chapter.append("<b>Notes for the Chapter:</b>")
-                chapter.append(chapnotes)
+                chapnotes = chapnotes.find('blockquote')
+                if chapnotes != None:
+                    chapter.append("<b>Notes for the Chapter:</b>")
+                    chapter.append(chapnotes)
 		
         text = soup.find('div', {'class' : "userstuff module"})
         chtext = text.find('h3', {'class' : "landmark heading"})
@@ -332,17 +353,19 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
             chtext.extract()
         chapter.append(text)
 		
-        chapfoot = soup.find('div', {'class' : "end notes module", 'role' : "complementary"})
-        if chapfoot != None:
-            chapfoot = chapfoot.find('blockquote')
-            chapter.append("<b>Notes for the Chapter:</b>")
-            chapter.append(chapfoot)
+        if 'chapterfootnotes' not in exclude_notes:
+            chapfoot = soup.find('div', {'class' : "end notes module", 'role' : "complementary"})
+            if chapfoot != None:
+                chapfoot = chapfoot.find('blockquote')
+                chapter.append("<b>Notes for the Chapter:</b>")
+                chapter.append(chapfoot)
 		
-        footnotes = soup.find('div', {'id' : "work_endnotes"})
-        if footnotes != None:
-            footnotes = footnotes.find('blockquote')
-            chapter.append("<b>Author's Note:</b>")
-            chapter.append(footnotes)
+        if 'authorfootnotes' not in exclude_notes:
+            footnotes = soup.find('div', {'id' : "work_endnotes"})
+            if footnotes != None:
+                footnotes = footnotes.find('blockquote')
+                chapter.append("<b>Author's Note:</b>")
+                chapter.append(footnotes)
 			
         if None == soup:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)

fanficdownloader/adapters/adapter_buffynfaithnet.py

@@ -87,7 +87,7 @@ class BuffyNFaithNetAdapter(BaseSiteAdapter):
     
     def setHeader(self):
         "buffynfaith.net wants a Referer for images.  Used both above and below(after cookieproc added)"
-        self.opener.addheaders = [('Referer', 'http://'+self.getSiteDomain()+'/')]
+        self.opener.addheaders.append(('Referer', 'http://'+self.getSiteDomain()+'/'))
         
     @classmethod
     def getSiteExampleURLs(self):

fanficdownloader/adapters/adapter_darksolaceorg.py

@@ -94,11 +94,10 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
         else:
             params['penname'] = self.getConfig("username")
             params['password'] = self.getConfig("password")
-        params['rememberme'] = '1'
         params['action'] = 'login'
         params['submit'] = 'Submit'
     
-        loginUrl = 'http://' + self.getSiteDomain() + '/elysian/user.php'
+        loginUrl = 'http://www.' + self.getSiteDomain() + '/elysian/user.php'
         logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['penname']))
     

fanficdownloader/adapters/adapter_fanfictionnet.py

@@ -16,6 +16,7 @@
 #
 
 import time
+from datetime import datetime
 import logging
 logger = logging.getLogger(__name__)
 import re
@@ -43,7 +44,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
 
         # normalized story URL.
-        self._setURL("http://"+self.getSiteDomain()\
+        self._setURL("https://"+self.getSiteDomain()\
                          +"/s/"+self.story.getMetadata('storyId')+"/1/")
 
         # ffnet update emails have the latest chapter URL.
@@ -52,9 +53,11 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         # chapter list doesn't get the latest.  So save and use the
         # original URL given to pull chapter list & metadata.
         self.origurl = url
-        if "http://m." in self.origurl:
+        if "https://m." in self.origurl:
             ## accept m(mobile)url, but use www.
-            self.origurl = self.origurl.replace("http://m.","http://www.")
+            self.origurl = self.origurl.replace("https://m.","https://www.")
+
+        self.opener.addheaders.append(('Referer',self.origurl))
 
     @staticmethod
     def getSiteDomain():
@@ -66,10 +69,17 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
 
     @classmethod
     def getSiteExampleURLs(self):
-        return "http://www.fanfiction.net/s/1234/1/ http://www.fanfiction.net/s/1234/12/ http://www.fanfiction.net/s/1234/1/Story_Title http://m.fanfiction.net/s/1234/1/"
+        return "https://www.fanfiction.net/s/1234/1/ https://www.fanfiction.net/s/1234/12/ http://www.fanfiction.net/s/1234/1/Story_Title http://m.fanfiction.net/s/1234/1/"
 
     def getSiteURLPattern(self):
-        return r"http://(www|m)?\.fanfiction\.net/s/\d+(/\d+)?(/|/[^/]+)?/?$"
+        return r"https?://(www|m)?\.fanfiction\.net/s/\d+(/\d+)?(/|/[^/]+)?/?$"
+
+    def _fetchUrl(self,url):
+        time.sleep(1.0) ## ffnet(and, I assume, fpcom) tends to fail
+                        ## more if hit too fast.  This is in
+                        ## additional to what ever the
+                        ## slow_down_sleep_time setting is.
+        return BaseSiteAdapter._fetchUrl(self,url)
 
     def extractChapterUrlsAndMetadata(self):
 
@@ -97,32 +107,35 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         if "not found. Please check to see you are not using an outdated url." in data:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  'Chapter not found. Please check to see you are not using an outdated url.'" % url)
 
-        try:
-            # rather nasty way to check for a newer chapter.  ffnet has a
-            # tendency to send out update notices in email before all
-            # their servers are showing the update on the first chapter.
+        if self.getConfig('check_next_chapter'):
             try:
-                chapcount = len(soup.find('select', { 'name' : 'chapter' } ).findAll('option'))
-            # get chapter part of url.
+                ## ffnet used to have a tendency to send out update
+                ## notices in email before all their servers were
+                ## showing the update on the first chapter.  It
+                ## generates another server request and doesn't seem
+                ## to be needed lately, so now default it to off.
+                try:
+                    chapcount = len(soup.find('select', { 'name' : 'chapter' } ).findAll('option'))
+                # get chapter part of url.
+                except:
+                    chapcount = 1
+                chapter = url.split('/',)[5]
+                tryurl = "https://%s/s/%s/%d/"%(self.getSiteDomain(),
+                                                self.story.getMetadata('storyId'),
+                                                chapcount+1)
+                logger.debug('=Trying newer chapter: %s' % tryurl)
+                newdata = self._fetchUrl(tryurl)
+                if "not found. Please check to see you are not using an outdated url." \
+                        not in newdata:
+                    logger.debug('=======Found newer chapter: %s' % tryurl)
+                    soup = bs.BeautifulSoup(newdata)
             except:
-                chapcount = 1
-            chapter = url.split('/',)[5]
-            tryurl = "http://%s/s/%s/%d/"%(self.getSiteDomain(),
-                                           self.story.getMetadata('storyId'),
-                                           chapcount+1)
-            logger.debug('=Trying newer chapter: %s' % tryurl)
-            newdata = self._fetchUrl(tryurl)
-            if "not found. Please check to see you are not using an outdated url." \
-                    not in newdata:
-                logger.debug('=======Found newer chapter: %s' % tryurl)
-                soup = bs.BeautifulSoup(newdata)
-        except:
-            pass
-        
+                pass
+            
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"^/u/\d+"))
         self.story.setMetadata('authorId',a['href'].split('/')[2])
-        self.story.setMetadata('authorUrl','http://'+self.host+a['href'])
+        self.story.setMetadata('authorUrl','https://'+self.host+a['href'])
         self.story.setMetadata('author',a.string)
 
         ## Pull some additional data from html.
@@ -141,9 +154,9 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
             # of Book, Movie, etc.
             self.story.addToList('category',stripHTML(categories[1]))
         elif 'Crossover' in categories[0]['href']:
-            caturl = "http://%s%s"%(self.getSiteDomain(),categories[0]['href'])
+            caturl = "https://%s%s"%(self.getSiteDomain(),categories[0]['href'])
             catsoup = bs.BeautifulSoup(self._fetchUrl(caturl))
-            for a in catsoup.findAll('a',href=re.compile(r"^/crossovers/")):
+            for a in catsoup.findAll('a',href=re.compile(r"^/crossovers/.+?/\d+/")):
                 self.story.addToList('category',stripHTML(a))
             else:
                 # Fall back.  I ran across a story with a Crossver
@@ -156,7 +169,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
                 
                 
             
-        a = soup.find('a', href='http://www.fictionratings.com/')
+        a = soup.find('a', href=re.compile(r'https?://www\.fictionratings\.com/'))
         rating = a.string
         if 'Fiction' in rating: # if rating has 'Fiction ', strip that out for consistency with past.
             rating = rating[8:]
@@ -165,7 +178,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
 
         # after Rating, the same bit of text containing id:123456 contains
         # Complete--if completed.
-        gui_table1i = soup.find('table',{'cellpadding':'5'})
+        gui_table1i = soup.find('div',{'id':'content_wrapper_inner'})
 
         self.story.setMetadata('title', stripHTML(gui_table1i.find('b'))) # title appears to be only(or at least first) bold tag in gui_table1i
 
@@ -205,9 +218,17 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
             self.story.extendList('genre',genrelist)
             metalist=metalist[1:]
 
+        # Updated: <span data-xutime='1368059198'>5/8</span> - Published: <span data-xutime='1278984264'>7/12/2010</span>
+        # Published: <span data-xutime='1384358726'>8m ago</span>
+        dates = soup.findAll('span',{'data-xutime':re.compile(r'^\d+$')})
+        if len(dates) > 1 :
+            # updated get set to the same as published upstream if not found.
+            self.story.setMetadata('dateUpdated',datetime.fromtimestamp(float(dates[0]['data-xutime'])))
+        self.story.setMetadata('datePublished',datetime.fromtimestamp(float(dates[-1]['data-xutime'])))
+            
         donechars = False
         while len(metalist) > 0:
-            if  metalist[0].startswith('Chapters') or metalist[0].startswith('Status') or metalist[0].startswith('id:'):
+            if  metalist[0].startswith('Chapters') or metalist[0].startswith('Status') or metalist[0].startswith('id:') or metalist[0].startswith('Updated:') or metalist[0].startswith('Published:'):
                 pass
             elif  metalist[0].startswith('Reviews'):
                 self.story.setMetadata('reviews',metalist[0].split(':')[1].strip())
@@ -215,21 +236,21 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
                 self.story.setMetadata('favs',metalist[0].split(':')[1].strip())
             elif  metalist[0].startswith('Follows:'):
                 self.story.setMetadata('follows',metalist[0].split(':')[1].strip())
-            elif  metalist[0].startswith('Updated'):
-                self.story.setMetadata('dateUpdated',makeDate(metalist[0].split(':')[1].strip(), '%m-%d-%y'))
-            elif  metalist[0].startswith('Published'):
-                self.story.setMetadata('datePublished',makeDate(metalist[0].split(':')[1].strip(), '%m-%d-%y'))
             elif  metalist[0].startswith('Words'):
                 self.story.setMetadata('numWords',metalist[0].split(':')[1].strip())
             elif not donechars:
-                self.story.extendList('characters',metalist[0].split('&'))
+                # with 'pairing' support, pairings are bracketed w/o comma after
+                # [Caspian X, Lucy Pevensie] Edmund Pevensie, Peter Pevensie
+                self.story.extendList('characters',metalist[0].replace('[','').replace(']',',').split(','))
+
+                l = metalist[0]
+                while '[' in l:
+                    self.story.addToList('ships',l[l.index('[')+1:l.index(']')].replace(', ','/'))
+                    l = l[l.index(']')+1:]
+
                 donechars = True
             metalist=metalist[1:]
 
-        # next might be characters, otherwise Reviews, Updated, Published, Words
-        # if not ( metalist[0].startswith('Reviews') or metalist[0].startswith('Updated') or metalist[0].startswith('Published') or metalist[0].startswith('Words') or metalist[0].startswith('Chapters') ):
-        #     self.story.extendList('characters',metalist[0].split('&'))
-        
         if 'Status: Complete' in metatext:
             self.story.setMetadata('status', 'Completed')
         else:
@@ -253,9 +274,9 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         else:
             allOptions = select.findAll('option')
             for o in allOptions:
-                url = u'http://%s/s/%s/%s/' % ( self.getSiteDomain(),
-                                            self.story.getMetadata('storyId'),
-                                            o['value'])
+                url = u'https://%s/s/%s/%s/' % ( self.getSiteDomain(),
+                                                 self.story.getMetadata('storyId'),
+                                                 o['value'])
                 # just in case there's tags, like <i> in chapter titles.
                 title = u"%s" % o
                 title = re.sub(r'<[^>]+>','',title)
@@ -266,7 +287,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         return
 
     def getChapterText(self, url):
-        time.sleep(5.0) ## ffnet(and, I assume, fpcom) tends to fail
+        time.sleep(4.0) ## ffnet(and, I assume, fpcom) tends to fail
                         ## more if hit too fast.  This is in
                         ## additional to what ever the
                         ## slow_down_sleep_time setting is.
@@ -281,7 +302,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         # don't care about anything before "<div class='storytextp"
         # (there's a space after storytextp, so no close quote(')) and
         # this kills any body tags.
-        divstr = "<div role='main' class='storytextp"
+        divstr = "<div role='main'"
         if divstr not in data:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
         else:

fanficdownloader/adapters/adapter_fictionpadcom.py

@@ -180,7 +180,11 @@ class FictionPadSiteAdapter(BaseSiteAdapter):
             
     def getChapterText(self, url):
         logger.debug('Getting chapter text from: %s' % url)
-        soup = bs.BeautifulSoup(self._fetchUrl(url))
+        if not url:
+            data = u"<em>This chapter has no text.</em>"
+        else:
+            data = self._fetchUrl(url)
+        soup = bs.BeautifulSoup(u"<div id='story'>"+data+u"</div>")
         return self.utf8FromSoup(url,soup)
     
 def getClass():

fanficdownloader/adapters/adapter_fictionpresscom.py

@@ -41,10 +41,10 @@ class FictionPressComSiteAdapter(FanFictionNetSiteAdapter):
 
     @classmethod
     def getSiteExampleURLs(self):
-        return "http://www.fictionpress.com/s/1234/1/ http://www.fictionpress.com/s/1234/12/ http://www.fictionpress.com/s/1234/1/Story_Title http://m.fictionpress.com/s/1234/1/"
+        return "https://www.fictionpress.com/s/1234/1/ https://www.fictionpress.com/s/1234/12/ http://www.fictionpress.com/s/1234/1/Story_Title http://m.fictionpress.com/s/1234/1/"
 
     def getSiteURLPattern(self):
-        return r"http://(www|m)?\.fictionpress\.com/s/\d+(/\d+)?(/|/[a-zA-Z0-9_-]+)?/?$"
+        return r"https?://(www|m)?\.fictionpress\.com/s/\d+(/\d+)?(/|/[a-zA-Z0-9_-]+)?/?$"
 
 def getClass():
     return FictionPressComSiteAdapter

fanficdownloader/adapters/adapter_ficwadcom.py

@@ -85,7 +85,11 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
 
         # use BeautifulSoup HTML parser to make everything easier to find.
         try:
-            soup = bs.BeautifulSoup(self._fetchUrl(url))
+            data = self._fetchUrl(url)
+            # non-existent/removed story urls get thrown to the front page.
+            if "<h2>Welcome to FicWad</h2>" in data:
+                raise exceptions.StoryDoesNotExist(self.url)
+            soup = bs.BeautifulSoup(data)
         except urllib2.HTTPError, e:
             if e.code == 404:
                 raise exceptions.StoryDoesNotExist(self.url)

fanficdownloader/adapters/adapter_fimfictionnet.py

@@ -101,7 +101,7 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
         # if "/images/missing_story.png" in data:
         #     raise exceptions.StoryDoesNotExist(self.url)
         
-        if "This story has been marked as having adult content." in data:
+        if "This story has been marked as having adult content. Please click below to confirm you are of legal age to view adult material in your country." in data:
             raise exceptions.AdultCheckRequired(self.url)
         
         if self.password:
@@ -249,11 +249,35 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
                     value = unicode(value)
                 self.story.setMetadata(metakey, value)
 
-        rawGroupList = soup.find('ul', {'id':'story_group_list'})
-        if rawGroupList is not None:
-            for groupName in rawGroupList.findAll('a', {'href':re.compile('^/group/')}):
-                self.story.addToList("groups",stripHTML(groupName))
-            
+        ## Groups and sequels code from FaceDeer
+        allGroupLists = soup.findAll('ul', {'id':'story_group_list'})
+        for groupList in allGroupLists:
+            for groupName in groupList.findAll('a', {'href':re.compile('^/group/')}):
+                self.story.addToList("groupsUrl", 'http://'+self.host+groupName["href"]) 
+                self.story.addToList("groups",stripHTML(groupName).replace(',', ';'))
+
+        sequelStoryHeader = soup.find('h1', {'class':'header-stories'}, text="Sequels")
+        if not sequelStoryHeader == None:
+            sequelContainer = sequelStoryHeader.parent.parent
+            for sequel in sequelContainer.findAll('a', {'class':'story_link'}):
+                self.story.addToList("sequelsUrl", 'http://'+self.host+sequel["href"]) 
+                self.story.addToList("sequels", stripHTML(sequel).replace(',', ';'))
+                
+        #The link to the prequel is embedded in the description text, so erring
+        #on the side of caution and wrapping this whole thing in a try block.
+        #If anything goes wrong this probably wasn't a valid prequel link.
+        try:
+            description = soup.find('div', {'class':'description'})
+            firstHR = description.find("hr")
+            nextSib = firstHR.nextSibling
+            if "This story is a sequel to" in nextSib.string:
+                link = nextSib.nextSibling
+                if link.name == "a":
+                    self.story.setMetadata("prequelUrl", 'http://'+self.host+link["href"])
+                    self.story.setMetadata("prequel", stripHTML(link))
+        except:
+            pass
+        
     def hookForUpdates(self,chaptercount):
         if self.oldchapters and len(self.oldchapters) > self.newestChapterNum:
             print("Existing epub has %s chapters\nNewest chapter is %s.  Discarding old chapters from there on."%(len(self.oldchapters), self.newestChapterNum+1))

fanficdownloader/adapters/adapter_finestoriescom.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-# Copyright 2012 Fanficdownloader team
+# Copyright 2013 Fanficdownloader team
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -37,11 +37,6 @@ class FineStoriesComAdapter(BaseSiteAdapter):
     def __init__(self, config, url):
         BaseSiteAdapter.__init__(self, config, url)
 
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
         self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
         self.password = ""
         self.is_adult=False
@@ -51,7 +46,6 @@ class FineStoriesComAdapter(BaseSiteAdapter):
         if 'storyInfo' in self.story.getMetadata('storyId'):
             self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
         
-        
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/s/storyInfo.php?id='+self.story.getMetadata('storyId'))
         
@@ -93,7 +87,7 @@ class FineStoriesComAdapter(BaseSiteAdapter):
             params['theusername'] = self.getConfig("username")
             params['thepassword'] = self.getConfig("password")
         params['rememberMe'] = '1'
-        params['page'] = 'http://finestories.com/'
+        params['page'] = 'http://'+self.getSiteDomain()+'/'
         params['submit'] = 'Login'
     
         loginUrl = 'http://' + self.getSiteDomain() + '/login.php'
@@ -262,8 +256,6 @@ class FineStoriesComAdapter(BaseSiteAdapter):
                 last[len(last)-1]=last[len(last)-1].append(next)
                 div.append(div1)
             
-            
-            
         # removing all the left-over stuff    
         for a in div.findAll('span'):
             a.extract()

fanficdownloader/adapters/adapter_hpfanficarchivecom.py

@@ -128,23 +128,22 @@ class HPFanficArchiveComAdapter(BaseSiteAdapter):
         # <span class="label">Rated:</span> NC-17<br /> etc
         labels = soup.findAll('span',{'class':'label'})
         for labelspan in labels:
-            value = labelspan.nextSibling
+            val = labelspan.nextSibling
+            value = unicode('')
+            while val and not defaultGetattr(val,'class') == 'label':
+                value += unicode(val)
+                val = val.nextSibling
             label = labelspan.string
+            #print("label:%s\nvalue:%s"%(label,value))
 
             if 'Summary' in label:
-                ## Everything until the next span class='label'
-                svalue = ""
-                while not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
-                    value = value.nextSibling
-                self.setDescription(url,svalue)
-                #self.story.setMetadata('description',stripHTML(svalue))
+                self.setDescription(url,value)
 
             if 'Rated' in label:
-                self.story.setMetadata('rating', value)
+                self.story.setMetadata('rating', stripHTML(value))
 
             if 'Word count' in label:
-                self.story.setMetadata('numWords', value)
+                self.story.setMetadata('numWords', stripHTML(value))
 
             if 'Categories' in label:
                 cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
@@ -167,7 +166,7 @@ class HPFanficArchiveComAdapter(BaseSiteAdapter):
                     self.story.addToList('warnings',warning.string)
 
             if 'Completed' in label:
-                if 'Yes' in value:
+                if 'Yes' in stripHTML(value):
                     self.story.setMetadata('status', 'Completed')
                 else:
                     self.story.setMetadata('status', 'In-Progress')

fanficdownloader/adapters/adapter_literotica.py

@@ -0,0 +1,237 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+import urlparse
+import time
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter, makeDate
+
+class LiteroticaSiteAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["utf8",
+                       "Windows-1252"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        
+        self.story.setMetadata('siteabbrev','litero')
+
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
+        
+        self.origurl = url
+        if "http://www.i." in self.origurl:
+            ## accept m(mobile)url, but use www.
+            self.origurl = self.origurl.replace("http://www.i.","http://www.")
+
+        # normalized story URL.
+        self._setURL("http://"+self.getSiteDomain()\
+                         +"/s/"+self.story.getMetadata('storyId'))
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = '%m/%d/%y'
+        
+    @staticmethod
+    def getSiteDomain():
+        return 'www.literotica.com'
+
+    @classmethod
+    def getAcceptDomains(cls):
+        return ['www.literotica.com', 'www.i.literotica.com']
+
+    @classmethod
+    def getSiteExampleURLs(self):
+        #return "http://www.literotica.com/s/story-title http://www.literotica.com/stories/showstory.php?id=1234 http://www.i.literotica.com/stories/showstory.php?id=1234"
+        return "http://www.literotica.com/s/story-title"
+
+    def getSiteURLPattern(self):
+        return r"http://www(\.i)?\.literotica\.com/s/([a-zA-Z0-9_-]+)"
+
+    def extractChapterUrlsAndMetadata(self):
+
+        if not (self.is_adult or self.getConfig("is_adult")):
+            raise exceptions.AdultCheckRequired(self.url)
+        
+        url1 = self.origurl
+        logger.debug("first page URL: "+url1)
+
+        try:
+            data1 = self._fetchUrl(url1)
+            soup1 = bs.BeautifulSoup(data1)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(url1)
+            else:
+                raise e
+
+        #strip comments from soup
+        [comment.extract() for comment in soup1.findAll(text=lambda text:isinstance(text, bs.Comment))]
+
+        # author
+        a = soup1.find("span", "b-story-user-y")
+        self.story.setMetadata('authorId', urlparse.parse_qs(a.a['href'].split('?')[1])['uid'])
+        self.story.setMetadata('authorUrl', a.a['href'])
+        self.story.setMetadata('author', a.text)
+
+        # get the author page
+        try:
+            dataAuth = self._fetchUrl(a.a['href'])
+            soupAuth = bs.BeautifulSoup(dataAuth)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(a.a['href'])
+            else:
+                raise e
+
+        storyLink = soupAuth.find('a', href=url1)
+
+        if storyLink is not None:
+            # pull the published date from the author page
+            # default values from single link.  Updated below if multiple chapter.
+            date = storyLink.parent.parent.findAll('td')[-1].text
+            self.story.setMetadata('datePublished', makeDate(date, self.dateformat))
+            self.story.setMetadata('dateUpdated',makeDate(date, self.dateformat))
+
+        # find num of pages
+        # find a "3 Pages:" string on the page and parse it
+        pgs = soup1.find("span", "b-pager-caption-t r-d45").string.split(' ')[0]
+
+        # If there are multiple pages, find and request the last page
+        if "1" != pgs:
+            logger.debug("last page number: "+pgs)
+            try:
+                data2 = self._fetchUrl(url1, {'page': pgs})
+                soup2 = bs.BeautifulSoup(data2)
+                [comment.extract() for comment in soup2.findAll(text=lambda text:isinstance(text, bs.Comment))]
+            except urllib2.HTTPError, e:
+                if e.code == 404:
+                    # TODO: Probably should reformat this
+                    raise exceptions.StoryDoesNotExist(url1, {'page': pgs})
+                else:
+                    raise e
+        else:
+            #If we're already on the last page, copy the soup
+            soup2 = soup1
+
+        # parse out the list of chapters
+        chaps = soup2.find('div', id='b-series')
+        if chaps:  # may be one post only
+            #self.chapterUrls = [(ch.a.text, ch.a['href']) for ch in chaps.findAll('li')]
+
+            # if there are chapters, lets pull them and title from the
+            # author page because *this* chapter is omitted from the
+            # list on the last page.
+            row = storyLink.parent.parent.previousSibling
+            while row['class'] != 'ser-ttl':
+                row = row.previousSibling
+
+            seriesTitle = stripHTML(row)
+            if seriesTitle:
+                # this regex is deliberately greedy. We want to get the biggest match before a ':'
+                self.story.setMetadata('title', re.match('(.*):[^:]*$', seriesTitle).group(1))
+            else:
+                self.story.setMetadata('title', soup1.h1.string)
+
+            # now chapter list.  Assumed oldest to newest.
+            self.chapterUrls = []
+            row = row.nextSibling
+                
+            self.story.setMetadata('datePublished',makeDate(stripHTML(row.find('td',{'class':'dt'})), self.dateformat))
+            while row['class'] == 'sl':
+                # pages include full URLs.
+                self.chapterUrls.append((row.a.string,row.a['href']))
+                if not row.nextSibling:
+                    break
+                row = row.nextSibling
+
+            row = row.previousSibling
+            self.story.setMetadata('dateUpdated',makeDate(stripHTML(row.find('td',{'class':'dt'})), self.dateformat))
+                
+        else:  # if one post only
+            self.chapterUrls = [(soup1.h1.string, url1)]
+            self.story.setMetadata('title', soup1.h1.string)
+
+        # normalize on first chapter URL.
+        self._setURL(self.chapterUrls[0][1])
+
+        # reset storyId to first chapter.
+        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
+
+        self.story.setMetadata('numChapters', len(self.chapterUrls))
+
+        self.story.setMetadata('category', soup1.find('div', 'b-breadcrumbs').findAll('a')[1].string)
+        self.story.setMetadata('description', soup1.find('meta', {'name': 'description'})['content'])
+
+        return
+
+    def getChapterText(self, url):
+        logger.debug('Getting chapter text from: %s' % url)
+        time.sleep(0.5)
+        data1 = self._fetchUrl(url)
+        soup1 = bs.BeautifulSoup(data1)
+
+        #strip comments from soup
+        [comment.extract() for comment in soup1.findAll(text=lambda text:isinstance(text, bs.Comment))]
+
+        # get story text
+        story1 = soup1.find('div', 'b-story-body-x').p
+        story1.name='div'
+        story1.append('<br>')
+        storytext = self.utf8FromSoup(url,story1)
+
+        # find num pages
+        pgs = int(soup1.find("span", "b-pager-caption-t r-d45").string.split(' ')[0])
+        logger.debug("pages: "+str(pgs))
+
+        # get all the pages
+        for i in xrange(2, pgs+1):
+            try:
+                logger.debug("fetching page "+str(i))
+                time.sleep(0.5)
+                data2 = self._fetchUrl(url, {'page': i})
+                soup2 = bs.BeautifulSoup(data2)
+                [comment.extract() for comment in soup2.findAll(text=lambda text:isinstance(text, bs.Comment))]
+                story2 = soup2.find('div', 'b-story-body-x').p
+                story2.name='div'
+                story2.append('<br>')
+                storytext += self.utf8FromSoup(url,story2)
+            except urllib2.HTTPError, e:
+                if e.code == 404:
+                    raise exceptions.StoryDoesNotExist(url)
+                else:
+                    raise e
+        return storytext
+
+
+def getClass():
+    return LiteroticaSiteAdapter
+
+
+

fanficdownloader/adapters/adapter_nickandgregnet.py

@@ -166,10 +166,11 @@ class NickAndGregNetAdapter(BaseSiteAdapter):
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.find('table', {'class' : 'tblborder6'})
 
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
+        # wrap a div around it.
+        divsoup = bs.BeautifulStoneSoup('<div class="story"></div>',
+                                        selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        div = divsoup.find('div')
+        div.append(soup.find('table', {'class' : 'tblborder6'}))
+
         return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_potionsandsnitchesnet.py

@@ -163,12 +163,11 @@ class PotionsAndSnitchesNetSiteAdapter(BaseSiteAdapter):
                     self.story.setMetadata('status', 'In-Progress')
 
             if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), "%d %b %Y"))
+                # limit date values, there's some extra chars.
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value[:12]), "%d %b %Y"))
             
             if 'Updated' in label:
-                # there's a stray [ at the end.
-                #value = value[0:-1]
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), "%d %b %Y"))
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value[:12]), "%d %b %Y"))
 
         try:
             # Find Series name from series URL.

fanficdownloader/adapters/adapter_simplyundeniablecom.py

@@ -214,5 +214,6 @@ class SimplyUndeniableComAdapter(BaseSiteAdapter):
 
         if None == div:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+        div.name='div'
     
         return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_siyecouk.py

@@ -241,5 +241,6 @@ class SiyeCoUkAdapter(BaseSiteAdapter): # XXX
         
         if None == story:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+        story.name='div'
     
         return self.utf8FromSoup(url,story)

fanficdownloader/adapters/adapter_squidgeorgpeja.py

@@ -103,6 +103,9 @@ class SquidgeOrgPejaAdapter(BaseSiteAdapter):
                 raise e
 
         data = self._fetchUrl(url)
+
+        if "fatal MySQL error was encountered" in data:
+            raise exceptions.FailedToDownload("Site SQL Error--bad story")
             
         # use BeautifulSoup HTML parser to make everything easier to find.
         soup = bs.BeautifulSoup(data)

fanficdownloader/adapters/adapter_storiesonlinenet.py

@@ -0,0 +1,401 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return StoriesOnlineNetAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class StoriesOnlineNetAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url
+        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2].split(':')[0])
+        if 'storyInfo' in self.story.getMetadata('storyId'):
+            self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/s/'+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','strol')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%Y-%m-%d"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'storiesonline.net'
+
+    @classmethod
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/s/1234 http://"+self.getSiteDomain()+"/s/1234:4010"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain())+r"/s/\d+((:\d+)?(;\d+)?$|(:i)?$)"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Free Registration' in data \
+                or "Invalid Password!" in data \
+                or "Invalid User Name!" in data \
+                or "Log In" in data \
+                or "Access to unlinked chapters requires" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['theusername'] = self.username
+            params['thepassword'] = self.password
+        else:
+            params['theusername'] = self.getConfig("username")
+            params['thepassword'] = self.getConfig("password")
+        params['rememberMe'] = '1'
+        params['page'] = 'http://'+self.getSiteDomain()+'/'
+        params['submit'] = 'Login'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/login.php'
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['theusername']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "My Account" not in d : #Member Account
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['theusername']))
+            raise exceptions.FailedToLogin(url,params['theusername'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url+":i")
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url+":i")
+        
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+        elif "Error! The story you're trying to access is being filtered by your choice of contents filtering." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Error! The story you're trying to access is being filtered by your choice of contents filtering.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        #print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('h1')
+        self.story.setMetadata('title',stripHTML(a))
+        
+        notice = soup.find('div', {'class' : 'notice'})
+        if notice:
+            self.story.setMetadata('notice',unicode(notice))
+
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"/a/\w+"))
+        self.story.setMetadata('authorId',a['href'].split('/')[2])
+        self.story.setMetadata('authorUrl','http://'+self.host+a['href'])
+        self.story.setMetadata('author',stripHTML(a).replace("'s Page",""))
+
+        # Find the chapters:
+        chapters = soup.findAll('a', href=re.compile(r'^/s/'+self.story.getMetadata('storyId')+":\d+$"))
+        if len(chapters) != 0:
+            for chapter in chapters:
+                # just in case there's tags, like <i> in chapter titles.
+                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+chapter['href']))
+        else:
+            self.chapterUrls.append((self.story.getMetadata('title'),'http://'+self.host+'/s/'+self.story.getMetadata('storyId')))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # surprisingly, the detailed page does not give enough details, so go to author's page
+        skip=0
+        i=0
+        while i == 0:
+            asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')+"&skip="+str(skip)))
+
+            a = asoup.findAll('td', {'class' : 'lc2'})
+            for lc2 in a:
+                if lc2.find('a')['href'] == '/s/'+self.story.getMetadata('storyId'):
+                    i=1
+                    break
+                if a[len(a)-1] == lc2:
+                    skip=skip+10
+
+        for cat in lc2.findAll('div', {'class' : 'typediv'}):
+            self.story.addToList('genre',cat.text)
+
+        # in lieu of word count.
+        self.story.setMetadata('size', lc2.findNext('td', {'class' : 'num'}).text)
+        
+        lc4 = lc2.findNext('td', {'class' : 'lc4'})        
+        desc = lc4.contents[0]
+
+        try:
+            a = lc4.find('a', href=re.compile(r"/library/show_series.php\?id=\d+"))
+            if a:
+                # if there's a number after the series name, series_contents is a two element list:
+                # [<a href="...">Title</a>, u' (2)']
+                series_contents = a.parent.contents
+                i = 0 if len(series_contents) == 1 else series_contents[1].strip(' ()')
+                seriesUrl = 'http://'+self.host+a['href']
+                self.story.setMetadata('seriesUrl',seriesUrl)
+                series_name = stripHTML(a)
+                logger.debug("Series name= %s" % series_name)
+                series_soup = bs.BeautifulSoup(self._fetchUrl(seriesUrl))
+                if series_soup:
+                    logger.debug("Retrieving Series - looking for name")
+                    series_name = series_soup.find('span', {'id' : 'ptitle'}).text.partition(' — ')[0]
+                    logger.debug("Series name: '{0}'".format(series_name))
+                self.setSeries(series_name, i)
+                desc = lc4.contents[2]
+                # Check if series is in a universe
+                universes_soup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')  + "&type=uni")) 
+#                logger.debug("Universe page=", universes_soup)
+                if universes_soup:
+                    universes = universes_soup.findAll('div', {'class' : 'ser-box'})
+                    logger.debug("Number of Universes: %d" % len(universes))
+                    for universe in universes:
+                        logger.debug("universe.find('a')={0}".format(universe.find('a')))
+                        # The universe id is in an "a" tag that has an id but nothing else. It is the first tag.
+                        # The id is prefixed with the letter "u".
+                        universe_id = universe.find('a')['id'][1:]
+                        logger.debug("universe_id='%s'" % universe_id)
+                        universe_name = universe.find('div', {'class' : 'ser-name'}).text.partition(' ')[2]
+                        logger.debug("universe_name='%s'" % universe_name)
+                        # If there is link to the story, we have the right universe
+                        story_a = universe.find('a', {'href' : '/s/'+self.story.getMetadata('storyId')})
+                        if story_a:
+                            logger.debug("Story is in a series that is in a universe! The universe is '%s'" % universe_name)
+                            self.story.setMetadata("universe", universe_name)
+                            self.story.setMetadata('universeUrl','http://'+self.host+ '/library/universe.php?id=' + universe_id)
+                            break
+        except:
+            pass
+        try:
+            a = lc4.find('a', href=re.compile(r"/library/universe.php\?id=\d+"))
+            if a:
+                self.story.setMetadata("universe",stripHTML(a))
+                desc = lc4.contents[2]
+                # Assumed only one universe, but it does have a URL--use universeHTML
+                universe_name = stripHTML(a)
+                universeUrl = 'http://'+self.host+a['href']
+                logger.debug("Retrieving Universe - about to get page")
+                universe_soup = bs.BeautifulSoup(self._fetchUrl(universeUrl))
+                logger.debug("Retrieving Universe - have page")
+                if universe_soup:
+                    logger.debug("Retrieving Universe - looking for name")
+                    universe_name = universe_soup.find('span', {'id' : 'ptitle'}).text.partition(' &mdash;')[0]
+                    logger.debug("Universes name: '{0}'".format(universe_name))
+
+                self.story.setMetadata('universeUrl',universeUrl)
+                logger.debug("Setting universe name: '{0}'".format(universe_name))
+                self.story.setMetadata('universe',universe_name)
+                if self.getConfig("universe_as_series"):
+                    self.setSeries(universe_name, 0)
+                    self.story.setMetadata('seriesUrl',universeUrl)
+        except:
+            pass
+
+        self.setDescription('http://'+self.host+'/s/'+self.story.getMetadata('storyId'),desc)
+            
+        for b in lc4.findAll('b'):
+            #logger.debug('Getting metadata: "%s"' % b)
+            label = b.text
+            if label in ['Posted:', 'Concluded:', 'Updated:']:
+                value = b.findNext('noscript').text
+                #logger.debug('Have a date field label: "%s", value: "%s"' % (label, value))
+            else:
+                value = b.nextSibling
+            #logger.debug('label: "%s", value: "%s"' % (label, value))
+            
+            if 'Sex' in label:
+                self.story.setMetadata('rating', value)
+                
+            if 'Codes' in label:
+                for code in value.split(' '):
+                    self.story.addToList('codes',code)
+                    
+            if 'Posted' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+                
+            if 'Concluded' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+                
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+#                
+        status = lc4.find('span', {'class' : 'ab'})
+        if  status != None:
+            self.story.setMetadata('status', 'In-Progress')
+            if "Last Activity" in status.text:
+                # date is passed as a timestamp and converted in JS.
+                value = status.findNext('noscript').text
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+        else:
+            self.story.setMetadata('status', 'Completed')
+
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+        
+        # some big chapters are split over several pages
+        pager = div.find('span', {'class' : 'pager'})
+        if pager != None:
+            a = pager.previousSibling
+            while a != None:
+                logger.debug("before pager: {0}".format(a))
+                b = a.previousSibling
+                a.extract()
+                a = b
+
+            urls=pager.findAll('a')
+            urls=urls[:len(urls)-1]
+            pager.extract()
+            div.contents = div.contents[2:]
+#            logger.debug(div)
+                        
+            for ur in urls:
+                soup = bs.BeautifulSoup(self._fetchUrl("http://"+self.getSiteDomain()+ur['href']),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+                div1 = soup.find('div', {'id' : 'story'})
+                
+                # Find the "Continues" marker on the current page and remove everything after that. 
+                continues = div.find('span', {'class' : 'conTag'})
+                if continues != None:
+                    while continues != None:
+#                        logger.debug("removing end: {0}".format(continues))
+                        b = continues.nextSibling
+                        continues.extract()
+                        continues = b
+
+                # Find the "Continued" marker and delete everything before that
+                continued = div1.find('span', {'class' : 'conTag'})
+                if continued != None:
+                    a = continued.previousSibling
+                    while a != None:
+#                        logger.debug("before conTag: {0}".format(a))
+                        b = a.previousSibling
+                        a.extract()
+                        a = b
+                # Remove the pager from the end if this is the last page
+                endPager = div1.find('span', {'class' : 'pager'})
+                if endPager != None:
+                    b = endPager.nextSibling
+                    while endPager != None:
+                        logger.debug("removing end: {0}".format(endPager))
+                        b = endPager.nextSibling
+                        endPager.extract()
+                        endPager = b
+                    div1.contents = div1.contents[:len(div1) - 2]
+#                logger.debug("after removing pager: {0}".format(div1))
+                for tag in div1.contents[2:]:
+                    div.append(tag)
+
+        # If it is a chapter, there are dates at the start for when it was posted or modified. These plus 
+        # everything before them can be discarded. 
+        postedDates = div.findAll('div', {'class' : 'date'})
+        if postedDates:
+            a = postedDates[0].previousSibling
+            while a != None:
+#                logger.debug("before dates: {0}".format(a))
+                b = a.previousSibling
+                a.extract()
+                a = b
+            for a in div.findAll('div', {'class' : 'date'}):
+                a.extract()
+
+        # For single chapter stories, there is a copyright statement. Remove this and everything
+        # before it.
+        copy = div.find('h4', {'class': 'copy'})
+        while copy != None:
+#            logger.debug("before copyright: {0}".format(copy))
+            b = copy.previousSibling
+            copy.extract()
+            copy = b
+
+        # For a story or the last chapter, remove voting form and the in library box
+        a = div.find('div', {'id' : 'vote-form'})
+        if a != None:
+            a.extract()
+        a = div.find('div', {'id' : 'b-man-div'})
+        if a != None:
+            a.extract()
+
+        # Kill the "The End" header and everything after it.
+        a = div.find(['h2', 'h3'], {'class' : 'end'})
+        logger.debug("Chapter end= '{0}'".format(a))
+        while a != None:
+            b = a.nextSibling
+            a.extract()
+            a=b
+
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_test1.py

@@ -50,6 +50,7 @@ class TestSiteAdapter(BaseSiteAdapter):
     def extractChapterUrlsAndMetadata(self):
         idstr = self.story.getMetadata('storyId')
         idnum = int(idstr)
+        self.do_sleep()
 
         if idnum >= 1000:
             logger.warn("storyId:%s - Custom INI data will be used."%idstr)
@@ -296,6 +297,7 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
 
     def getChapterText(self, url):
         logger.debug('Getting chapter text from: %s' % url)
+        self.do_sleep()
         if self.story.getMetadata('storyId').startswith('670') or \
                 self.story.getMetadata('storyId').startswith('672'):
             time.sleep(1.0)
@@ -323,7 +325,7 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
 </div>
 '''
         elif self.story.getMetadata('storyId') == '0':
-            text=u'''
+            text=u'''<div>
 <h3>45. Pronglet Returns to Hogwarts: Chapter 7</h3>
 <br />
     eyes… but I’m not convinced we should automatically<br />
@@ -332,6 +334,7 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
 <br /><br />
     “Sure, invite her along. Does she have children?”<br />
 <br />
+</div>
 '''
         else:
             if self.story.getMetadata('storyId') == '667':

fanficdownloader/adapters/adapter_trekiverseorg.py

@@ -0,0 +1,325 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return TrekiverseOrgAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class TrekiverseOrgAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["iso-8859-1",
+                       "Windows-1252"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        
+        # normalized story URL.
+        self._setURL("http://"+self.getSiteDomain()\
+                         +"/efiction/viewstory.php?sid="+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','trkvs')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%d/%m/%Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.
+        return 'trekiverse.org'
+
+    @classmethod
+    def getAcceptDomains(cls):
+        return ['trekiverse.org','efiction.trekiverse.org']
+
+    @classmethod
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/efiction/viewstory.php?sid=1234 http://efiction."+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return r'(http://trekiverse\.org/efiction/viewstory\.php\?sid=\d+|http://efiction\.trekiverse\.org/viewstory\.php\?sid=\d+)'
+       
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Registered Users Only' in data \
+                or 'There is no such account on our website' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['cookiecheck'] = '1'
+        params['submit'] = 'Submit'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/efiction/user.php?action=login'
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "Member Account" not in d : #Member Account
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&index=1&ageconsent=ok&warning=5"
+        else:
+            addurl="&index=1"
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+addurl
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+            
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid & error in url.
+                addurl = addurl.replace("&","&")
+                url = self.url+'&index=1'+addurl
+                logger.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title and author
+        a = soup.find('div', {'id' : 'pagetitle'})
+        aut = a.find('a', href=re.compile(r"^viewuser\.php\?uid="))
+        self.story.setMetadata('authorId',aut['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/efiction/'+aut['href'])
+        self.story.setMetadata('author',aut.string)
+        
+        ttl = a.find('a', href=re.compile(r'^viewstory.php\?sid=%s$'%self.story.getMetadata('storyId')))
+        self.story.setMetadata('title',ttl.string)
+
+        # Find the chapters:
+        outputdiv = soup.find('div', {'id':'output'})
+        # (amp;)? because it should be &, but is escaped to & in URL.
+        # viewstory.php?sid=35&chapter=3
+        chapters=outputdiv.findAll('a', href=re.compile(r'^viewstory.php\?sid=%s&(amp;)?chapter=\d+$'%self.story.getMetadata('storyId')))
+        if len(chapters)==0:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: No php/html chapters found.")
+        if len(chapters)==1:
+            self.chapterUrls.append((self.story.getMetadata('title'),'http://'+self.host+'/efiction/'+chapters[0]['href']))
+        else:
+            for chapter in chapters:
+                # just in case there's tags, like <i> in chapter titles.
+                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/efiction/'+chapter['href']))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+        
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ''
+                while value and not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                # sometimes poorly formated desc (<p> w/o </p>) leads
+                # to all labels being included.
+                svalue=svalue[:svalue.find('<span class="label">')]
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                catstext = [cat.string for cat in cats]
+                for cat in catstext:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                charstext = [char.string for char in chars]
+                for char in charstext:
+                    self.story.addToList('characters',char.string)
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=9'))
+                genrestext = [genre.string for genre in genres]
+                self.genre = ', '.join(genrestext)
+                for genre in genrestext:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Awards' in label:
+                awards = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=12'))
+                awardstext = [award.string for award in awards]
+                self.award = ', '.join(awardstext)
+                for award in awardstext:
+                    self.story.addToList('awards',award.string)
+
+            if 'Pairing' in label:
+                ships = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=10'))
+                shipstext = [ship.string for ship in ships]
+                self.ship = ', '.join(shipstext)
+                for ship in shipstext:
+                    self.story.addToList('ships',ship.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=11'))
+                warningstext = [warning.string for warning in warnings]
+                self.warning = ', '.join(warningstext)
+                for warning in warningstext:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(value.strip(), "%d %b %Y"))
+            
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(value.strip(), "%d %b %Y"))
+                    
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/efiction/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulSoup(self._fetchUrl(url)) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+
+        notesdiv = soup.find('div', {'class':'noteinfo'})
+        if notesdiv != None:
+            div.insert(0,"<hr>")
+            div.insert(0,notesdiv)
+            div.insert(0,"<hr>")
+        
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_tthfanficorg.py

@@ -134,18 +134,18 @@ class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter):
         if "<h2>Story Not Found</h2>" in data:
             raise exceptions.StoryDoesNotExist(url)
         
+        if self.is_adult or self.getConfig("is_adult"):
+            form = soup.find('form', {'id':'sitemaxratingform'})
+            params={'ctkn':form.find('input', {'name':'ctkn'})['value'],
+                    'sitemaxrating':'5'}
+            logger.info("Attempting to get rating cookie for %s" % url)
+            data = self._postUrl("http://"+self.getSiteDomain()+'/setmaxrating.php',params)
+            # refetch story page.
+            data = self._fetchUrl(url)
+            soup = bs.BeautifulSoup(data)
+
         if "NOTE: This story is rated FR21 which is above your chosen filter level." in data:
-            if self.is_adult or self.getConfig("is_adult"):
-                form = soup.find('form', {'id':'sitemaxratingform'})
-                params={'ctkn':form.find('input', {'name':'ctkn'})['value'],
-                        'sitemaxrating':'5'}
-                logger.info("Attempting to get rating cookie for %s" % url)
-                data = self._postUrl("http://"+self.getSiteDomain()+'/setmaxrating.php',params)
-                # refetch story page.
-                data = self._fetchUrl(url)
-                soup = bs.BeautifulSoup(data)
-            else:
-                raise exceptions.AdultCheckRequired(self.url)
+            raise exceptions.AdultCheckRequired(self.url)
 
         # http://www.tthfanfic.org/AuthorStories-3449/Greywizard.htm
         # Find authorid and URL from... author url.
@@ -163,7 +163,14 @@ class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter):
             authorsoup = bs.BeautifulSoup(authordata)
             # author can have several pages, scan until we find it.
             while( not authorsoup.find('a', href=re.compile(r"^/Story-"+self.story.getMetadata('storyId'))) ):
-                nextpage = 'http://'+self.host+authorsoup.find('a', {'class':'arrowf'})['href']
+                nextarrow = authorsoup.find('a', {'class':'arrowf'})
+                if not nextarrow:
+                    ## if rating is set lower than story, it won't be
+                    ## visible on author lists unless.  The *story* is
+                    ## visible via the url, just not the entry on
+                    ## author list.
+                    raise exceptions.AdultCheckRequired(self.url)
+                nextpage = 'http://'+self.host+nextarrow['href']
                 logger.debug("**AUTHOR** nextpage URL: "+nextpage)
                 authordata = self._fetchUrl(nextpage)
                 descurl=nextpage

fanficdownloader/adapters/adapter_whoficcom.py

@@ -228,7 +228,8 @@ class WhoficComSiteAdapter(BaseSiteAdapter):
 
         if None == span:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
+
+        span.name='div'
         return self.utf8FromSoup(url,span)
 
 def getClass():

fanficdownloader/adapters/base_adapter.py

@@ -26,6 +26,7 @@ from functools import partial
 
 from .. import BeautifulSoup as bs
 from ..htmlcleanup import stripHTML
+from ..htmlheuristics import replace_br_with_p
 
 logger = logging.getLogger(__name__)
 
@@ -75,6 +76,8 @@ class BaseSiteAdapter(Configurable):
         self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
         self.password = ""
         self.is_adult=False
+
+        self.override_sleep = None
         
         self.opener = u2.build_opener(u2.HTTPCookieProcessor(),GZipProcessor())
         ## Specific UA because too many sites are blocking the default python UA.
@@ -146,8 +149,7 @@ class BaseSiteAdapter(Configurable):
 
     # Assumes application/x-www-form-urlencoded.  parameters, headers are dict()s
     def _postUrl(self, url, parameters={}, headers={}):
-        if self.getConfig('slow_down_sleep_time'):
-            time.sleep(float(self.getConfig('slow_down_sleep_time')))
+        self.do_sleep()
 
         ## u2.Request assumes POST when data!=None.  Also assumes data
         ## is application/x-www-form-urlencoded.
@@ -158,24 +160,38 @@ class BaseSiteAdapter(Configurable):
         req = u2.Request(url,
                          data=urllib.urlencode(parameters),
                          headers=headers)
-        return self._decode(self.opener.open(req).read())
+        return self._decode(self.opener.open(req,None,float(self.getConfig('connect_timeout',30.0))).read())
 
     def _fetchUrlRaw(self, url, parameters=None):
         if parameters != None:
-            return self.opener.open(url.replace(' ','%20'),urllib.urlencode(parameters)).read()
+            return self.opener.open(url.replace(' ','%20'),urllib.urlencode(parameters),float(self.getConfig('connect_timeout',30.0))).read()
         else:
-            return self.opener.open(url.replace(' ','%20')).read()
+            return self.opener.open(url.replace(' ','%20'),None,float(self.getConfig('connect_timeout',30.0))).read()
+
+    def set_sleep(self,val):
+        print("\n===========\n set sleep time %s\n==========="%val)
+        self.override_sleep = val
     
+    def do_sleep(self):
+        if self.override_sleep:
+            time.sleep(float(self.override_sleep))
+        elif self.getConfig('slow_down_sleep_time'):
+            time.sleep(float(self.getConfig('slow_down_sleep_time')))
+        
     # parameters is a dict()
     def _fetchUrl(self, url, parameters=None):
-        if self.getConfig('slow_down_sleep_time'):
-            time.sleep(float(self.getConfig('slow_down_sleep_time')))
+        self.do_sleep()
 
         excpt=None
         for sleeptime in [0, 0.5, 4, 9]:
             time.sleep(sleeptime)	
             try:
                 return self._decode(self._fetchUrlRaw(url,parameters))
+            except u2.HTTPError, he:
+                if he.code == 404:
+                    logger.warn("Caught an exception reading URL: %s  Exception %s."%(unicode(url),unicode(he)))
+                    excpt=he
+                    break # break out on 404
             except Exception, e:
                 excpt=e
                 logger.warn("Caught an exception reading URL: %s  Exception %s."%(unicode(url),unicode(e)))
@@ -351,18 +367,13 @@ class BaseSiteAdapter(Configurable):
             if t.name in ('center'):
                 t['class']=t.name
                 t.name='div'
-	    # removes paired, but empty tags.
-            if t.string != None and len(t.string.strip()) == 0 :
+	    # removes paired, but empty non paragraph tags.
+            if t.name not in ('p') and t.string != None and len(t.string.strip()) == 0 :
                 t.extract()
-                
+
         retval = soup.__str__('utf8').decode('utf-8')
 
-        if self.getConfig('replace_hr'):
-            # replacing a self-closing tag with a container tag in the
-            # soup is more difficult than it first appears.  So cheat.
-            retval = retval.replace("<hr />","<div class='center'>* * *</div>")
-
-        if self.getConfig('nook_img_fix'):
+        if self.getConfig('nook_img_fix') and not self.getConfig('replace_br_with_p'):
             # if the <img> tag doesn't have a div or a p around it,
             # nook gets confused and displays it on every page after
             # that under the text for the rest of the chapter.
@@ -371,7 +382,19 @@ class BaseSiteAdapter(Configurable):
             
         # Don't want body tags in chapter html--writers add them.
         # This is primarily for epub updates.
-        return re.sub(r"</?body>\r?\n?","",retval)
+        retval = re.sub(r"</?body[^>]*>\r?\n?","",retval)
+        
+        if self.getConfig("replace_br_with_p"):
+            # Apply heuristic processing to replace <br> paragraph
+            # breaks with <p> tags.
+            retval = replace_br_with_p(retval)
+            
+        if self.getConfig('replace_hr'):
+            # replacing a self-closing tag with a container tag in the
+            # soup is more difficult than it first appears.  So cheat.
+            retval = retval.replace("<hr />","<div class='center'>* * *</div>")
+
+        return retval
 
 def cachedfetch(realfetch,cache,url):
     if url in cache:

fanficdownloader/htmlheuristics.py

@@ -0,0 +1,348 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import logging
+logger = logging.getLogger(__name__)
+import re
+import codecs
+import BeautifulSoup as bs
+import HtmlTagStack as stack
+
+from . import exceptions as exceptions
+
+def replace_br_with_p(body):
+
+    # Ascii character (and Unicode as well) xA0 is a non-breaking space, ascii code 160.
+    # However, Python Regex does not recognize it as a whitespace, so we'll be changing it to a reagular space.
+    body = body.replace(u'\xa0', u' ')
+
+    if body.find('>') == -1 or body.rfind('<') == -1:
+        return body
+
+    # logger.debug(u'BODY start.: ' + body[:250])
+    # logger.debug(u'BODY end...: ' + body[-250:])
+    # logger.debug(u'BODY.......: ' + body)
+
+    # clean breaks (<br />), removing whitespaces between them.
+    body = re.sub(r'\s*<br[^>]*>\s*', r'<br />', body)
+
+    # change surrounding div to a p and remove attrs Top surrounding
+    # tag in all cases now should be div, to just strip the first and
+    # last tags.
+    if is_valid_block(body) and body.find('<div') == 0:
+        body = body[body.index('>')+1:body.rindex('<')]
+
+    body = soup_up_div(u'<div>' + body + u'</div>')
+
+    body = body[body.index('>')+1:body.rindex('<')]
+
+    # Find all bexisting blocks with p, pre and blockquote tags, we need to shields break tags inside those.
+    # This is for "lenient" mode, however it is also used to clear break tags before and after the block elements.
+    blocksRegex = re.compile(r'(\s*<br\ />\s*)*\s*<(pre|p|blockquote|table)([^>]*)>(.+?)</\2>\s*(\s*<br\ />\s*)*', re.DOTALL)
+    body = blocksRegex.sub(r'\n<\2\3>\4</\2>\n', body)
+
+    # if aggressive mode = true
+        # blocksRegex = re.compile(r'(\s*<br\ */*>\s*)*\s*<(pre)([^>]*)>(.+?)</\2>\s*(\s*<br\ */*>\s*)*', re.DOTALL)
+        # In aggressive mode, we also check breakes inside blockquotes, meaning we can get orphaned paragraph tags.
+        # body = re.sub(r'<blockquote([^>]*)>(.+?)</blockquote>', r'<blockquote\1><p>\2</p></blockquote>', body, re.DOTALL)
+    # end aggressive mode
+
+    blocks = blocksRegex.finditer(body)
+    # For our replacements to work, we need to work backwards, so we reverse the iterator.
+    blocksList = []
+    for match in blocks:
+        blocksList.insert(0, match)
+
+    for match in blocksList:
+        group4 =  match.group(4).replace(u'<br />', u'{br /}')
+        body = body[:match.start(4)] + group4 + body[match.end(4):]
+
+    # change surrounding div to a p and remove attrs Top surrounding
+    # tag in all cases now should be div, to just strip the first and
+    # last tags.
+    # body = u'<p>' + body + u'</p>'
+
+    # Nuke div tags surrounding a HR tag.
+    body = re.sub(r'<div[^>]+>\s*<hr[^>]+>\s*</div>', r'\n<hr />\n', body)
+
+    # So many people add formatting to their HR tags, and ePub does not allow those, we are supposed to use css.
+    # This nukes the hr tag attributes.
+    body = re.sub(r'\s*<hr[^>]+>\s*', r'\n<hr />\n', body)
+
+    # Remove leading and trailing breaks from HR tags
+    body = re.sub(r'\s*(<br\ \/>)*\s*<hr\ \/>\s*(<br\ \/>)*\s*', r'\n<hr />\n', body)
+    # Nuking breaks leading paragraps that may be in the body. They are eventually treated as <p><br /></p>
+    body = re.sub(r'\s*(<br\ \/>)+\s*<p', r'\n<p></p>\n<p', body)
+    # Nuking breaks trailing paragraps that may be in the body. They are eventually treated as <p><br /></p>
+    body = re.sub(r'</p>\s*(<br\ \/>)+\s*', r'</p>\n<p></p>\n', body)
+
+    # Because a leading or trailing non break tag will break the following code, we have to mess around rather badly for a few lines.
+    body = body.replace(u'[',u'&squareBracketStart;')
+    body = body.replace(u']',u'&squareBracketEnd;')
+    body = body.replace(u'<br />',u'[br /]')
+
+    breaksRegexp = [
+        re.compile(r'([^\]])(\[br\ \/\])([^\[])'),
+        re.compile(r'([^\]])(\[br\ \/\]){2}([^\[])'),
+        re.compile(r'([^\]])(\[br\ \/\]){3}([^\[])'),
+        re.compile(r'([^\]])(\[br\ \/\]){4}([^\[])'),
+        re.compile(r'([^\]])(\[br\ \/\]){5}([^\[])'),
+        re.compile(r'([^\]])(\[br\ \/\]){6}([^\[])'),
+        re.compile(r'([^\]])(\[br\ \/\]){7}([^\[])'),
+        re.compile(r'([^\]])(\[br\ \/\]){8}([^\[])'),
+        re.compile(r'(\[br\ \/\]){9,}')]
+
+    breaksCount = [
+        len(breaksRegexp[0].findall(body)),
+        len(breaksRegexp[1].findall(body)),
+        len(breaksRegexp[2].findall(body)),
+        len(breaksRegexp[3].findall(body)),
+        len(breaksRegexp[4].findall(body)),
+        len(breaksRegexp[5].findall(body)),
+        len(breaksRegexp[6].findall(body)),
+        len(breaksRegexp[7].findall(body))]
+
+    breaksMax = 0
+    breaksMaxIndex = 0;
+
+    for i in range(1,len(breaksCount)):
+        if breaksCount[i] >= breaksMax:
+            breaksMax = breaksCount[i]
+            breaksMaxIndex = i
+
+    lines = body.split(u'[br /]')
+    contentLines = 0;
+    contentLinesSum = 0;
+    longestLineLength = 0;
+    averageLineLength = 0;
+
+    for line in lines:
+        lineLen = len(line.strip())
+        if lineLen > 0:
+            contentLines += 1
+            contentLinesSum += lineLen
+            if lineLen > longestLineLength:
+                longestLineLength = lineLen
+                
+    if contentLines == 0:
+        contentLines = 1
+
+    averageLineLength = contentLinesSum/contentLines
+
+    logger.debug(u'---')
+    logger.debug(u'Lines.............: ' + unicode(len(lines)))
+    logger.debug(u'contentLines......: ' + unicode(contentLines))
+    logger.debug(u'contentLinesSum...: ' + unicode(contentLinesSum))
+    logger.debug(u'longestLineLength.: ' + unicode(longestLineLength))
+    logger.debug(u'averageLineLength.: ' + unicode(averageLineLength))
+
+    if breaksMaxIndex == len(breaksCount)-1 and breaksMax < 2:
+        breaksMaxIndex = 0
+        breaksMax = breaksCount[0]
+
+    logger.debug(u'---')
+    logger.debug(u'breaks 1: ' + unicode(breaksCount[0]))
+    logger.debug(u'breaks 2: ' + unicode(breaksCount[1]))
+    logger.debug(u'breaks 3: ' + unicode(breaksCount[2]))
+    logger.debug(u'breaks 4: ' + unicode(breaksCount[3]))
+    logger.debug(u'breaks 5: ' + unicode(breaksCount[4]))
+    logger.debug(u'breaks 6: ' + unicode(breaksCount[5]))
+    logger.debug(u'breaks 7: ' + unicode(breaksCount[6]))
+    logger.debug(u'breaks 8: ' + unicode(breaksCount[7]))
+    logger.debug(u'----')
+    logger.debug(u'max found: ' + unicode(breaksMax))
+    logger.debug(u'max Index: ' + unicode(breaksMaxIndex))
+    logger.debug(u'----')
+
+    if breaksMaxIndex > 0 and breaksCount[0] > breaksMax and averageLineLength < 90:
+        body = breaksRegexp[0].sub(r'\1 \n\3', body)
+
+    # Find all instances of consecutive breaks less than otr equal to the max count use most often
+    #  replase those tags to inverted p tag pairs, those with more connsecutive breaks are replaced them with a horisontal line
+    for i in range(len(breaksCount)):
+        # if i > 0 or breaksMaxIndex == 0:
+        if i <= breaksMaxIndex:
+            logger.debug(unicode(i) + u' <= breaksMaxIndex (' + unicode(breaksMaxIndex) + u')')
+            body = breaksRegexp[i].sub(r'\1</p>\n<p>\3', body)
+        elif i == breaksMaxIndex+1:
+            logger.debug(unicode(i) + u' == breaksMaxIndex+1 (' + unicode(breaksMaxIndex+1) + u')')
+            body = breaksRegexp[i].sub(r'\1</p>\n<p><br/></p>\n<p>\3', body)
+        else:
+            logger.debug(unicode(i) + u' > breaksMaxIndex+1 (' + unicode(breaksMaxIndex+1) + u')')
+            body = breaksRegexp[i].sub(r'\1</p>\n<hr />\n<p>\3', body)
+
+    body = breaksRegexp[8].sub(r'</p>\n<hr />\n<p>', body)
+
+    # Reverting the square brackets
+    body = body.replace(u'[', u'<')
+    body = body.replace(u']', u'>')
+    body = body.replace(u'&squareBracketStart;', u'[')
+    body = body.replace(u'&squareBracketEnd;', u']')
+
+    body = body.replace(u'{p}', u'<p>')
+    body = body.replace(u'{/p}', u'</p>')
+
+    # If for some reason, a third break makes its way inside the paragraph, preplace that with the empty paragraph for the additional linespaing.
+    body = re.sub(r'<p>\s*(<br\ \/>)+', r'<p><br /></p>\n<p>', body)
+
+    # change empty p tags to include a br to force spacing.
+    body = re.sub(r'<p>\s*</p>', r'<p><br/></p>', body)
+
+    # Clean up hr tags, and add inverted p tag pairs
+    body = re.sub(r'(<div[^>]+>)*\s*<hr\ \/>\s*(</div>)*', r'\n<hr />\n', body)
+
+    # Clean up hr tags, and add inverted p tag pairs
+    body = re.sub(r'\s*<hr\ \/>\s*', r'</p>\n<hr />\n<p>', body)
+
+    # Because the previous regexp may cause trouble if the hr tag already had a p tag pair around it, w nee dot repair that.
+    # Repeated opening p tags are condenced to one. As we added the extra leading opening p tags, we can safely assume that
+    #  the last in such a chain must be the original. Lets keep its attributes if they are there.
+    body = re.sub(r'\s*(<p[^>]*>\s*)+<p([^>]*)>\s*', r'\n<p\2>', body)
+    # Repeated closing p tags are condenced to one
+    body = re.sub(r'\s*(<\/\s*p>\s*){2,}', r'</p>\n', body)
+
+    # superflous cleaning, remove whitespaces traling opening p tags. These does affect formatting.
+    body = re.sub(r'\s*<p([^>]*)>\s*', r'\n<p\1>', body)
+    # superflous cleaning, remove whitespaces leading closing p tags. These does not affect formatting.
+    body = re.sub(r'\s*</p>\s*', r'</p>\n', body)
+
+    # Remove empty tag pairs
+    body = re.sub(r'\s*<(\S+)[^>]*>\s*</\1>', r'', body)
+
+    body = body.replace(u'{br /}', u'<br />')
+    body = body.strip()
+
+    # re-wrap in div tag.
+    body = u'<div>\n' + body + u'</div>\n'
+
+    # return body
+    return tag_sanitizer(body)
+
+def is_valid_block(block):
+    return unicode(block).find('<') == 0 and unicode(block).find('<!') != 0
+
+def soup_up_div(body):
+    blockTags = ['address', 'blockquote', 'del', 'div', 'dl', 'fieldset', 'form', 'ins', 'noscript', 'ol', 'p', 'pre', 'table', 'ul']
+    recurseTags = ['blockquote', 'div', 'noscript']
+
+    tag = body[:body.index('>')+1]
+    tagend = body[body.rindex('<'):]
+
+    body = body.replace(u'<br />', u'[br /]')
+
+    soup = bs.BeautifulSoup(body)
+
+    body = u''
+    lastElement = 1 # 1 = block, 2 = nested, 3 = invalid
+
+    for i in soup.contents[0]:
+        if unicode(i).strip().__len__() > 0:
+            s = unicode(i)
+            if  type(i) == bs.Tag:
+                if  i.name in blockTags:
+                    if lastElement > 1:
+                        body = body.strip(r'\s*(\[br\ \/\]\s*)*\s*')
+                        body += u'{/p}'
+
+                    lastElement = 1
+
+                    if i.name in recurseTags:
+                        s = soup_up_div(s)
+
+                    body += s.strip() + '\n'
+                else:
+                    if lastElement == 1:
+                        body = body.strip(r'\s*(\[br\ \/\]\s*)*\s*')
+                        body += u'{p}'
+
+                    lastElement = 2
+                    body += s
+            elif type(i) == bs.Comment:
+                body += s
+            else:
+                if lastElement == 1:
+                    body = body.strip(r'\s*(\[br\ \/\]\s*)*\s*')
+                    body += u'{p}'
+
+                lastElement = 3
+                body += s
+
+    if lastElement > 1:
+        body = body.strip(r'\s*(\[br\ \/\]\s*)*\s*')
+        body += u'{/p}'
+
+    body = body.replace(u'[br /]', u'<br />')
+
+    return tag + body + tagend
+
+
+def is_end_tag(tag):
+    return re.match(r'</([^\ >]+)>', tag) != None
+
+def is_comment_tag(tag):
+    return re.match(r'<\!\-\-([^>]+)>', tag) != None
+
+def is_closed_tag(tag):
+    return re.match(r'<(.+?)/>', tag) != None
+
+def tag_sanitizer(html):
+    blockTags = ['address', 'blockquote', 'del', 'div', 'dl', 'fieldset', 'form', 'ins', 'noscript', 'ol', 'pre', 'table', 'ul']
+
+    body = u''
+    tags = re.findall(r'(<[^>]+>)([^<]*)', html)
+
+    for rTag in tags:
+        name = stack.get_tag_name(rTag[0])
+        is_end = is_end_tag(rTag[0])
+        is_closed = is_closed_tag(rTag[0]) or is_comment_tag(rTag[0])
+
+        # is_comment = is_comment_tag(rTag[0])
+        # logger.debug(u'%s >  isEnd: %s >  isClosed: %s >  isComment: %s'%(name, unicode(is_end), unicode(is_closed), unicode(is_comment)))
+        # logger.debug(u'> %s%s\n'%(rTag[0], rTag[1]))
+
+        if name in blockTags:
+            body += rTag[0]
+            body += rTag[1]
+        elif name == u'p':
+            if is_end:
+                body += stack.spool_end()
+                body += rTag[0]
+                body += rTag[1]
+            elif is_closed:
+                body += rTag[0]
+                body += rTag[1]
+            else:
+                body += rTag[0]
+                body += stack.spool_start()
+                body += rTag[1]
+        else:
+            if is_end:
+                t = stack.get_last()
+                tn = stack.get_tag_name(t)
+                rTn = stack.get_tag_name(rTag[0])
+                if tn == rTn:
+                    body += rTag[0]
+                    stack.pop()
+            elif not is_closed:
+                stack.push(rTag[0])
+                body += rTag[0]
+            else:
+                body += rTag[0]
+
+            body += rTag[1]
+    stack.flush()
+    return body

fanficdownloader/story.py

@@ -30,7 +30,7 @@ from configurable import Configurable
 
 # Create convert_image method depending on which graphics lib we can
 # load.  Preferred: calibre, PIL, none
-        
+
 imagetypes = {
     'jpg':'image/jpeg',
     'jpeg':'image/jpeg',
@@ -48,14 +48,14 @@ try:
         export = False
         img = Image()
         img.load(data)
-        
+
         owidth, oheight = img.size
         nwidth, nheight = sizes
         scaled, nwidth, nheight = fit_image(owidth, oheight, nwidth, nheight)
         if scaled:
             img.size = (nwidth, nheight)
             export = True
-            
+
         if normalize_format_name(img.format) != imgtype:
             export = True
 
@@ -65,7 +65,7 @@ try:
             canvas.compose(img)
             img = canvas
             export = True
-            
+
         if grayscale and img.type != "GrayscaleType":
             img.type = "GrayscaleType"
             export = True
@@ -75,7 +75,7 @@ try:
         else:
             logger.debug("image used unchanged")
             return (data,imgtype,imagetypes[imgtype])
-        
+
 except:
 
     # No calibre routines, try for PIL for CLI.
@@ -87,14 +87,14 @@ except:
                           removetrans,imgtype="jpg",background='#ffffff'):
             export = False
             img = Image.open(StringIO(data))
-            
+
             owidth, oheight = img.size
             nwidth, nheight = sizes
             scaled, nwidth, nheight = fit_image(owidth, oheight, nwidth, nheight)
             if scaled:
                 img = img.resize((nwidth, nheight),Image.ANTIALIAS)
                 export = True
-                
+
             if normalize_format_name(img.format) != imgtype:
                 if img.mode == "P":
                     # convert pallete gifs to RGB so jpg save doesn't fail.
@@ -119,7 +119,7 @@ except:
             else:
                 logger.debug("image used unchanged")
                 return (data,imgtype,imagetypes[imgtype])
-        
+
     except:
         # No calibre or PIL, simple pass through with mimetype.
         def convert_image(url,data,sizes,grayscale,
@@ -129,16 +129,16 @@ except:
 ## also used for explicit no image processing.
 def no_convert_image(url,data):
     parsedUrl = up.urlparse(url)
-    
+
     ext=parsedUrl.path[parsedUrl.path.rfind('.')+1:].lower()
-    
+
     if ext not in imagetypes:
         logger.debug("no_convert_image url:%s - no known extension"%url)
         # doesn't have extension? use jpg.
         ext='jpg'
-        
+
     return (data,ext,imagetypes[ext])
-        
+
 def normalize_format_name(fmt):
     if fmt:
         fmt = fmt.lower()
@@ -222,7 +222,7 @@ langs = {
     }
 
 class Story(Configurable):
-    
+
     def __init__(self, configuration):
         Configurable.__init__(self, configuration)
         try:
@@ -234,7 +234,7 @@ class Story(Configurable):
         self.chapters = [] # chapters will be tuples of (title,html)
         self.imgurls = []
         self.imgtuples = []
-        
+
         self.cover=None # *href* of new cover image--need to create html.
         self.oldcover=None # (oldcoverhtmlhref,oldcoverhtmltype,oldcoverhtmldata,oldcoverimghref,oldcoverimgtype,oldcoverimgdata)
         self.calibrebookmark=None # cheesy way to carry calibre bookmark file forward across update.
@@ -251,7 +251,7 @@ class Story(Configurable):
                 self.addToList(metadata,val)
 
         self.setReplace(self.getConfig('replace_metadata'))
-        
+
     def setMetadata(self, key, value, condremoveentities=True):
         ## still keeps < < and &
         if condremoveentities:
@@ -297,9 +297,9 @@ class Story(Configurable):
                 # A way to explicitly include spaces in the
                 # replacement string.  The .ini parser eats any
                 # trailing spaces.
-                replacement=replacement.replace('\s',' ') 
+                replacement=replacement.replace('\s',' ')
                 self.replacements.append([metakeys,regexp,replacement,condkey,condregexp])
-    
+
     def doReplacements(self,value,key):
         for (metakeys,regexp,replacement,condkey,condregexp) in self.replacements:
             if (metakeys == None or key in metakeys) \
@@ -309,11 +309,11 @@ class Story(Configurable):
                 if condkey and condkey != key: # prevent infinite recursion.
                     condval = self.getMetadata(condkey)
                     doreplace = condval != None and condregexp.search(condval)
-                    
+
                 if doreplace:
                     value = regexp.sub(replacement,value)
         return value
-        
+
     def getMetadataRaw(self,key):
         if self.isValidMetaEntry(key) and self.metadata.has_key(key):
             return self.metadata[key]
@@ -326,7 +326,7 @@ class Story(Configurable):
             return value
 
         if self.isList(key):
-            join_string = self.getConfig("join_string_"+key,u", ").replace('\s',' ') 
+            join_string = self.getConfig("join_string_"+key,u", ").replace('\s',' ')
             value = join_string.join(self.getList(key, removeallentities, doreplacements=True))
             if doreplacements:
                 value = self.doReplacements(value,key+"_LIST")
@@ -351,7 +351,7 @@ class Story(Configurable):
                 return value
         else: #if self.getConfig("default_value_"+key):
             return self.getConfig("default_value_"+key)
-        
+
     def getAllMetadata(self,
                        removeallentities=False,
                        doreplacements=True,
@@ -360,13 +360,13 @@ class Story(Configurable):
         All single value *and* list value metadata as strings (unless keeplists=True, then keep lists).
         '''
         allmetadata = {}
-        
+
         # special handling for authors/authorUrls
         linkhtml="<a class='%slink' href='%s'>%s</a>"
         if self.isList('author'): # more than one author, assume multiple authorUrl too.
             htmllist=[]
             for i, v in enumerate(self.getList('author')):
-                aurl = self.getList('authorUrl')[i]                    
+                aurl = self.getList('authorUrl')[i]
                 auth = v
                 # make sure doreplacements & removeallentities are honored.
                 if doreplacements:
@@ -375,9 +375,9 @@ class Story(Configurable):
                 if removeallentities:
                     aurl=removeAllEntities(aurl)
                     auth=removeAllEntities(auth)
-                
+
                 htmllist.append(linkhtml%('author',aurl,auth))
-            join_string = self.getConfig("join_string_authorHTML",u", ").replace('\s',' ') 
+            join_string = self.getConfig("join_string_authorHTML",u", ").replace('\s',' ')
             self.setMetadata('authorHTML',join_string.join(htmllist))
         else:
             self.setMetadata('authorHTML',linkhtml%('author',self.getMetadata('authorUrl', removeallentities, doreplacements),
@@ -388,20 +388,43 @@ class Story(Configurable):
                                                     self.getMetadata('series', removeallentities, doreplacements)))
         elif self.getMetadataRaw('series') != None:
             self.setMetadata('seriesHTML',self.getMetadataRaw('series'))
-            
+
+        # logger.debug("make_linkhtml_entries:%s"%self.getConfig('make_linkhtml_entries'))
+        for k in self.getConfigList('make_linkhtml_entries'):
+            # Assuming list, because it has to be site specific and
+            # they are all lists.  Bail if kUrl list not the same
+            # length.
+            # logger.debug("\nk:%s\nlist:%s\nlistURL:%s"%(k,self.getList(k),self.getList(k+'Url')))
+            if len(self.getList(k+'Url')) != len(self.getList(k)):
+                continue
+            htmllist=[]
+            for i, v in enumerate(self.getList(k)):
+                url = self.getList(k+'Url')[i]
+                # make sure doreplacements & removeallentities are honored.
+                if doreplacements:
+                    url=self.doReplacements(url,k+'Url')
+                    v=self.doReplacements(v,k)
+                if removeallentities:
+                    url=removeAllEntities(url)
+                    v=removeAllEntities(v)
+
+                htmllist.append(linkhtml%(k,url,v))
+            join_string = self.getConfig("join_string_"+k+"HTML",u", ").replace('\s',' ')
+            self.setMetadata(k+'HTML',join_string.join(htmllist))
+
         for k in self.getValidMetaList():
             if self.isList(k) and keeplists:
                 allmetadata[k] = self.getList(k, removeallentities, doreplacements)
             else:
                 allmetadata[k] = self.getMetadata(k, removeallentities, doreplacements)
-                
+
         return allmetadata
 
     # just for less clutter in adapters.
     def extendList(self,listname,l):
         for v in l:
             self.addToList(listname,v.strip())
-    
+
     def addToList(self,listname,value):
         if value==None:
             return
@@ -421,24 +444,24 @@ class Story(Configurable):
         return self.hasConfig("include_in_"+listname) or \
             ( self.isValidMetaEntry(listname) and self.metadata.has_key(listname) \
                   and isinstance(self.metadata[listname],list) )
-    
+
     def getList(self,listname,
                 removeallentities=False,
                 doreplacements=True,
                 includelist=[]):
         #print("getList(%s,%s)"%(listname,includelist))
         retlist = []
-        
+
         if not self.isValidMetaEntry(listname):
             return retlist
-        
+
         # includelist prevents infinite recursion of include_in_'s
         if self.hasConfig("include_in_"+listname) and listname not in includelist:
             for k in self.getConfigList("include_in_"+listname):
                 retlist.extend(self.getList(k,removeallentities=False,
                                             doreplacements=doreplacements,includelist=includelist+[listname]))
         else:
-        
+
             if not self.isList(listname):
                 retlist = [self.getMetadata(listname,removeallentities=False,
                                             doreplacements=doreplacements)]
@@ -458,7 +481,7 @@ class Story(Configurable):
         # ships=>[ ]*(/|&amp;|&)[ ]*=>/
         if listname == 'ships' and self.getConfig('sort_ships'):
             retlist = [ '/'.join(sorted(x.split('/'))) for x in retlist ]
-                
+
         if retlist:
             if listname in ('author','authorUrl','authorId') or self.getConfig('keep_in_order_'+listname):
                 # need to retain order for author & authorUrl so the
@@ -473,9 +496,9 @@ class Story(Configurable):
     def getSubjectTags(self, removeallentities=False):
         # set to avoid duplicates subject tags.
         subjectset = set()
-        
+
         tags_list = self.getConfigList("include_subject_tags") + self.getConfigList("extra_subject_tags")
-            
+
         # metadata all go into dc:subject tags, but only if they are configured.
         for (name,value) in self.getAllMetadata(removeallentities=removeallentities,keeplists=True).iteritems():
             if name in tags_list:
@@ -491,7 +514,7 @@ class Story(Configurable):
             subjectset.remove('')
 
         return list(subjectset | set(self.getConfigList("extratags")))
-            
+
     def addChapter(self, url, title, html):
         if self.getConfig('strip_chapter_numbers') and \
                 self.getConfig('chapter_title_strip_pattern'):
@@ -512,7 +535,7 @@ class Story(Configurable):
                                 html) )
         else:
             retval = self.chapters
-            
+
         return retval
 
     def formatFileName(self,template,allowunsafefilename=True):
@@ -526,7 +549,7 @@ class Story(Configurable):
             pattern = re.compile(self.getConfig("output_filename_safepattern",r"[^a-zA-Z0-9_\. \[\]\(\)&'-]+"))
             for k in origvalues.keys():
                 values[k]=re.sub(pattern,'_', removeAllEntities(self.getMetadata(k)))
-    
+
         return string.Template(template).substitute(values).encode('utf8')
 
     # pass fetch in from adapter in case we need the cookies collected
@@ -537,7 +560,7 @@ class Story(Configurable):
         # isn't used anywhere.
         if cover and self.getConfig('never_make_cover'):
             return
-        
+
         url = url.strip() # ran across an image with a space in the
                           # src. Browser handled it, so we'd better, too.
 
@@ -545,12 +568,18 @@ class Story(Configurable):
         # gets too big too fast and breaks things.
         if is_appengine:
             return
-        
+
         if url.startswith("http") or url.startswith("file") or parenturl == None:
             imgurl = url
         else:
             parsedUrl = urlparse.urlparse(parenturl)
-            if url.startswith("/") :
+            if url.startswith("//") :
+                imgurl = urlparse.urlunparse(
+                    (parsedUrl.scheme,
+                     '',
+                     url,
+                     '','',''))
+            elif url.startswith("/") :
                 imgurl = urlparse.urlunparse(
                     (parsedUrl.scheme,
                      parsedUrl.netloc,
@@ -598,7 +627,7 @@ class Story(Configurable):
             except Exception, e:
                 logger.info("Failed to load or convert image, skipping:\n%s\nException: %s"%(imgurl,e))
                 return "failedtoload"
-            
+
             # explicit cover, make the first image.
             if cover and not self.getConfig('never_make_cover'):
                 if len(self.imgtuples) > 0 and 'cover' in self.imgtuples[0]['newsrc']:
@@ -624,19 +653,19 @@ class Story(Configurable):
                     self.cover=newsrc
                     self.imgtuples.append({'newsrc':newsrc,'mime':mime,'data':data})
                     self.imgurls.append(imgurl)
-            
+
                 newsrc = "images/%s-%s.%s"%(
                     prefix,
                     self.imgurls.index(imgurl),
                     ext)
                 self.imgtuples.append({'newsrc':newsrc,'mime':mime,'data':data})
-                
+
             #logger.debug("\nimgurl:%s\nnewsrc:%s\nimage size:%d\n"%(imgurl,newsrc,len(data)))
         else:
             newsrc = self.imgtuples[self.imgurls.index(imgurl)]['newsrc']
-            
+
         #print("===============\n%s\nimg url:%s\n============"%(newsrc,self.imgurls[-1]))
-        
+
         return newsrc
 
     def getImgUrls(self):
@@ -645,9 +674,9 @@ class Story(Configurable):
             #parsedUrl = urlparse.urlparse(url)
             retlist.append(self.imgtuples[i])
         return retlist
-    
+
     def __str__(self):
-        return "Metadata: " +str(self.metadata) 
+        return "Metadata: " +str(self.metadata)
 
 def commaGroups(s):
     groups = []

fanficdownloader/writers/writer_epub.py

@@ -361,7 +361,7 @@ div { margin: 0pt; padding: 0pt; }
 
         metadata.appendChild(newTag(contentdom,"dc:contributor",text="fanficdownloader [http://fanficdownloader.googlecode.com]",attrs={"opf:role":"bkp"}))
         metadata.appendChild(newTag(contentdom,"dc:rights",text=""))
-        if self.story.getMetadata('langcode') != None:
+        if self.story.getMetadata('langcode'):
             metadata.appendChild(newTag(contentdom,"dc:language",text=self.story.getMetadata('langcode')))
         else:
             metadata.appendChild(newTag(contentdom,"dc:language",text='en'))

index.html

@@ -46,6 +46,13 @@
       {{yourfile}}
       <!--			</div> -->
 
+          <h3>fanfiction.net / fimfiction.net</h3>
+          <p>
+            As of Jan 13, 2014, fanfiction.net &amp; fimfiction.net
+            are working again.  I'd ask that users limit the number of
+            stories they download from those sites, thanks.
+          </p>
+	  
       {% if authorized %}
       <form action="/fdown" method="post">
 	<div id='urlbox'>
@@ -57,18 +64,10 @@
 	  <h3>Changes:</h3>
 	  <p>
 	    <ul>
-              <li>New site: fictionpad.com</li>
+              <li>Add option to turn off author notes on AO3 chapters.</li>
+              <li>Fix for fictionpress.com https urls.</li>
 	    </ul>
 	  </p>
-          
-          <h3>fanfiction.net</h3>
-          <p>
-            Fanfiction.net appears to be blocking access from Google
-            App Engine, which prevents this web service.  There's
-            nothing I can do about it.  At the time of writing, the
-            latest CLI and calibre plugin versions worked.
-          </p>
-	  
 	  <p>
 	    Questions?  Check out our 
 	    <a href="http://code.google.com/p/fanficdownloader/wiki/FanFictionDownloaderFAQs">FAQs</a>.
@@ -77,7 +76,7 @@
 	    If you have any problems with this application, please
 	    report them in
 	    the <a href="http://groups.google.com/group/fanfic-downloader">FanFictionDownLoader Google Group</a>.  The
-	    <a href="http://4-4-75.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
+	    <a href="http://4-4-95.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
 	  </p>
 	  <div id='error'>
 	    {{ error_message }}

makeplugin.py

@@ -24,14 +24,14 @@ from makezip import createZipFile
 
 if __name__=="__main__":
     filename="FanFictionDownLoader.zip"
-    exclude=['*.pyc','*~','*.xcf','*[0-9].png']
+    exclude=['*.pyc','*~','*.xcf','*[0-9].png','*.po','*.pot','*default.mo']
     # from top dir. 'w' for overwrite
     createZipFile(filename,"w",
                   ['plugin-defaults.ini','plugin-example.ini','fanficdownloader','downloader.py','defaults.ini'],
                   exclude=exclude)
     #from calibre-plugin dir. 'a' for append
     os.chdir('calibre-plugin')
-    files=['about.txt','images',]
+    files=['about.txt','images','translations']
     files.extend(glob('*.py'))
     files.extend(glob('plugin-import-name-*.txt'))
     createZipFile("../"+filename,"a",

plugin-defaults.ini

@@ -154,6 +154,11 @@ extratags: FanFiction
 ## useful if pulling large numbers of stories or if the site is slow.
 #slow_down_sleep_time:0.5
 
+## How long to wait for each HTTP connection to finish.  Longer times
+## are better for sites that are slow to respond.  Shorter times
+## prevent excessive wait when your network or the site is down.
+connect_timeout:60.0
+
 ## Use regular expressions to find and replace (or remove) metadata.
 ## For example, you could change Sci-Fi=>SF, remove *-Centered tags,
 ## etc.  See http://docs.python.org/library/re.html (look for re.sub)
@@ -187,6 +192,11 @@ extratags: FanFiction
 ## doesn't work on some devices either.)
 #replace_hr: false
 
+## Some sites/authors/stories use br tags instead of p tags for
+## paragraphs.  This feature uses some heuristics to find and replace
+## br paragraphs with p tags while preserving scene breaks.
+#replace_br_with_p: false
+
 ## If you have the Generate Cover plugin installed, you can use the
 ## generate_cover_settings parameter to intelligently decide which GC
 ## setting to run.  There are three parts 1) a template of which
@@ -437,7 +447,8 @@ image_max_size: 580, 725
 ## confused and displays it on every page after that under the text
 ## for the rest of the chapter.  I doubt adding a div around the img
 ## will break any other readers, but in case it does, the fix can be
-## turned off.
+## turned off.  This setting is not used if replace_br_with_p is
+## true--replace_br_with_p also fixes the problem.
 nook_img_fix:true
 
 [mobi]
@@ -501,6 +512,13 @@ extratags: FanFiction,Testing,HTML
 ## personal.ini, not defaults.ini.
 #is_adult:true
 
+## archiveofourown.org stories allow chapters to be added out of
+## order.  So the newest chapter may not be the last one.  FFDL update
+## doesn't like that.  If do_update_hook is uncommented and set true,
+## the adapter will discard all existing chapters from the newest one
+## on when updating to enforce accurate chapters.
+#do_update_hook:false
+
 ## AO3 adapter defines a few extra metadata entries.
 ## If there's ever more than 4 series, add series04,series04Url etc.
 extra_valid_entries:fandoms,freeformtags,freefromtags,ao3categories,comments,kudos,hits,bookmarks,collections,series00,series01,series02,series03,series00Url,series01Url,series02Url,series03Url
@@ -524,12 +542,13 @@ include_in_freefromtags:freeformtags
 ## adds to include_subject_tags instead of replacing it.
 #extra_subject_tags:fandoms,freeformtags,ao3categories
 
-[ashwinder.sycophanthex.com]
-## Site dedicated to these categories/characters/ships
-extracategories:Harry Potter
-extracharacters:Severus Snape,Hermione Granger
-extraships:Severus Snape/Hermione Granger
+## AO3 chapters can include several different types of notes.  We've
+## traditional included them all in the chapter text, but this allows
+## you to customize which you include.  Copy this parameter to your
+## personal.ini and list the ones you don't want.
+#exclude_notes:authorheadnotes,chaptersummary,chapterheadnotes,chapterfootnotes,authorfootnotes
 
+[ashwinder.sycophanthex.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -537,6 +556,11 @@ extraships:Severus Snape/Hermione Granger
 #username:YourName
 #password:yourpassword
 
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+extracharacters:Severus Snape,Hermione Granger
+extraships:Severus Snape/Hermione Granger
+
 [asr3.slashzone.org]
 ## Site dedicated to these categories/characters/ships
 extracategories:The Sentinel
@@ -547,9 +571,6 @@ extracategories:The Sentinel
 #is_adult:true
 
 [bloodties-fans.com]
-## Site dedicated to these categories/characters/ships
-extracategories:Blood Ties
-
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -562,6 +583,9 @@ extracategories:Blood Ties
 ## personal.ini, not defaults.ini.
 #is_adult:true
 
+## Site dedicated to these categories/characters/ships
+extracategories:Blood Ties
+
 [buffynfaith.net]
 ## Site dedicated to these categories/characters/ships
 extracategories:Buffy: The Vampire Slayer
@@ -572,9 +596,6 @@ extracategories:Buffy: The Vampire Slayer
 #is_adult:true
 
 [castlefans.org]
-## Site dedicated to these categories/characters/ships
-extracategories:Castle
-
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -587,6 +608,9 @@ extracategories:Castle
 ## personal.ini, not defaults.ini.
 #is_adult:true
 
+## Site dedicated to these categories/characters/ships
+extracategories:Castle
+
 [chaos.sycophanthex.com]
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
@@ -599,10 +623,23 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 
 [dark-solace.org]
 ## Site dedicated to these categories/characters/ships
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
 extracategories:Buffy: The Vampire Slayer
 extracharacters:Buffy, Spike
 extraships:Spike/Buffy
 
+[dramione.org]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -615,24 +652,11 @@ extraships:Spike/Buffy
 ## personal.ini, not defaults.ini.
 #is_adult:true
 
-[dramione.org]
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter
 extracharacters:Draco Malfoy,Hermione Granger
 extraships:Draco Malfoy/Hermione Granger
 
-## Some sites require login (or login for some rated stories) The
-## program can prompt you, or you can save it in config.  In
-## commandline version, this should go in your personal.ini, not
-## defaults.ini.
-#username:YourName
-#password:yourpassword
-
-## Some sites also require the user to confirm they are adult for
-## adult content.  In commandline version, this should go in your
-## personal.ini, not defaults.ini.
-#is_adult:true
-
 ## some sites include images that we don't ever want becoming the
 ## cover image.  This lets you exclude them.
 cover_exclusion_regexp:/images/.*?ribbon.gif
@@ -678,35 +702,34 @@ cliches_label:Character Cliches
 ## 'mode'.  'r' to Replace any existing values, 'a' to Add to existing
 ## value (use with tag-like columns), and 'n' for setting on New books
 ## only.  (Default is 'r'.)
+## Literal strings can be set into custom columns using double quotes.
 ## Each metadata=>column mapping must be on a separate line and each
 ## needs to have one space at the start of each line.
 
 #custom_columns_settings:
-# cliches=>#acolumn,r
+# cliches=>#acolumn
 # themes=>#bcolumn,a
 # timeline=>#ccolumn,n
+# "FanFiction"=>#collection
 
 [efiction.esteliel.de]
 ## Site dedicated to these categories/characters/ships
 extracategories:Lord of the Rings
 
 [erosnsappho.sycophanthex.com]
-## Site dedicated to these categories/characters/ships
-extracategories:Harry Potter
-
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
 ## some sites include images that we don't ever want becoming the
 ## cover image.  This lets you exclude them.
 cover_exclusion_regexp:/images/.*?ribbon.gif
 
 [fanfiction.mugglenet.com]
-## Site dedicated to these categories/characters/ships
-extracategories:Harry Potter
-
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
@@ -719,10 +742,10 @@ extracategories:Harry Potter
 #username:YourName
 #password:yourpassword
 
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
 [fanfic.potterheadsanonymous.com]
-## Site dedicated to these categories/characters/ships
-extracategories:Harry Potter
-
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
@@ -735,16 +758,19 @@ extracategories:Harry Potter
 #username:YourName
 #password:yourpassword
 
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
 [fanfiction.portkey.org]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter
 extraships:Harry Potter/Hermione Granger
 
-## Some sites do not require a login, but do require the user to
-## confirm they are adult for adult content.  In commandline version,
-## this should go in your personal.ini, not defaults.ini.
-#is_adult:true
-
 [fanfiction.tenhawkpresents.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -778,11 +804,37 @@ dislikes_label:Dislikes
 #username:YourName
 #password:yourpassword
 
-[grangerenchanted.com]
-## Site dedicated to these categories/characters/ships
-extracategories:Harry Potter
-extracharacters:Hermione Granger
+[storiesonline.net]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
 
+## Clear FanFiction from defaults, site is original fiction.
+extratags:
+
+extra_valid_entries:size,universe,universeUrl,universeHTML,codes,notice
+#extra_titlepage_entries:size,universeHTML,codes,notice
+
+size_label:Size
+universe_label:Universe
+universeUrl_label:Universe URL
+universeHTML_label:Universe
+codes_label:Codes
+notice_label:Notice
+
+## Assume entryUrl, apply to "<a class='%slink' href='%s'>%s</a>" to
+## make entryHTML.
+make_linkhtml_entries:universe
+
+## storiesonline.net stories can be in a series or a universe, but not
+## both.  By default, universe will be populated in 'series' with
+## index=0
+universe_as_series: true
+
+[grangerenchanted.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -795,6 +847,10 @@ extracharacters:Hermione Granger
 ## personal.ini, not defaults.ini.
 #is_adult:true
 
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+extracharacters:Hermione Granger
+
 ## Extra metadata that this adapter knows about.  See [dramione.org]
 ## for examples of how to use them.
 extra_valid_entries:read,reviews
@@ -831,18 +887,15 @@ extracharacters:Kirk,Spock
 extraships:Kirk/Spock
 
 [lumos.sycophanthex.com]
-## Site dedicated to these categories/characters/ships
-extracategories:Harry Potter
-
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
-[merlinfic.dtwins.co.uk]
 ## Site dedicated to these categories/characters/ships
-extracategories:Merlin
+extracategories:Harry Potter
 
+[merlinfic.dtwins.co.uk]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -855,6 +908,9 @@ extracategories:Merlin
 ## personal.ini, not defaults.ini.
 #is_adult:true
 
+## Site dedicated to these categories/characters/ships
+extracategories:Merlin
+
 [national-library.net]
 ## Site dedicated to these categories/characters/ships
 extracategories:West Wing
@@ -866,24 +922,32 @@ extracategories:NCIS
 [netraptor.org]
 
 [nfacommunity.com]
-## Site dedicated to these categories/characters/ships
-extracategories:NCIS
-
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+## Site dedicated to these categories/characters/ships
+extracategories:NCIS
+
 [nha.magical-worlds.us]
 ## Site dedicated to these categories/characters/ships
 extracategories:Buffy: The Vampire Slayer
 extracharacters:Willow
 
 [occlumency.sycophanthex.com]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter
 extracharacters:Severus Snape
 
+[onedirectionfanfiction.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -891,10 +955,15 @@ extracharacters:Severus Snape
 #username:YourName
 #password:yourpassword
 
-[onedirectionfanfiction.com]
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
 ## Site dedicated to these categories/characters/ships
 extracategories:One Direction
 
+[pommedesang.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -907,40 +976,27 @@ extracategories:One Direction
 ## personal.ini, not defaults.ini.
 #is_adult:true
 
-[pommedesang.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:Anita Blake Vampire Hunter 
 
-## Some sites require login (or login for some rated stories) The
-## program can prompt you, or you can save it in config.  In
-## commandline version, this should go in your personal.ini, not
-## defaults.ini.
-#username:YourName
-#password:yourpassword
-
-## Some sites also require the user to confirm they are adult for
-## adult content.  In commandline version, this should go in your
-## personal.ini, not defaults.ini.
+[ponyfictionarchive.net]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
-[ponyfictionarchive.net]
 ## Site dedicated to these categories/characters/ships
 extracategories:My Little Pony: Friendship is Magic
 
+[pretendercentre.com]
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
-[pretendercentre.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:The Pretender
 
-## Some sites do not require a login, but do require the user to
-## confirm they are adult for adult content.  In commandline version,
-## this should go in your personal.ini, not defaults.ini.
-#is_adult:true
-
 [samdean.archive.nu]
 ## Site dedicated to these categories/characters/ships
 extracategories:Supernatural
@@ -948,9 +1004,6 @@ extracharacters:Sam,Dean
 extraships:Sam/Dean
 
 [scarhead.net]
-## Site dedicated to these categories/characters/ships
-extracategories:Harry Potter
-
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -963,6 +1016,9 @@ extracategories:Harry Potter
 ## personal.ini, not defaults.ini.
 #is_adult:true
 
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
 [sg1-heliopolis.com]
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
@@ -993,9 +1049,6 @@ extracharacters:Draco Malfoy,Harry Potter
 extraships:Harry Potter/Draco Malfoy
 
 [thehookupzone.net]
-## Site dedicated to these categories/characters/ships
-extracategories:Criminal Minds
-
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -1008,6 +1061,9 @@ extracategories:Criminal Minds
 ## personal.ini, not defaults.ini.
 #is_adult:true
 
+## Site dedicated to these categories/characters/ships
+extracategories:Criminal Minds
+
 [themasque.net]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -1022,9 +1078,6 @@ extracategories:Criminal Minds
 #is_adult:true
 
 [thequidditchpitch.org]
-## Site dedicated to these categories/characters/ships
-extracategories:Harry Potter
-
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -1037,50 +1090,71 @@ extracategories:Harry Potter
 ## personal.ini, not defaults.ini.
 #is_adult:true
 
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
 [tokra.fandomnet.com]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
 ## Site dedicated to these categories/characters/ships
 extracategories:Stargate: SG-1
 
-## Some sites do not require a login, but do require the user to
-## confirm they are adult for adult content.  In commandline version,
-## this should go in your personal.ini, not defaults.ini.
+[trekiverse.org]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
 #is_adult:true
 
-[www.adastrafanfic.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:Star Trek
 
+extra_valid_entries:awards
+awards_label:Awards
+
+cover_exclusion_regexp:art/.*Awards.jpg
+
+[www.adastrafanfic.com]
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+## Site dedicated to these categories/characters/ships
+extracategories:Star Trek
+
 [www.dracoandginny.com]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Some sites also require the user to confirm they are adult for
+## adult content.  In commandline version, this should go in your
+## personal.ini, not defaults.ini.
+#is_adult:true
+
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter
 extracharacters:Draco Malfoy,Ginny Weasley
 extraships:Draco Malfoy/Ginny Weasley
 
-## Some sites require login (or login for some rated stories) The
-## program can prompt you, or you can save it in config.  In
-## commandline version, this should go in your personal.ini, not
-## defaults.ini.
-#username:YourName
-#password:yourpassword
-
-## Some sites also require the user to confirm they are adult for
-## adult content.  In commandline version, this should go in your
-## personal.ini, not defaults.ini.
-#is_adult:true
-
 [www.thealphagate.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:Stargate: SG-1
 
 [www.checkmated.com]
-## Site dedicated to these categories/characters/ships
-extracategories:Harry Potter
-
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -1088,6 +1162,9 @@ extracategories:Harry Potter
 #username:YourName
 #password:yourpassword
 
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
 [www.destinysgateway.com]
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
@@ -1095,11 +1172,6 @@ extracategories:Harry Potter
 #is_adult:true
 
 [www.dokuga.com]
-## Site dedicated to these categories/characters/ships
-extracategories:InuYasha
-extracharacters:Sesshoumaru,Kagome
-extraships:Sesshoumaru/Kagome
-
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -1107,6 +1179,11 @@ extraships:Sesshoumaru/Kagome
 #username:YourName
 #password:yourpassword
 
+## Site dedicated to these categories/characters/ships
+extracategories:InuYasha
+extracharacters:Sesshoumaru,Kagome
+extraships:Sesshoumaru/Kagome
+
 [www.dotmoon.net]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -1136,10 +1213,42 @@ user_agent:
 ## Change this to false to use them anyway.
 never_make_cover: true
 
+## fanfiction.net is blocking people more aggressively.  If you
+## download fewer stories less often you can likely get by with
+## reducing this sleep.
+slow_down_sleep_time:4
+
+## ffnet is sensitive to too many hits.  Users are sensitive to long
+## waits during the initial metadata collection in the foreground.
+## When used, these settings will speed up metadata downloads in the
+## foreground linearly.
+tweak_fg_sleep:true
+min_fg_sleep:1.0
+max_fg_sleep:4.0
+max_fg_sleep_at_downloads: 10
+
 ## Extra metadata that this adapter knows about.  See [dramione.org]
 ## for examples of how to use them.
 extra_valid_entries:reviews,favs,follows
 
+## ffnet uses 'Pairings', not 'Relationship', stating they don't have
+## to be romantic pairings.
+ships_label:Pairings
+
+## Date formats used by FFDL.  Published and Update don't usually have
+## time, but they do now on ffnet.
+## See http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+## Note that ini format requires % to be escaped as %%.
+#dateCreated_format:%%Y-%%m-%%d %%H:%%M:%%S
+datePublished_format:%%Y-%%m-%%d %%H:%%M:%%S
+dateUpdated_format:%%Y-%%m-%%d %%H:%%M:%%S
+
+## ffnet used to have a tendency to send out update notices in email
+## before all their servers were showing the update on the first
+## chapter.  It generates another server request and doesn't seem to
+## be needed lately, so now default it to off.
+check_next_chapter:false
+
 [www.fanfiktion.de]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -1151,14 +1260,14 @@ extra_valid_entries:reviews,favs,follows
 [www.ficbook.net]
 
 [www.fictionalley.org]
-## Site dedicated to these categories/characters/ships
-extracategories:Harry Potter
-
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
 ## fictionalley.org doesn't have a status metadatum.  If uncommented,
 ## this will be used for status.
 #default_value_status:Unknown
@@ -1181,19 +1290,6 @@ extra_valid_entries:reviews,favs,follows
 #password:yourpassword
 
 [www.fimfiction.net]
-## Site dedicated to these categories/characters/ships
-extracategories:My Little Pony: Friendship is Magic
-
-## Extra metadata that this adapter knows about.  See [dramione.org]
-## for examples of how to use them.
-extra_valid_entries:likes,dislikes,views,total_views,short_description,groups
-likes_label:Likes
-dislikes_label:Dislikes
-views_label:Highest Single Chapter Views
-total_views_label:Total Views
-short_description_label:Short Summary
-groups_label:Groups
-
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
@@ -1211,38 +1307,63 @@ groups_label:Groups
 ## when updating to enforce accurate chapters.
 #do_update_hook:false
 
-[www.harrypotterfanfiction.com]
 ## Site dedicated to these categories/characters/ships
-extracategories:Harry Potter
+extracategories:My Little Pony: Friendship is Magic
 
+## Extra metadata that this adapter knows about.  See [dramione.org]
+## for examples of how to use them.
+extra_valid_entries:likes,dislikes,views,total_views,short_description,groups,groupsUrl,groupsHTML,prequel,prequelUrl,prequelHTML,sequels,sequelsUrl,sequelsHTML
+likes_label:Likes
+dislikes_label:Dislikes
+views_label:Highest Single Chapter Views
+total_views_label:Total Views
+short_description_label:Short Summary
+groups_label:Groups
+groupsUrl_label:Groups URLs
+groupsHTML_label:Groups
+prequel_label:Prequel
+prequelUrl_label:Prequel URL
+prequelHTML_label:Prequel
+sequels_label:Sequels
+sequelsUrl_label:Sequel URLs
+sequelsHTML_label:Sequels
+
+keep_in_order_sequels:true
+keep_in_order_sequelsUrl:true
+keep_in_order_groups:true
+keep_in_order_groupsUrl:true
+
+## Assume entryUrl, apply to "<a class='%slink' href='%s'>%s</a>" to
+## make entryHTML.
+make_linkhtml_entries:prequel,sequels,groups
+
+[www.harrypotterfanfiction.com]
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
 [www.henneth-annun.net]
 ## Site dedicated to these categories/characters/ships
 extracategories:The Hobbit
 
 [www.hpfandom.net]
-## Site dedicated to these categories/characters/ships
-extracategories:Harry Potter
-
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+
 [www.hpfanficarchive.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter
 
 [www.ik-eternal.net]
-## Site dedicated to these categories/characters/ships
-extracategories:InuYasha
-extracharacters:InuYasha,Kagome
-extraships:InuYasha/Kagome
-
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -1255,30 +1376,35 @@ extraships:InuYasha/Kagome
 ## personal.ini, not defaults.ini.
 #is_adult:true
 
+## Site dedicated to these categories/characters/ships
+extracategories:InuYasha
+extracharacters:InuYasha,Kagome
+extraships:InuYasha/Kagome
+
 [www.jlaunlimited.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:JLA
 
 [www.libraryofmoria.com]
-## Site dedicated to these categories/characters/ships
-extracategories:Lord of the Rings
-
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+## Site dedicated to these categories/characters/ships
+extracategories:Lord of the Rings
+
 [www.mediaminer.org]
 
 [www.midnightwhispers.ca]
-## Site dedicated to these categories/characters/ships
-extracategories:Queer as Folk
-
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+## Site dedicated to these categories/characters/ships
+extracategories:Queer as Folk
+
 ## some sites include images that we don't ever want becoming the
 ## cover image.  This lets you exclude them.
 cover_exclusion_regexp:/stories/999/images/.*?_trophy.png
@@ -1293,10 +1419,6 @@ extracategories:CSI
 extraships:Nick Stokes/Greg Sanders
 
 [www.phoenixsong.net]
-## Site dedicated to these categories/characters/ships
-extracategories:Harry Potter
-extraships:Harry Potter/Ginny Weasley
-
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -1307,9 +1429,13 @@ extraships:Harry Potter/Ginny Weasley
 ## phoenixsong.net, oddly, can have high rated chapters (login
 ## required) in the middle of a lower rated story.  Use this to force
 ## FFDL to always login to phoenixsong.net so those stories download
-## correctly.
+## correctly.  If you have a login, this is recommended.
 #force_login:true
 
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+extraships:Harry Potter/Ginny Weasley
+
 [www.potionsandsnitches.net]
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter
@@ -1332,17 +1458,15 @@ extracategories:Psych
 #is_adult:true
 
 [www.qaf-fic.com]
-## Site dedicated to these categories/characters/ships
-extracategories:Queer as Folk
-
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+## Site dedicated to these categories/characters/ships
+extracategories:Queer as Folk
+
 [www.restrictedsection.org]
-extracategories:Harry Potter
-extragenres:Erotica
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -1350,20 +1474,21 @@ extragenres:Erotica
 #username:YourName
 #password:yourpassword
 
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+extragenres:Erotica
+
 [www.scarvesandcoffee.net]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
 ## Site dedicated to these categories/characters/ships
 extracategories:Glee
 extracharacters:Kurt Hummel,Blaine Anderson
 
-## Some sites do not require a login, but do require the user to
-## confirm they are adult for adult content.  In commandline version,
-## this should go in your personal.ini, not defaults.ini.
-#is_adult:true
-
 [www.simplyundeniable.com]
-## Site dedicated to these categories/characters/ships
-extracategories:Harry Potter
-
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -1371,15 +1496,18 @@ extracategories:Harry Potter
 #username:YourName
 #password:yourpassword
 
-[www.sinful-desire.org]
 ## Site dedicated to these categories/characters/ships
-extracategories:Supernatural
+extracategories:Harry Potter
 
+[www.sinful-desire.org]
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+## Site dedicated to these categories/characters/ships
+extracategories:Supernatural
+
 [www.siye.co.uk]
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter
@@ -1387,25 +1515,26 @@ extracharacters:Harry Potter,Ginny Weasley
 extraships:Harry Potter/Ginny Weasley
 
 [www.squidge.org/peja]
-# www.squidge.org/peja calls it Fandom <shrug>
+## www.squidge.org/peja calls it Fandom <shrug>
 category_label:Fandom
-# Remove numWords -- www.squidge.org/peja word counts are inaccurate
+
+## Remove numWords -- www.squidge.org/peja word counts are inaccurate
 titlepage_entries: seriesHTML,category,genre,language,characters,ships,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,site,description
 
 [www.squidge.org/peja:txt]
-## Add URLs since there aren't links.
-# Remove numWords -- www.squidge.org/peja word counts are inaccurate
+## Add URLs since there aren't links and remove numWords --
+## www.squidge.org/peja word counts are inaccurate
 titlepage_entries: series,seriesUrl,category,genre,language,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,site,storyUrl, authorUrl, description
 
 [www.storiesofarda.com]
-## Site dedicated to these categories/characters/ships
-extracategories:Lord of the Rings
-
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+## Site dedicated to these categories/characters/ships
+extracategories:Lord of the Rings
+
 [www.thepetulantpoetess.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -1448,9 +1577,6 @@ collect_series: false
 extracategories:Twilight
 
 [www.twilighted.net]
-## Site dedicated to these categories/characters/ships
-extracategories:Twilight
-
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -1458,13 +1584,13 @@ extracategories:Twilight
 #username:YourName
 #password:yourpassword
 
+## Site dedicated to these categories/characters/ships
+extracategories:Twilight
+
 ## twilighted.net (ab)uses series as personal reading lists.
 collect_series: false
 
 [www.twiwrite.net]
-## Site dedicated to these categories/characters/ships
-extracategories:Twilight
-
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
 ## commandline version, this should go in your personal.ini, not
@@ -1472,20 +1598,23 @@ extracategories:Twilight
 #username:YourName
 #password:yourpassword
 
+## Site dedicated to these categories/characters/ships
+extracategories:Twilight
+
 ## twiwrite.net (ab)uses series as personal reading lists.
 collect_series: false
 
 [www.walkingtheplank.org]
-## Site dedicated to these categories/characters/ships
-extracategories:Harry Potter
-extracharacters:Severus Snape,Harry Potter
-extraships:Severus Snape/Harry Potter
-
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+## Site dedicated to these categories/characters/ships
+extracategories:Harry Potter
+extracharacters:Severus Snape,Harry Potter
+extraships:Severus Snape/Harry Potter
+
 [www.whofic.com]
 
 [www.wizardtales.net]
@@ -1507,14 +1636,14 @@ extracategories:X-Men Movie
 extracharacters:Wolverine,Rogue
 
 [www.wraithbait.com]
-## Site dedicated to these categories/characters/ships
-extracategories:Stargate: Atlantis
-
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+## Site dedicated to these categories/characters/ships
+extracategories:Stargate: Atlantis
+
 [overrides]
 ## It may sometimes be useful to override all of the specific format,
 ## site and site:format sections in your private configuration.  For

status.html

@@ -51,12 +51,11 @@
 	      by {{ fic.author }} ({{ fic.format }})
 	  {% endif %}
 	  {% if fic.failure %}
-          <h3>fanfiction.net</h3>
+          <h3>fanfiction.net / fimfiction.net</h3>
           <p>
-            FYI, fanfiction.net appears to be blocking access from Google
-            App Engine, which prevents this web service.  There's
-            nothing I can do about it.  At the time of writing, the
-            latest CLI and calibre plugin versions worked.
+            As of Jan 13, 2014, fanfiction.net &amp; fimfiction.net
+            are working again.  I'd ask that users limit the number of
+            stories they download from those sites, thanks.
           </p>
 	  <span id='error'>{{ fic.failure }}</span>
 	  {% endif %}