Update CLI download zip.

Also don't retry 404 errors and bump versions.

app.yaml

@@ -1,6 +1,6 @@
 # ffd-retief-hrd fanfictiondownloader
 application: fanfictiondownloader
-version: 4-4-81
+version: 4-5-04
 runtime: python27
 api_version: 1
 threadsafe: true

calibre-plugin/__init__.py

@@ -17,6 +17,12 @@ if sys.version_info >= (2, 7):
     loghandler.setLevel(logging.DEBUG)
     logger.setLevel(logging.DEBUG)
 
+# pulls in translation files for _() strings
+try:
+    load_translations()
+except NameError:
+    pass # load_translations() added in calibre 1.9
+
 # The class that all Interface Action plugin wrappers must inherit from
 from calibre.customize import InterfaceActionBase
 
@@ -33,11 +39,11 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
     calibre utilities to run without needing to load the GUI libraries.
     '''
     name                = 'FanFictionDownLoader'
-    description         = 'UI plugin to download FanFiction stories from various sites.'
+    description         = _('UI plugin to download FanFiction stories from various sites.')
     supported_platforms = ['windows', 'osx', 'linux']
     author              = 'Jim Miller'
-    version             = (1, 7, 52)
-    minimum_calibre_version = (0, 8, 57)
+    version             = (1, 8, 23)
+    minimum_calibre_version = (1, 13, 0)
 
     #: This field defines the GUI plugin class that contains all the code
     #: that actually does something. Its format is module_path:class_name

calibre-plugin/config.py

@@ -22,13 +22,40 @@ from calibre.gui2.ui import get_gui
 from calibre.gui2 import dynamic, info_dialog
 from calibre.constants import numeric_version as calibre_version
 
+# pulls in translation files for _() strings
+try:
+    load_translations()
+except NameError:
+    pass # load_translations() added in calibre 1.9
+
+# There are a number of things used several times that shouldn't be
+# translated.  This is just a way to make that easier by keeping them
+# out of the _() strings.
+# I'm tempted to override _() to include them...
+no_trans = { 'pini':'personal.ini',
+             'imgset':'\n\n[epub]\ninclude_images:true\nkeep_summary_html:true\nmake_firstimage_cover:true\n\n',
+             'gcset':'generate_cover_settings',
+             'ccset':'custom_columns_settings',
+             'gc':'Generate Cover',
+             'rl':'Reading List',
+             'cp':'Count Pages',
+             'cmplt':'Completed',
+             'inprog':'In-Progress',
+             'lul':'Last Updated',
+             'lus':'lastupdate',
+             'is':'include_subject',
+             'isa':'is_adult',
+             'u':'username',
+             'p':'password',
+             }
+
 from calibre_plugins.fanfictiondownloader_plugin.prefs import prefs, PREFS_NAMESPACE
 from calibre_plugins.fanfictiondownloader_plugin.dialogs \
-    import (UPDATE, UPDATEALWAYS, OVERWRITE, collision_order, RejectListDialog,
+    import (UPDATE, UPDATEALWAYS, collision_order, save_collisions, RejectListDialog,
             EditTextDialog, RejectUrlEntry)
     
 from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.adapters \
-    import (getConfigSections, getNormalStoryURL)
+    import getConfigSections
 
 from calibre_plugins.fanfictiondownloader_plugin.common_utils \
     import ( KeyboardConfigDialog, PrefsViewerDialog )
@@ -125,7 +152,7 @@ class ConfigWidget(QWidget):
         self.l = QVBoxLayout()
         self.setLayout(self.l)
 
-        label = QLabel('<a href="http://code.google.com/p/fanficdownloader/wiki/FanFictionDownloaderSupportedsites">List of Supported Sites</a> -- <a href="http://code.google.com/p/fanficdownloader/wiki/FanFictionDownloaderFAQs">FAQs</a>')
+        label = QLabel('<a href="http://code.google.com/p/fanficdownloader/wiki/FanFictionDownloaderSupportedsites">'+_('List of Supported Sites')+'</a> -- <a href="http://code.google.com/p/fanficdownloader/wiki/FanFictionDownloaderFAQs">'+_('FAQs')+'</a>')
         label.setOpenExternalLinks(True)
         self.l.addWidget(label)
 
@@ -139,7 +166,7 @@ class ConfigWidget(QWidget):
         self.scroll_area.setWidget(tab_widget)
         
         self.basic_tab = BasicTab(self, plugin_action)
-        tab_widget.addTab(self.basic_tab, 'Basic')
+        tab_widget.addTab(self.basic_tab, _('Basic'))
 
         self.personalini_tab = PersonalIniTab(self, plugin_action)
         tab_widget.addTab(self.personalini_tab, 'personal.ini')
@@ -160,20 +187,20 @@ class ConfigWidget(QWidget):
             self.countpages_tab.setEnabled(False)
 
         self.std_columns_tab = StandardColumnsTab(self, plugin_action)
-        tab_widget.addTab(self.std_columns_tab, 'Standard Columns')
+        tab_widget.addTab(self.std_columns_tab, _('Standard Columns'))
 
         self.cust_columns_tab = CustomColumnsTab(self, plugin_action)
-        tab_widget.addTab(self.cust_columns_tab, 'Custom Columns')
+        tab_widget.addTab(self.cust_columns_tab, _('Custom Columns'))
 
         self.other_tab = OtherTab(self, plugin_action)
-        tab_widget.addTab(self.other_tab, 'Other')
+        tab_widget.addTab(self.other_tab, _('Other'))
 
 
     def save_settings(self):
 
         # basic
         prefs['fileform'] = unicode(self.basic_tab.fileform.currentText())
-        prefs['collision'] = unicode(self.basic_tab.collision.currentText())
+        prefs['collision'] = save_collisions[unicode(self.basic_tab.collision.currentText())]
         prefs['updatemeta'] = self.basic_tab.updatemeta.isChecked()
         prefs['updatecover'] = self.basic_tab.updatecover.isChecked()
         prefs['updateepubcover'] = self.basic_tab.updateepubcover.isChecked()
@@ -182,6 +209,7 @@ class ConfigWidget(QWidget):
         prefs['suppresstitlesort'] = self.basic_tab.suppresstitlesort.isChecked()
         prefs['mark'] = self.basic_tab.mark.isChecked()
         prefs['showmarked'] = self.basic_tab.showmarked.isChecked()
+        prefs['autoconvert'] = self.basic_tab.autoconvert.isChecked()
         prefs['urlsfromclip'] = self.basic_tab.urlsfromclip.isChecked()
         prefs['updatedefault'] = self.basic_tab.updatedefault.isChecked()
         prefs['deleteotherforms'] = self.basic_tab.deleteotherforms.isChecked()
@@ -221,6 +249,7 @@ class ConfigWidget(QWidget):
                 #print("gc_site_settings[%s]:%s"%(site,gc_site_settings[site]))
         prefs['gc_site_settings'] = gc_site_settings
         prefs['allow_gc_from_ini'] = self.generatecover_tab.allow_gc_from_ini.isChecked()
+        prefs['gc_polish_cover'] = self.generatecover_tab.gc_polish_cover.isChecked()
 
         # Count Pages tab
         countpagesstats = []
@@ -284,17 +313,17 @@ class BasicTab(QWidget):
         topl = QVBoxLayout()
         self.setLayout(topl)
 
-        label = QLabel('These settings control the basic features of the plugin--downloading FanFiction.')
+        label = QLabel(_('These settings control the basic features of the plugin--downloading FanFiction.'))
         label.setWordWrap(True)
         topl.addWidget(label)
 
-        defs_gb = groupbox = QGroupBox("Defaults Options on Download")
+        defs_gb = groupbox = QGroupBox(_("Defaults Options on Download"))
         self.l = QVBoxLayout()
         groupbox.setLayout(self.l)
 
-        tooltip = "On each download, FFDL offers an option to select the output format. <br />This sets what that option will default to."
+        tooltip = _("On each download, FFDL offers an option to select the output format. <br />This sets what that option will default to.")
         horz = QHBoxLayout()
-        label = QLabel('Default Output &Format:')
+        label = QLabel(_('Default Output &Format:'))
         label.setToolTip(tooltip)
         horz.addWidget(label)
         self.fileform = QComboBox(self)
@@ -309,15 +338,15 @@ class BasicTab(QWidget):
         horz.addWidget(self.fileform)
         self.l.addLayout(horz)
 
-        tooltip = "On each download, FFDL offers an option of what happens if that story already exists. <br />This sets what that option will default to."
+        tooltip = _("On each download, FFDL offers an option of what happens if that story already exists. <br />This sets what that option will default to.")
         horz = QHBoxLayout()
-        label = QLabel('Default If Story Already Exists?')
+        label = QLabel(_('Default If Story Already Exists?'))
         label.setToolTip(tooltip)
         horz.addWidget(label)
         self.collision = QComboBox(self)
         # add collision options
         self.set_collisions()
-        i = self.collision.findText(prefs['collision'])
+        i = self.collision.findText(save_collisions[prefs['collision']])
         if i > -1:
             self.collision.setCurrentIndex(i)
         self.collision.setToolTip(tooltip)
@@ -325,127 +354,131 @@ class BasicTab(QWidget):
         horz.addWidget(self.collision)
         self.l.addLayout(horz)
 
-        self.updatemeta = QCheckBox('Default Update Calibre &Metadata?',self)
-        self.updatemeta.setToolTip("On each download, FFDL offers an option to update Calibre's metadata (title, author, URL, tags, custom columns, etc) from the web site. <br />This sets whether that will default to on or off. <br />Columns set to 'New Only' in the column tabs will only be set for new books.")
+        self.updatemeta = QCheckBox(_('Default Update Calibre &Metadata?'),self)
+        self.updatemeta.setToolTip(_("On each download, FFDL offers an option to update Calibre's metadata (title, author, URL, tags, custom columns, etc) from the web site. <br />This sets whether that will default to on or off. <br />Columns set to 'New Only' in the column tabs will only be set for new books."))
         self.updatemeta.setChecked(prefs['updatemeta'])
         self.l.addWidget(self.updatemeta)
 
-        self.updateepubcover = QCheckBox('Default Update EPUB Cover when Updating EPUB?',self)
-        self.updateepubcover.setToolTip("On each download, FFDL offers an option to update the book cover image <i>inside</i> the EPUB from the web site when the EPUB is updated.<br />This sets whether that will default to on or off.")
+        self.updateepubcover = QCheckBox(_('Default Update EPUB Cover when Updating EPUB?'),self)
+        self.updateepubcover.setToolTip(_("On each download, FFDL offers an option to update the book cover image <i>inside</i> the EPUB from the web site when the EPUB is updated.<br />This sets whether that will default to on or off."))
         self.updateepubcover.setChecked(prefs['updateepubcover'])
         self.l.addWidget(self.updateepubcover)
 
-        self.smarten_punctuation = QCheckBox('Smarten Punctuation (EPUB only)',self)
-        self.smarten_punctuation.setToolTip("Run Smarten Punctuation from Calibre's Polish Book feature on each EPUB download and update.")
+        self.smarten_punctuation = QCheckBox(_('Smarten Punctuation (EPUB only)'),self)
+        self.smarten_punctuation.setToolTip(_("Run Smarten Punctuation from Calibre's Polish Book feature on each EPUB download and update."))
         self.smarten_punctuation.setChecked(prefs['smarten_punctuation'])
         if calibre_version >= (0, 9, 39):
             self.l.addWidget(self.smarten_punctuation)
 
-        cali_gb = groupbox = QGroupBox("Updating Calibre Options")
+        cali_gb = groupbox = QGroupBox(_("Updating Calibre Options"))
         self.l = QVBoxLayout()
         groupbox.setLayout(self.l)
 
-        self.deleteotherforms = QCheckBox('Delete other existing formats?',self)
-        self.deleteotherforms.setToolTip('Check this to automatically delete all other ebook formats when updating an existing book.\nHandy if you have both a Nook(epub) and Kindle(mobi), for example.')
+        self.deleteotherforms = QCheckBox(_('Delete other existing formats?'),self)
+        self.deleteotherforms.setToolTip(_('Check this to automatically delete all other ebook formats when updating an existing book.\nHandy if you have both a Nook(epub) and Kindle(mobi), for example.'))
         self.deleteotherforms.setChecked(prefs['deleteotherforms'])
         self.l.addWidget(self.deleteotherforms)
         
-        self.updatecover = QCheckBox('Update Calibre Cover when Updating Metadata?',self)
-        self.updatecover.setToolTip("Update calibre book cover image from EPUB when metadata is updated.  (EPUB only.)\nDoesn't go looking for new images on 'Update Calibre Metadata Only'.")
+        self.updatecover = QCheckBox(_('Update Calibre Cover when Updating Metadata?'),self)
+        self.updatecover.setToolTip(_("Update calibre book cover image from EPUB when metadata is updated.  (EPUB only.)\nDoesn't go looking for new images on 'Update Calibre Metadata Only'."))
         self.updatecover.setChecked(prefs['updatecover'])
         self.l.addWidget(self.updatecover)
 
-        self.keeptags = QCheckBox('Keep Existing Tags when Updating Metadata?',self)
-        self.keeptags.setToolTip("Existing tags will be kept and any new tags added.\nCompleted and In-Progress tags will be still be updated, if known.\nLast Updated tags will be updated if lastupdate in include_subject_tags.\n(If Tags is set to 'New Only' in the Standard Columns tab, this has no effect.)")
+        self.keeptags = QCheckBox(_('Keep Existing Tags when Updating Metadata?'),self)
+        self.keeptags.setToolTip(_("Existing tags will be kept and any new tags added.\n%(cmplt)s and %(inprog)s tags will be still be updated, if known.\n%(lul)s tags will be updated if %(lus)s in %(is)s.\n(If Tags is set to 'New Only' in the Standard Columns tab, this has no effect.)")%no_trans)
         self.keeptags.setChecked(prefs['keeptags'])
         self.l.addWidget(self.keeptags)
 
-        self.suppressauthorsort = QCheckBox('Force Author into Author Sort?',self)
-        self.suppressauthorsort.setToolTip("If checked, the author(s) as given will be used for the Author Sort, too.\nIf not checked, calibre will apply it's built in algorithm which makes 'Bob Smith' sort as 'Smith, Bob', etc.")
+        self.suppressauthorsort = QCheckBox(_('Force Author into Author Sort?'),self)
+        self.suppressauthorsort.setToolTip(_("If checked, the author(s) as given will be used for the Author Sort, too.\nIf not checked, calibre will apply it's built in algorithm which makes 'Bob Smith' sort as 'Smith, Bob', etc."))
         self.suppressauthorsort.setChecked(prefs['suppressauthorsort'])
         self.l.addWidget(self.suppressauthorsort)
 
-        self.suppresstitlesort = QCheckBox('Force Title into Title Sort?',self)
-        self.suppresstitlesort.setToolTip("If checked, the title as given will be used for the Title Sort, too.\nIf not checked, calibre will apply it's built in algorithm which makes 'The Title' sort as 'Title, The', etc.")
+        self.suppresstitlesort = QCheckBox(_('Force Title into Title Sort?'),self)
+        self.suppresstitlesort.setToolTip(_("If checked, the title as given will be used for the Title Sort, too.\nIf not checked, calibre will apply it's built in algorithm which makes 'The Title' sort as 'Title, The', etc."))
         self.suppresstitlesort.setChecked(prefs['suppresstitlesort'])
         self.l.addWidget(self.suppresstitlesort)
 
-        self.checkforseriesurlid = QCheckBox("Check for existing Series Anthology books?",self)
-        self.checkforseriesurlid.setToolTip("Check for existings Series Anthology books using each new story's series URL before downloading.\nOffer to skip downloading if a Series Anthology is found.")
+        self.checkforseriesurlid = QCheckBox(_("Check for existing Series Anthology books?"),self)
+        self.checkforseriesurlid.setToolTip(_("Check for existings Series Anthology books using each new story's series URL before downloading.\nOffer to skip downloading if a Series Anthology is found."))
         self.checkforseriesurlid.setChecked(prefs['checkforseriesurlid'])
         self.l.addWidget(self.checkforseriesurlid)
 
-        self.checkforurlchange = QCheckBox("Check for changed Story URL?",self)
-        self.checkforurlchange.setToolTip("Warn you if an update will change the URL of an existing book.")
+        self.checkforurlchange = QCheckBox(_("Check for changed Story URL?"),self)
+        self.checkforurlchange.setToolTip(_("Warn you if an update will change the URL of an existing book.\nfanfiction.net URLs will change from http to https silently."))
         self.checkforurlchange.setChecked(prefs['checkforurlchange'])
         self.l.addWidget(self.checkforurlchange)
 
-        self.lookforurlinhtml = QCheckBox("Search EPUB text for Story URL?",self)
-        self.lookforurlinhtml.setToolTip("Look for first valid story URL inside EPUB text if not found in metadata.\nSomewhat risky, could find wrong URL depending on EPUB content.\nAlso finds and corrects bad ffnet URLs from ficsaver.com files.")
+        self.lookforurlinhtml = QCheckBox(_("Search EPUB text for Story URL?"),self)
+        self.lookforurlinhtml.setToolTip(_("Look for first valid story URL inside EPUB text if not found in metadata.\nSomewhat risky, could find wrong URL depending on EPUB content.\nAlso finds and corrects bad ffnet URLs from ficsaver.com files."))
         self.lookforurlinhtml.setChecked(prefs['lookforurlinhtml'])
         self.l.addWidget(self.lookforurlinhtml)
 
-        self.mark = QCheckBox("Mark added/updated books when finished?",self)
-        self.mark.setToolTip("Mark added/updated books when finished.  Use with option below.\nYou can also manually search for 'marked:ffdl_success'.\n'marked:ffdl_failed' is also available, or search 'marked:ffdl' for both.")
+        self.mark = QCheckBox(_("Mark added/updated books when finished?"),self)
+        self.mark.setToolTip(_("Mark added/updated books when finished.  Use with option below.\nYou can also manually search for 'marked:ffdl_success'.\n'marked:ffdl_failed' is also available, or search 'marked:ffdl' for both."))
         self.mark.setChecked(prefs['mark'])
         self.l.addWidget(self.mark)
 
-        self.showmarked = QCheckBox("Show Marked books when finished?",self)
-        self.showmarked.setToolTip("Show Marked added/updated books only when finished.\nYou can also manually search for 'marked:ffdl_success'.\n'marked:ffdl_failed' is also available, or search 'marked:ffdl' for both.")
+        self.showmarked = QCheckBox(_("Show Marked books when finished?"),self)
+        self.showmarked.setToolTip(_("Show Marked added/updated books only when finished.\nYou can also manually search for 'marked:ffdl_success'.\n'marked:ffdl_failed' is also available, or search 'marked:ffdl' for both."))
         self.showmarked.setChecked(prefs['showmarked'])
         self.l.addWidget(self.showmarked)
 
-        gui_gb = groupbox = QGroupBox("GUI Options")
+        self.autoconvert = QCheckBox(_("Automatically Convert new/update books?"),self)
+        self.autoconvert.setToolTip(_("Automatically call calibre's Convert for new/update books.\nConverts to the current output format as chosen in calibre's\nPreferences->Behavior settings."))
+        self.autoconvert.setChecked(prefs['autoconvert'])
+        self.l.addWidget(self.autoconvert)
+
+        gui_gb = groupbox = QGroupBox(_("GUI Options"))
         self.l = QVBoxLayout()
         groupbox.setLayout(self.l)
 
-        self.urlsfromclip = QCheckBox('Take URLs from Clipboard?',self)
-        self.urlsfromclip.setToolTip('Prefill URLs from valid URLs in Clipboard when Adding New.')
+        self.urlsfromclip = QCheckBox(_('Take URLs from Clipboard?'),self)
+        self.urlsfromclip.setToolTip(_('Prefill URLs from valid URLs in Clipboard when Adding New.'))
         self.urlsfromclip.setChecked(prefs['urlsfromclip'])
         self.l.addWidget(self.urlsfromclip)
 
-        self.updatedefault = QCheckBox('Default to Update when books selected?',self)
-        self.updatedefault.setToolTip('The top FanFictionDownLoader plugin button will start Update if\n'+
-                                      'books are selected.  If unchecked, it will always bring up \'Add New\'.')
+        self.updatedefault = QCheckBox(_('Default to Update when books selected?'),self)
+        self.updatedefault.setToolTip(_('The top FanFictionDownLoader plugin button will start Update if\nbooks are selected.  If unchecked, it will always bring up \'Add New\'.'))
         self.updatedefault.setChecked(prefs['updatedefault'])
         self.l.addWidget(self.updatedefault)
 
-        self.adddialogstaysontop = QCheckBox("Keep 'Add New from URL(s)' dialog on top?",self)
-        self.adddialogstaysontop.setToolTip("Instructs the OS and Window Manager to keep the 'Add New from URL(s)'\ndialog on top of all other windows.  Useful for dragging URLs onto it.")
+        self.adddialogstaysontop = QCheckBox(_("Keep 'Add New from URL(s)' dialog on top?"),self)
+        self.adddialogstaysontop.setToolTip(_("Instructs the OS and Window Manager to keep the 'Add New from URL(s)'\ndialog on top of all other windows.  Useful for dragging URLs onto it."))
         self.adddialogstaysontop.setChecked(prefs['adddialogstaysontop'])
         self.l.addWidget(self.adddialogstaysontop)
 
-        misc_gb = groupbox = QGroupBox("Misc Options")
+        misc_gb = groupbox = QGroupBox(_("Misc Options"))
         self.l = QVBoxLayout()
         groupbox.setLayout(self.l)
 
         # this is a cheat to make it easier for users to realize there's a new include_images features.
-        self.includeimages = QCheckBox("Include images in EPUBs?",self)
-        self.includeimages.setToolTip("Download and include images in EPUB stories.  This is equivalent to adding:\n\n[epub]\ninclude_images:true\nkeep_summary_html:true\nmake_firstimage_cover:true\n\n ...to the top of personal.ini.  Your settings in personal.ini will override this.")
+        self.includeimages = QCheckBox(_("Include images in EPUBs?"),self)
+        self.includeimages.setToolTip(_("Download and include images in EPUB stories.  This is equivalent to adding:%(imgset)s ...to the top of %(pini)s.  Your settings in %(pini)s will override this.")%no_trans)
         self.includeimages.setChecked(prefs['includeimages'])
         self.l.addWidget(self.includeimages)
 
-        self.injectseries = QCheckBox("Inject calibre Series when none found?",self)
-        self.injectseries.setToolTip("If no series is found, inject the calibre series (if there is one) so it appears on the FFDL title page(not cover).")
+        self.injectseries = QCheckBox(_("Inject calibre Series when none found?"),self)
+        self.injectseries.setToolTip(_("If no series is found, inject the calibre series (if there is one) so it appears on the FFDL title page(not cover)."))
         self.injectseries.setChecked(prefs['injectseries'])
         self.l.addWidget(self.injectseries)
 
-        rej_gb = groupbox = QGroupBox("Reject List")
+        rej_gb = groupbox = QGroupBox(_("Reject List"))
         self.l = QVBoxLayout()
         groupbox.setLayout(self.l)
 
-        self.rejectlist = QPushButton('Edit Reject URL List', self)
-        self.rejectlist.setToolTip("Edit list of URLs FFDL will automatically Reject.")
+        self.rejectlist = QPushButton(_('Edit Reject URL List'), self)
+        self.rejectlist.setToolTip(_("Edit list of URLs FFDL will automatically Reject."))
         self.rejectlist.clicked.connect(self.show_rejectlist)
         self.l.addWidget(self.rejectlist)
         
-        self.reject_urls = QPushButton('Add Reject URLs', self)
-        self.reject_urls.setToolTip("Add additional URLs to Reject as text.")
+        self.reject_urls = QPushButton(_('Add Reject URLs'), self)
+        self.reject_urls.setToolTip(_("Add additional URLs to Reject as text."))
         self.reject_urls.clicked.connect(self.add_reject_urls)
         self.l.addWidget(self.reject_urls)
         
-        self.reject_reasons = QPushButton('Edit Reject Reasons List', self)
-        self.reject_reasons.setToolTip("Customize the Reasons presented when Rejecting URLs")
+        self.reject_reasons = QPushButton(_('Edit Reject Reasons List'), self)
+        self.reject_reasons.setToolTip(_("Customize the Reasons presented when Rejecting URLs"))
         self.reject_reasons.clicked.connect(self.show_reject_reasons)
         self.l.addWidget(self.reject_reasons)
 
@@ -483,7 +516,7 @@ class BasicTab(QWidget):
         d = RejectListDialog(self,
                              rejecturllist.get_list(),
                              rejectreasons=rejecturllist.get_reject_reasons(),
-                             header="Edit Reject URLs List",
+                             header=_("Edit Reject URLs List"),
                              show_delete=False,
                              show_all_reasons=False)
         d.exec_()
@@ -497,22 +530,22 @@ class BasicTab(QWidget):
         d = EditTextDialog(self,
                            prefs['rejectreasons'],
                            icon=self.windowIcon(),
-                           title="Reject Reasons",
-                           label="Customize Reject List Reasons",
-                           tooltip="Customize the Reasons presented when Rejecting URLs")
+                           title=_("Reject Reasons"),
+                           label=_("Customize Reject List Reasons"),
+                           tooltip=_("Customize the Reasons presented when Rejecting URLs"))
         d.exec_()
         if d.result() == d.Accepted:
             prefs['rejectreasons'] = d.get_plain_text()
                 
     def add_reject_urls(self):
         d = EditTextDialog(self,
-                           "http://example.com/story.php?sid=5,Reason why I rejected it\nhttp://example.com/story.php?sid=6,Title by Author - Reason why I rejected it",
+                           "http://example.com/story.php?sid=5,"+_("Reason why I rejected it")+"\nhttp://example.com/story.php?sid=6,"+_("Title by Author")+" - "+_("Reason why I rejected it"),
                            icon=self.windowIcon(),
-                           title="Add Reject URLs",
-                           label="Add Reject URLs. Use: <b>http://...,note</b> or <b>http://...,title by author - note</b><br>Invalid story URLs will be ignored.",
-                           tooltip="One URL per line:\n<b>http://...,note</b>\n<b>http://...,title by author - note</b>",
+                           title=_("Add Reject URLs"),
+                           label=_("Add Reject URLs. Use: <b>http://...,note</b> or <b>http://...,title by author - note</b><br>Invalid story URLs will be ignored."),
+                           tooltip=_("One URL per line:\n<b>http://...,note</b>\n<b>http://...,title by author - note</b>"),
                            rejectreasons=rejecturllist.get_reject_reasons(),
-                           reasonslabel='Add this reason to all URLs added:')
+                           reasonslabel=_('Add this reason to all URLs added:'))
         d.exec_()
         if d.result() == d.Accepted:
             rejecturllist.add_text(d.get_plain_text(),d.get_reason_text())
@@ -527,7 +560,7 @@ class PersonalIniTab(QWidget):
         self.l = QVBoxLayout()
         self.setLayout(self.l)
 
-        label = QLabel('These settings provide more detailed control over what metadata will be displayed inside the ebook as well as let you set is_adult and user/password for different sites.')
+        label = QLabel(_('These settings provide more detailed control over what metadata will be displayed inside the ebook as well as let you set %(isa)s and %(u)s/%(p)s for different sites.')%no_trans)
         label.setWordWrap(True)
         self.l.addWidget(label)
         self.l.addSpacing(5)
@@ -545,8 +578,8 @@ class PersonalIniTab(QWidget):
         self.ini.setText(prefs['personal.ini'])
         self.l.addWidget(self.ini)
 
-        self.defaults = QPushButton('View Defaults (plugin-defaults.ini)', self)
-        self.defaults.setToolTip("View all of the plugin's configurable settings\nand their default settings.")
+        self.defaults = QPushButton(_('View Defaults')+' (plugin-defaults.ini)', self)
+        self.defaults.setToolTip(_("View all of the plugin's configurable settings\nand their default settings."))
         self.defaults.clicked.connect(self.show_defaults)
         self.l.addWidget(self.defaults)
         
@@ -564,14 +597,14 @@ class ShowDefaultsIniDialog(QDialog):
         self.resize(600, 500)
         self.l = QVBoxLayout()
         self.setLayout(self.l)
-        self.label = QLabel("Plugin Defaults (plugin-defaults.ini) (Read-Only)")
-        self.label.setToolTip("These are all of the plugin's configurable options\nand their default settings.")
+        self.label = QLabel(_("Plugin Defaults (%s) (Read-Only)")%'plugin-defaults.ini')
+        self.label.setToolTip(_("These are all of the plugin's configurable options\nand their default settings."))
         self.setWindowTitle(_('Plugin Defaults'))
         self.setWindowIcon(icon)
         self.l.addWidget(self.label)
         
         self.ini = QTextEdit(self)
-        self.ini.setToolTip("These are all of the plugin's configurable options\nand their default settings.")
+        self.ini.setToolTip(_("These are all of the plugin's configurable options\nand their default settings."))
         try:
             self.ini.setFont(QFont("Courier",
                                    get_gui().font().pointSize()+1))
@@ -582,7 +615,7 @@ class ShowDefaultsIniDialog(QDialog):
         self.ini.setReadOnly(True)
         self.l.addWidget(self.ini)
         
-        self.ok_button = QPushButton('OK', self)
+        self.ok_button = QPushButton(_('OK'), self)
         self.ok_button.clicked.connect(self.hide)
         self.l.addWidget(self.ok_button)
         
@@ -602,45 +635,45 @@ class ReadingListTab(QWidget):
         except KeyError:
             reading_lists= []
             
-        label = QLabel('These settings provide integration with the Reading List Plugin.  Reading List can automatically send to devices and change custom columns.  You have to create and configure the lists in Reading List to be useful.')
+        label = QLabel(_('These settings provide integration with the %(rl)s Plugin.  %(rl)s can automatically send to devices and change custom columns.  You have to create and configure the lists in %(rl)s to be useful.')%no_trans)
         label.setWordWrap(True)
         self.l.addWidget(label)
         self.l.addSpacing(5)
         
-        self.addtolists = QCheckBox('Add new/updated stories to "Send to Device" Reading List(s).',self)
-        self.addtolists.setToolTip('Automatically add new/updated stories to these lists in the Reading List plugin.')
+        self.addtolists = QCheckBox(_('Add new/updated stories to "Send to Device" Reading List(s).'),self)
+        self.addtolists.setToolTip(_('Automatically add new/updated stories to these lists in the %(rl)s plugin.')%no_trans)
         self.addtolists.setChecked(prefs['addtolists'])
         self.l.addWidget(self.addtolists)
             
         horz = QHBoxLayout()
-        label = QLabel('"Send to Device" Reading Lists')
-        label.setToolTip("When enabled, new/updated stories will be automatically added to these lists.")
+        label = QLabel(_('"Send to Device" Reading Lists'))
+        label.setToolTip(_("When enabled, new/updated stories will be automatically added to these lists."))
         horz.addWidget(label)        
         self.send_lists_box = MultiCompleteLineEdit(self)
-        self.send_lists_box.setToolTip("When enabled, new/updated stories will be automatically added to these lists.")
+        self.send_lists_box.setToolTip(_("When enabled, new/updated stories will be automatically added to these lists."))
         self.send_lists_box.update_items_cache(reading_lists)
         self.send_lists_box.setText(prefs['send_lists'])
         horz.addWidget(self.send_lists_box)
         self.l.addLayout(horz)
         
-        self.addtoreadlists = QCheckBox('Add new/updated stories to "To Read" Reading List(s).',self)
-        self.addtoreadlists.setToolTip('Automatically add new/updated stories to these lists in the Reading List plugin.\nAlso offers menu option to remove stories from the "To Read" lists.')
+        self.addtoreadlists = QCheckBox(_('Add new/updated stories to "To Read" Reading List(s).'),self)
+        self.addtoreadlists.setToolTip(_('Automatically add new/updated stories to these lists in the %(rl)s plugin.\nAlso offers menu option to remove stories from the "To Read" lists.')%no_trans)
         self.addtoreadlists.setChecked(prefs['addtoreadlists'])
         self.l.addWidget(self.addtoreadlists)
             
         horz = QHBoxLayout()
-        label = QLabel('"To Read" Reading Lists')
-        label.setToolTip("When enabled, new/updated stories will be automatically added to these lists.")
+        label = QLabel(_('"To Read" Reading Lists'))
+        label.setToolTip(_("When enabled, new/updated stories will be automatically added to these lists."))
         horz.addWidget(label)        
         self.read_lists_box = MultiCompleteLineEdit(self)
-        self.read_lists_box.setToolTip("When enabled, new/updated stories will be automatically added to these lists.")
+        self.read_lists_box.setToolTip(_("When enabled, new/updated stories will be automatically added to these lists."))
         self.read_lists_box.update_items_cache(reading_lists)
         self.read_lists_box.setText(prefs['read_lists'])
         horz.addWidget(self.read_lists_box)
         self.l.addLayout(horz)
         
-        self.addtolistsonread = QCheckBox('Add stories back to "Send to Device" Reading List(s) when marked "Read".',self)
-        self.addtolistsonread.setToolTip('Menu option to remove from "To Read" lists will also add stories back to "Send to Device" Reading List(s)')
+        self.addtolistsonread = QCheckBox(_('Add stories back to "Send to Device" Reading List(s) when marked "Read".'),self)
+        self.addtolistsonread.setToolTip(_('Menu option to remove from "To Read" lists will also add stories back to "Send to Device" Reading List(s)'))
         self.addtolistsonread.setChecked(prefs['addtolistsonread'])
         self.l.addWidget(self.addtolistsonread)
             
@@ -662,7 +695,7 @@ class GenerateCoverTab(QWidget):
         except KeyError:
             gc_settings= []
             
-        label = QLabel('The Generate Cover plugin can create cover images for books using various metadata and configurations.  If you have GC installed, FFDL can run GC on new downloads and metadata updates.  Pick a GC setting by site or Default.')
+        label = QLabel(_('The %(gc)s plugin can create cover images for books using various metadata and configurations.  If you have GC installed, FFDL can run GC on new downloads and metadata updates.  Pick a GC setting by site or Default.')%no_trans)
         label.setWordWrap(True)
         self.l.addWidget(label)
         self.l.addSpacing(5)
@@ -680,14 +713,15 @@ class GenerateCoverTab(QWidget):
 
         sitelist = getConfigSections()
         sitelist.sort()
-        sitelist.insert(0,u"Default")
+        sitelist.insert(0,_("Default"))
         for site in sitelist:
             horz = QHBoxLayout()
             label = QLabel(site)
-            if site == u"Default":
-                s = "On Metadata update, run Generate Cover with this setting, if not selected for specific site."
+            if site == _("Default"):
+                s = _("On Metadata update, run %(gc)s with this setting, if not selected for specific site.")%no_trans
             else:
-                s = "On Metadata update, run Generate Cover with this setting for %s stories."%site
+                no_trans['site']=site # not ideal, but, meh.
+                s = _("On Metadata update, run %(gc)s with this setting for %(site)s stories.")%no_trans
 
             label.setToolTip(s)
             horz.addWidget(label)
@@ -696,7 +730,12 @@ class GenerateCoverTab(QWidget):
             dropdown.addItem('',QVariant('none'))
             for setting in gc_settings:
                 dropdown.addItem(setting,QVariant(setting))
-            self.gc_dropdowns[site] = dropdown
+            if site == _("Default"):
+                self.gc_dropdowns["Default"] = dropdown
+                if 'Default' in prefs['gc_site_settings']:
+                    dropdown.setCurrentIndex(dropdown.findData(QVariant(prefs['gc_site_settings']['Default'])))
+            else:
+                self.gc_dropdowns[site] = dropdown
             if site in prefs['gc_site_settings']:
                 dropdown.setCurrentIndex(dropdown.findData(QVariant(prefs['gc_site_settings'][site])))
 
@@ -705,16 +744,21 @@ class GenerateCoverTab(QWidget):
         
         self.sl.insertStretch(-1)
         
-        self.gcnewonly = QCheckBox("Run Generate Cover Only on New Books",self)
-        self.gcnewonly.setToolTip("Default is to run GC any time the calibre metadata is updated.")
+        self.gcnewonly = QCheckBox(_("Run %(gc)s Only on New Books")%no_trans,self)
+        self.gcnewonly.setToolTip(_("Default is to run GC any time the calibre metadata is updated."))
         self.gcnewonly.setChecked(prefs['gcnewonly'])
         self.l.addWidget(self.gcnewonly)
 
-        self.allow_gc_from_ini = QCheckBox('Allow generate_cover_settings from personal.ini to override',self)
-        self.allow_gc_from_ini.setToolTip("The personal.ini parameter generate_cover_settings allows you to choose a GC setting based on metadata rather than site, but it's much more complex.<br \>generate_cover_settings is ignored when this is off.")
+        self.allow_gc_from_ini = QCheckBox(_('Allow %(gcset)s from %(pini)s to override')%no_trans,self)
+        self.allow_gc_from_ini.setToolTip(_("The %(pini)s parameter %(gcset)s allows you to choose a GC setting based on metadata rather than site, but it's much more complex.<br \>%(gcset)s is ignored when this is off.")%no_trans)
         self.allow_gc_from_ini.setChecked(prefs['allow_gc_from_ini'])
         self.l.addWidget(self.allow_gc_from_ini)
             
+        self.gc_polish_cover = QCheckBox(_("Use calibre's Polish feature to inject/update the cover"),self)
+        self.gc_polish_cover.setToolTip(_("Calibre's Polish feature will be used to inject or update the generated cover into the ebook, EPUB only."))
+        self.gc_polish_cover.setChecked(prefs['gc_polish_cover'])
+        self.l.addWidget(self.gc_polish_cover)
+            
 class CountPagesTab(QWidget):
 
     def __init__(self, parent_dialog, plugin_action):
@@ -725,39 +769,41 @@ class CountPagesTab(QWidget):
         self.l = QVBoxLayout()
         self.setLayout(self.l)
 
-        label = QLabel('These settings provide integration with the Count Pages Plugin.  Count Pages can automatically update custom columns with page, word and reading level statistics.  You have to create and configure the columns in Count Pages first.')
+        label = QLabel(_('These settings provide integration with the %(cp)s Plugin.  %(cp)s can automatically update custom columns with page, word and reading level statistics.  You have to create and configure the columns in %(cp)s first.')%no_trans)
         label.setWordWrap(True)
         self.l.addWidget(label)
         self.l.addSpacing(5)
 
-        label = QLabel('If any of the settings below are checked, when stories are added or updated, the Count Pages Plugin will be called to update the checked statistics.')
+        label = QLabel(_('If any of the settings below are checked, when stories are added or updated, the %(cp)s Plugin will be called to update the checked statistics.')%no_trans)
         label.setWordWrap(True)
         self.l.addWidget(label)
         self.l.addSpacing(5)
-        
+
+        # the same for all settings.  Mostly.
+        tooltip = _('Which column and algorithm to use are configured in %(cp)s.')%no_trans
         # 'PageCount', 'WordCount', 'FleschReading', 'FleschGrade', 'GunningFog'
         self.pagecount = QCheckBox('Page Count',self)
-        self.pagecount.setToolTip('Which column and algorithm to use are configured in Count Pages.')
+        self.pagecount.setToolTip(tooltip)
         self.pagecount.setChecked('PageCount' in prefs['countpagesstats'])
         self.l.addWidget(self.pagecount)
             
         self.wordcount = QCheckBox('Word Count',self)
-        self.wordcount.setToolTip('Which column and algorithm to use are configured in Count Words.\nWill overwrite word count from FFDL metadata if set to update the same custom column.')
+        self.wordcount.setToolTip(tooltip+"\n"+_('Will overwrite word count from FFDL metadata if set to update the same custom column.'))
         self.wordcount.setChecked('WordCount' in prefs['countpagesstats'])
         self.l.addWidget(self.wordcount)
 
         self.fleschreading = QCheckBox('Flesch Reading Ease',self)
-        self.fleschreading.setToolTip('Which column and algorithm to use are configured in Count Pages.')
+        self.fleschreading.setToolTip(tooltip)
         self.fleschreading.setChecked('FleschReading' in prefs['countpagesstats'])
         self.l.addWidget(self.fleschreading)
         
         self.fleschgrade = QCheckBox('Flesch-Kincaid Grade Level',self)
-        self.fleschgrade.setToolTip('Which column and algorithm to use are configured in Count Pages.')
+        self.fleschgrade.setToolTip(tooltip)
         self.fleschgrade.setChecked('FleschGrade' in prefs['countpagesstats'])
         self.l.addWidget(self.fleschgrade)
         
         self.gunningfog = QCheckBox('Gunning Fog Index',self)
-        self.gunningfog.setToolTip('Which column and algorithm to use are configured in Count Pages.')
+        self.gunningfog.setToolTip(tooltip)
         self.gunningfog.setChecked('GunningFog' in prefs['countpagesstats'])
         self.l.addWidget(self.gunningfog)
         
@@ -773,26 +819,23 @@ class OtherTab(QWidget):
         self.l = QVBoxLayout()
         self.setLayout(self.l)
 
-        label = QLabel("These controls aren't plugin settings as such, but convenience buttons for setting Keyboard shortcuts and getting all the FanFictionDownLoader confirmation dialogs back again.")
+        label = QLabel(_("These controls aren't plugin settings as such, but convenience buttons for setting Keyboard shortcuts and getting all the FanFictionDownLoader confirmation dialogs back again."))
         label.setWordWrap(True)
         self.l.addWidget(label)
         self.l.addSpacing(5)
         
-        keyboard_shortcuts_button = QPushButton('Keyboard shortcuts...', self)
-        keyboard_shortcuts_button.setToolTip(_(
-                    'Edit the keyboard shortcuts associated with this plugin'))
+        keyboard_shortcuts_button = QPushButton(_('Keyboard shortcuts...'), self)
+        keyboard_shortcuts_button.setToolTip(_('Edit the keyboard shortcuts associated with this plugin'))
         keyboard_shortcuts_button.clicked.connect(parent_dialog.edit_shortcuts)
         self.l.addWidget(keyboard_shortcuts_button)
 
         reset_confirmation_button = QPushButton(_('Reset disabled &confirmation dialogs'), self)
-        reset_confirmation_button.setToolTip(_(
-                    'Reset all show me again dialogs for the FanFictionDownLoader plugin'))
+        reset_confirmation_button.setToolTip(_('Reset all show me again dialogs for the FanFictionDownLoader plugin'))
         reset_confirmation_button.clicked.connect(self.reset_dialogs)
         self.l.addWidget(reset_confirmation_button)
         
-        view_prefs_button = QPushButton('&View library preferences...', self)
-        view_prefs_button.setToolTip(_(
-                    'View data stored in the library database for this plugin'))
+        view_prefs_button = QPushButton(_('&View library preferences...'), self)
+        view_prefs_button.setToolTip(_('View data stored in the library database for this plugin'))
         view_prefs_button.clicked.connect(self.view_prefs)
         self.l.addWidget(view_prefs_button)
         
@@ -852,35 +895,35 @@ permitted_values['text'] = permitted_values['enumeration']
 permitted_values['comments'] = permitted_values['enumeration']
 
 titleLabels = {
-    'category':'Category',
-    'genre':'Genre',
-    'language':'Language',
-    'status':'Status',
-    'status-C':'Status:Completed',
-    'status-I':'Status:In-Progress',
-    'series':'Series',
-    'characters':'Characters',
-    'ships':'Relationships',
-    'datePublished':'Published',
-    'dateUpdated':'Updated',
-    'dateCreated':'Created',
-    'rating':'Rating',
-    'warnings':'Warnings',
-    'numChapters':'Chapters',
-    'numWords':'Words',
-    'site':'Site',
-    'storyId':'Story ID',
-    'authorId':'Author ID',
-    'extratags':'Extra Tags',
-    'title':'Title',
-    'storyUrl':'Story URL',
-    'description':'Description',
-    'author':'Author',
-    'authorUrl':'Author URL',
-    'formatname':'File Format',
-    'formatext':'File Extension',
-    'siteabbrev':'Site Abbrev',
-    'version':'FFDL Version'
+    'category':_('Category'),
+    'genre':_('Genre'),
+    'language':_('Language'),
+    'status':_('Status'),
+    'status-C':_('Status:%(cmplt)s')%no_trans,
+    'status-I':_('Status:%(inprog)s')%no_trans,
+    'series':_('Series'),
+    'characters':_('Characters'),
+    'ships':_('Relationships'),
+    'datePublished':_('Published'),
+    'dateUpdated':_('Updated'),
+    'dateCreated':_('Created'),
+    'rating':_('Rating'),
+    'warnings':_('Warnings'),
+    'numChapters':_('Chapters'),
+    'numWords':_('Words'),
+    'site':_('Site'),
+    'storyId':_('Story ID'),
+    'authorId':_('Author ID'),
+    'extratags':_('Extra Tags'),
+    'title':_('Title'),
+    'storyUrl':_('Story URL'),
+    'description':_('Description'),
+    'author':_('Author'),
+    'authorUrl':_('Author URL'),
+    'formatname':_('File Format'),
+    'formatext':_('File Extension'),
+    'siteabbrev':_('Site Abbrev'),
+    'version':_('FFDL Version')
     }
 
 class CustomColumnsTab(QWidget):
@@ -895,7 +938,7 @@ class CustomColumnsTab(QWidget):
         self.l = QVBoxLayout()
         self.setLayout(self.l)
 
-        label = QLabel("If you have custom columns defined, they will be listed below.  Choose a metadata value type to fill your columns automatically.")
+        label = QLabel(_("If you have custom columns defined, they will be listed below.  Choose a metadata value type to fill your columns automatically."))
         label.setWordWrap(True)
         self.l.addWidget(label)
         self.l.addSpacing(5)
@@ -920,7 +963,7 @@ class CustomColumnsTab(QWidget):
                 #     print("column['%s'] => %s"%(k,v))
                 horz = QHBoxLayout()
                 label = QLabel(column['name'])
-                label.setToolTip("Update this %s column(%s) with..."%(key,column['datatype']))
+                label.setToolTip(_("Update this %s column(%s) with...")%(key,column['datatype']))
                 horz.addWidget(label)
                 dropdown = QComboBox(self)
                 dropdown.addItem('',QVariant('none'))
@@ -930,13 +973,13 @@ class CustomColumnsTab(QWidget):
                 if key in prefs['custom_cols']:
                     dropdown.setCurrentIndex(dropdown.findData(QVariant(prefs['custom_cols'][key])))
                 if column['datatype'] == 'enumeration':
-                    dropdown.setToolTip("Metadata values valid for this type of column.\nValues that aren't valid for this enumeration column will be ignored.")
+                    dropdown.setToolTip(_("Metadata values valid for this type of column.")+"\n"+_("Values that aren't valid for this enumeration column will be ignored."))
                 else:
-                    dropdown.setToolTip("Metadata values valid for this type of column.")
+                    dropdown.setToolTip(_("Metadata values valid for this type of column."))
                 horz.addWidget(dropdown)
 
-                newonlycheck = QCheckBox("New Only",self)
-                newonlycheck.setToolTip("Write to %s(%s) only for new\nbooks, not updates to existing books."%(column['name'],key))
+                newonlycheck = QCheckBox(_("New Only"),self)
+                newonlycheck.setToolTip(_("Write to %s(%s) only for new\nbooks, not updates to existing books.")%(column['name'],key))
                 self.custcol_newonlycheck[key] = newonlycheck
                 if key in prefs['custom_cols_newonly']:
                     newonlycheck.setChecked(prefs['custom_cols_newonly'][key])
@@ -947,19 +990,19 @@ class CustomColumnsTab(QWidget):
         self.sl.insertStretch(-1)
 
         self.l.addSpacing(5)
-        self.allow_custcol_from_ini = QCheckBox('Allow custom_columns_settings from personal.ini to override',self)
-        self.allow_custcol_from_ini.setToolTip("The personal.ini parameter custom_columns_settings allows you to set custom columns to site specific values that aren't common to all sites.<br \>custom_columns_settings is ignored when this is off.")
+        self.allow_custcol_from_ini = QCheckBox(_('Allow %(ccset)s from %(pini)s to override')%no_trans,self)
+        self.allow_custcol_from_ini.setToolTip(_("The %(pini)s parameter %(ccset)s allows you to set custom columns to site specific values that aren't common to all sites.<br />%(ccset)s is ignored when this is off.")%no_trans)
         self.allow_custcol_from_ini.setChecked(prefs['allow_custcol_from_ini'])
         self.l.addWidget(self.allow_custcol_from_ini)
         
         self.l.addSpacing(5)
-        label = QLabel("Special column:")
+        label = QLabel(_("Special column:"))
         label.setWordWrap(True)
         self.l.addWidget(label)
         
         horz = QHBoxLayout()
-        label = QLabel("Update/Overwrite Error Column:")
-        tooltip="When an update or overwrite of an existing story fails, record the reason in this column.\n(Text and Long Text columns only.)"
+        label = QLabel(_("Update/Overwrite Error Column:"))
+        tooltip=_("When an update or overwrite of an existing story fails, record the reason in this column.\n(Text and Long Text columns only.)")
         label.setToolTip(tooltip)
         horz.addWidget(label)
         self.errorcol = QComboBox(self)
@@ -984,21 +1027,21 @@ class StandardColumnsTab(QWidget):
 
         columns=OrderedDict()
         
-        columns["title"]="Title"
-        columns["authors"]="Author(s)"
-        columns["publisher"]="Publisher"
-        columns["tags"]="Tags"
-        columns["languages"]="Languages"
-        columns["pubdate"]="Published Date"
-        columns["timestamp"]="Date"
-        columns["comments"]="Comments"
-        columns["series"]="Series"
-        columns["identifiers"]="Ids(url id only)"
+        columns["title"]=_("Title")
+        columns["authors"]=_("Author(s)")
+        columns["publisher"]=_("Publisher")
+        columns["tags"]=_("Tags")
+        columns["languages"]=_("Languages")
+        columns["pubdate"]=_("Published Date")
+        columns["timestamp"]=_("Date")
+        columns["comments"]=_("Comments")
+        columns["series"]=_("Series")
+        columns["identifiers"]=_("Ids(url id only)")
 
         self.l = QVBoxLayout()
         self.setLayout(self.l)
 
-        label = QLabel("The standard calibre metadata columns are listed below.  You may choose whether FFDL will fill each column automatically on updates or only for new books.")
+        label = QLabel(_("The standard calibre metadata columns are listed below.  You may choose whether FFDL will fill each column automatically on updates or only for new books."))
         label.setWordWrap(True)
         self.l.addWidget(label)
         self.l.addSpacing(5)
@@ -1011,8 +1054,8 @@ class StandardColumnsTab(QWidget):
             #label.setToolTip("Update this %s column(%s) with..."%(key,column['datatype']))
             horz.addWidget(label)
 
-            newonlycheck = QCheckBox("New Only",self)
-            newonlycheck.setToolTip("Write to %s only for new\nbooks, not updates to existing books."%column)
+            newonlycheck = QCheckBox(_("New Only"),self)
+            newonlycheck.setToolTip(_("Write to %s only for new\nbooks, not updates to existing books.")%column)
             self.stdcol_newonlycheck[key] = newonlycheck
             if key in prefs['std_cols_newonly']:
                 newonlycheck.setChecked(prefs['std_cols_newonly'][key])

calibre-plugin/dialogs.py

@@ -29,6 +29,12 @@ from PyQt4.Qt import (QDialog, QTableWidget, QVBoxLayout, QHBoxLayout, QGridLayo
 from calibre.gui2.dialogs.confirm_delete import confirm
 from calibre.gui2.complete2 import EditWithComplete
 
+# pulls in translation files for _() strings
+try:
+    load_translations()
+except NameError:
+    pass # load_translations() added in calibre 1.9
+
 from calibre_plugins.fanfictiondownloader_plugin.common_utils \
     import (ReadOnlyTableWidgetItem, ReadOnlyTextIconWidgetItem, SizePersistedDialog,
             ImageTitleLayout, get_icon)
@@ -36,13 +42,13 @@ from calibre_plugins.fanfictiondownloader_plugin.common_utils \
 from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.geturls import get_urls_from_html, get_urls_from_text
 from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.adapters import getNormalStoryURL
 
-SKIP=u'Skip'
-ADDNEW=u'Add New Book'
-UPDATE=u'Update EPUB if New Chapters'
-UPDATEALWAYS=u'Update EPUB Always'
-OVERWRITE=u'Overwrite if Newer'
-OVERWRITEALWAYS=u'Overwrite Always'
-CALIBREONLY=u'Update Calibre Metadata Only'
+SKIP=_('Skip')
+ADDNEW=_('Add New Book')
+UPDATE=_('Update EPUB if New Chapters')
+UPDATEALWAYS=_('Update EPUB Always')
+OVERWRITE=_('Overwrite if Newer')
+OVERWRITEALWAYS=_('Overwrite Always')
+CALIBREONLY=_('Update Calibre Metadata Only')
 collision_order=[SKIP,
                  ADDNEW,
                  UPDATE,
@@ -51,6 +57,32 @@ collision_order=[SKIP,
                  OVERWRITEALWAYS,
                  CALIBREONLY,]
 
+# best idea I've had for how to deal with config/pref saving the
+# collision name in english.
+SAVE_SKIP='Skip'
+SAVE_ADDNEW='Add New Book'
+SAVE_UPDATE='Update EPUB if New Chapters'
+SAVE_UPDATEALWAYS='Update EPUB Always'
+SAVE_OVERWRITE='Overwrite if Newer'
+SAVE_OVERWRITEALWAYS='Overwrite Always'
+SAVE_CALIBREONLY='Update Calibre Metadata Only'
+save_collisions={
+    SKIP:SAVE_SKIP,
+    ADDNEW:SAVE_ADDNEW,
+    UPDATE:SAVE_UPDATE,
+    UPDATEALWAYS:SAVE_UPDATEALWAYS,
+    OVERWRITE:SAVE_OVERWRITE,
+    OVERWRITEALWAYS:SAVE_OVERWRITEALWAYS,
+    CALIBREONLY:SAVE_CALIBREONLY,
+    SAVE_SKIP:SKIP,
+    SAVE_ADDNEW:ADDNEW,
+    SAVE_UPDATE:UPDATE,
+    SAVE_UPDATEALWAYS:UPDATEALWAYS,
+    SAVE_OVERWRITE:OVERWRITE,
+    SAVE_OVERWRITEALWAYS:OVERWRITEALWAYS,
+    SAVE_CALIBREONLY:CALIBREONLY,
+    }
+    
 anthology_collision_order=[UPDATE,
                            UPDATEALWAYS,
                            OVERWRITEALWAYS]
@@ -104,6 +136,7 @@ class RejectUrlEntry:
     def fullnote(self):
         retval = ""
         if self.title and self.auth:
+            # don't translate--ends up being saved and confuses regex above.
             retval = retval + "%s by %s"%(self.title,self.auth)
             if self.note:
                 retval = retval + " - "
@@ -192,7 +225,7 @@ class AddNewDialog(SizePersistedDialog):
         self.l = QVBoxLayout()
         self.setLayout(self.l)
 
-        self.setWindowTitle('FanFictionDownLoader')
+        self.setWindowTitle(_('FanFictionDownLoader'))
         self.setWindowIcon(icon)
 
         self.toplabel=QLabel("Toplabel")
@@ -209,7 +242,7 @@ class AddNewDialog(SizePersistedDialog):
         # elements to show again when doing *update* merge
         self.mergeupdateshow = []
 
-        self.groupbox = QGroupBox("Show Download Options")
+        self.groupbox = QGroupBox(_("Show Download Options"))
         self.groupbox.setCheckable(True)
         self.groupbox.setChecked(False)
         self.groupbox.setFlat(True)
@@ -228,7 +261,7 @@ class AddNewDialog(SizePersistedDialog):
         self.groupbox.toggled.connect(self.gbf.setVisible)
         
         horz = QHBoxLayout()
-        label = QLabel('Output &Format:')
+        label = QLabel(_('Output &Format:'))
         self.mergehide.append(label)
         
         self.fileform = QComboBox(self)
@@ -236,7 +269,7 @@ class AddNewDialog(SizePersistedDialog):
         self.fileform.addItem('mobi')
         self.fileform.addItem('html')
         self.fileform.addItem('txt')
-        self.fileform.setToolTip('Choose output format to create.  May set default from plugin configuration.')
+        self.fileform.setToolTip(_('Choose output format to create.  May set default from plugin configuration.'))
         self.fileform.activated.connect(self.set_collisions)
         
         horz.addWidget(label)
@@ -252,7 +285,7 @@ class AddNewDialog(SizePersistedDialog):
         self.collision.setToolTip("CollisionToolTip")
         # add collision options
         self.set_collisions()
-        i = self.collision.findText(prefs['collision'])
+        i = self.collision.findText(save_collisions[prefs['collision']])
         if i > -1:
             self.collision.setCurrentIndex(i)
         self.collisionlabel.setBuddy(self.collision)
@@ -264,15 +297,15 @@ class AddNewDialog(SizePersistedDialog):
         self.mergeupdateshow.append(self.collision)
 
         horz = QHBoxLayout()
-        self.updatemeta = QCheckBox('Update Calibre &Metadata?',self)
-        self.updatemeta.setToolTip("Update metadata for existing stories in Calibre from web site?\n(Columns set to 'New Only' in the column tabs will only be set for new books.)")
+        self.updatemeta = QCheckBox(_('Update Calibre &Metadata?'),self)
+        self.updatemeta.setToolTip(_("Update metadata for existing stories in Calibre from web site?\n(Columns set to 'New Only' in the column tabs will only be set for new books.)"))
         self.updatemeta.setChecked(prefs['updatemeta'])
         horz.addWidget(self.updatemeta)
         self.mergehide.append(self.updatemeta)
         self.mergeupdateshow.append(self.updatemeta)
 
-        self.updateepubcover = QCheckBox('Update EPUB Cover?',self)
-        self.updateepubcover.setToolTip('Update book cover image from site or defaults (if found) <i>inside</i> the EPUB when EPUB is updated.')
+        self.updateepubcover = QCheckBox(_('Update EPUB Cover?'),self)
+        self.updateepubcover.setToolTip(_('Update book cover image from site or defaults (if found) <i>inside</i> the EPUB when EPUB is updated.'))
         self.updateepubcover.setChecked(prefs['updateepubcover'])
         horz.addWidget(self.updateepubcover)
         self.mergehide.append(self.updateepubcover)
@@ -319,10 +352,10 @@ class AddNewDialog(SizePersistedDialog):
         self.groupbox.setVisible(not(self.merge and self.newmerge))
         
         if self.merge:
-            self.toplabel.setText('Story URL(s) for anthology, one per line:')
-            self.url.setToolTip('URLs for stories to include in the anthology, one per line.\nWill take URLs from clipboard, but only valid URLs.')
-            self.collisionlabel.setText('If Story Already Exists in Anthology?')
-            self.collision.setToolTip("What to do if there's already an existing story with the same URL in the anthology.")
+            self.toplabel.setText(_('Story URL(s) for anthology, one per line:'))
+            self.url.setToolTip(_('URLs for stories to include in the anthology, one per line.\nWill take URLs from clipboard, but only valid URLs.'))
+            self.collisionlabel.setText(_('If Story Already Exists in Anthology?'))
+            self.collision.setToolTip(_("What to do if there's already an existing story with the same URL in the anthology."))
             for widget in self.mergehide:
                 widget.setVisible(False)
             if not self.newmerge:
@@ -331,10 +364,10 @@ class AddNewDialog(SizePersistedDialog):
         else:
             for widget in self.mergehide:
                 widget.setVisible(True)
-            self.toplabel.setText('Story URL(s), one per line:')
-            self.url.setToolTip('URLs for stories, one per line.\nWill take URLs from clipboard, but only valid URLs.\nAdd [1,5] after the URL to limit the download to chapters 1-5.')
-            self.collisionlabel.setText('If Story Already Exists?')
-            self.collision.setToolTip("What to do if there's already an existing story with the same URL or title and author.")
+            self.toplabel.setText(_('Story URL(s), one per line:'))
+            self.url.setToolTip(_('URLs for stories, one per line.\nWill take URLs from clipboard, but only valid URLs.\nAdd [1,5] after the URL to limit the download to chapters 1-5.'))
+            self.collisionlabel.setText(_('If Story Already Exists?'))
+            self.collision.setToolTip(_("What to do if there's already an existing story with the same URL or title and author."))
 
         # Need to re-able after hiding/showing
         self.setAcceptDrops(True)
@@ -350,10 +383,10 @@ class AddNewDialog(SizePersistedDialog):
 
         # add collision options
         self.set_collisions()
-        
-        i = self.collision.findText(self.prefs['collision'])
+        i = self.collision.findText(save_collisions[self.prefs['collision']])
         if i > -1:
             self.collision.setCurrentIndex(i)
+            
         self.updatemeta.setChecked(self.prefs['updatemeta'])
             
         if not self.merge:
@@ -434,18 +467,18 @@ class CollectURLDialog(SizePersistedDialog):
         self.url.setText(url_text)
         self.l.addWidget(self.url,1,1,1,2)
    
-        self.indiv_button = QPushButton('For Individual Books', self)
-        self.indiv_button.setToolTip('Get URLs and go to dialog for individual story downloads.')
+        self.indiv_button = QPushButton(_('For Individual Books'), self)
+        self.indiv_button.setToolTip(_('Get URLs and go to dialog for individual story downloads.'))
         self.indiv_button.clicked.connect(self.indiv)
         self.l.addWidget(self.indiv_button,2,0)
 
-        self.merge_button = QPushButton('For Anthology Epub', self)
-        self.merge_button.setToolTip('Get URLs and go to dialog for Anthology download.\nRequires EpubMerge 1.3.1+ plugin.')
+        self.merge_button = QPushButton(_('For Anthology Epub'), self)
+        self.merge_button.setToolTip(_('Get URLs and go to dialog for Anthology download.\nRequires %s plugin.')%'EpubMerge 1.3.1+')
         self.merge_button.clicked.connect(self.merge)
         self.l.addWidget(self.merge_button,2,1)
         self.merge_button.setEnabled(epubmerge_plugin!=None)
 
-        self.cancel_button = QPushButton('Cancel', self)
+        self.cancel_button = QPushButton(_('Cancel'), self)
         self.cancel_button.clicked.connect(self.cancel)
         self.l.addWidget(self.cancel_button,2,2)
 
@@ -477,29 +510,29 @@ class UserPassDialog(QDialog):
         self.setLayout(self.l)
 
         if exception.passwdonly:
-            self.setWindowTitle('Password')
-            self.l.addWidget(QLabel("Author requires a password for this story(%s)."%exception.url),0,0,1,2)
+            self.setWindowTitle(_('Password'))
+            self.l.addWidget(QLabel(_("Author requires a password for this story(%s).")%exception.url),0,0,1,2)
             # user isn't used, but it's easier to still have it for
             # post processing.
             self.user = FakeLineEdit()
         else:
-            self.setWindowTitle('User/Password')
-            self.l.addWidget(QLabel("%s requires you to login to download this story."%site),0,0,1,2)
+            self.setWindowTitle(_('User/Password'))
+            self.l.addWidget(QLabel(_("%s requires you to login to download this story.")%site),0,0,1,2)
         
-            self.l.addWidget(QLabel("User:"),1,0)
+            self.l.addWidget(QLabel(_("User:")),1,0)
             self.user = QLineEdit(self)
             self.l.addWidget(self.user,1,1)
    
-        self.l.addWidget(QLabel("Password:"),2,0)
+        self.l.addWidget(QLabel(_("Password:")),2,0)
         self.passwd = QLineEdit(self)
         self.passwd.setEchoMode(QLineEdit.Password)
         self.l.addWidget(self.passwd,2,1)
    
-        self.ok_button = QPushButton('OK', self)
+        self.ok_button = QPushButton(_('OK'), self)
         self.ok_button.clicked.connect(self.ok)
         self.l.addWidget(self.ok_button,3,0)
 
-        self.cancel_button = QPushButton('Cancel', self)
+        self.cancel_button = QPushButton(_('Cancel'), self)
         self.cancel_button.clicked.connect(self.cancel)
         self.l.addWidget(self.cancel_button,3,1)
 
@@ -521,9 +554,9 @@ class LoopProgressDialog(QProgressDialog):
                  book_list,
                  foreach_function,
                  finish_function,
-                 init_label="Fetching metadata for stories...",
-                 win_title="Downloading metadata for stories",
-                 status_prefix="Fetched metadata for"):
+                 init_label=_("Fetching metadata for stories..."),
+                 win_title=_("Downloading metadata for stories"),
+                 status_prefix=_("Fetched metadata for")):
         QProgressDialog.__init__(self,
                                  init_label,
                                  QString(), 0, len(book_list), gui)
@@ -541,7 +574,7 @@ class LoopProgressDialog(QProgressDialog):
         self.exec_()
 
     def updateStatus(self):
-        self.setLabelText("%s %d of %d"%(self.status_prefix,self.i+1,len(self.book_list)))
+        self.setLabelText("%s %d / %d"%(self.status_prefix,self.i+1,len(self.book_list)))
         self.setValue(self.i+1)
         #print(self.labelText())
 
@@ -647,7 +680,7 @@ class UpdateExistingDialog(SizePersistedDialog):
         spacerItem = QtGui.QSpacerItem(20, 40, QtGui.QSizePolicy.Minimum, QtGui.QSizePolicy.Expanding)
         button_layout.addItem(spacerItem)
         self.remove_button = QtGui.QToolButton(self)
-        self.remove_button.setToolTip('Remove selected books from the list')
+        self.remove_button.setToolTip(_('Remove selected books from the list'))
         self.remove_button.setIcon(get_icon('list_remove.png'))
         self.remove_button.clicked.connect(self.remove_from_list)
         button_layout.addWidget(self.remove_button)
@@ -656,7 +689,7 @@ class UpdateExistingDialog(SizePersistedDialog):
 
         options_layout = QHBoxLayout()
 
-        groupbox = QGroupBox("Show Download Options")
+        groupbox = QGroupBox(_("Show Download Options"))
         groupbox.setCheckable(True)
         groupbox.setChecked(False)
         groupbox.setFlat(True)
@@ -673,7 +706,7 @@ class UpdateExistingDialog(SizePersistedDialog):
         gbf.setVisible(False)
         groupbox.toggled.connect(gbf.setVisible)
         
-        label = QLabel('Output &Format:')
+        label = QLabel(_('Output &Format:'))
         gbl.addWidget(label)
         self.fileform = QComboBox(self)
         self.fileform.addItem('epub')
@@ -681,30 +714,30 @@ class UpdateExistingDialog(SizePersistedDialog):
         self.fileform.addItem('html')
         self.fileform.addItem('txt')
         self.fileform.setCurrentIndex(self.fileform.findText(prefs['fileform']))
-        self.fileform.setToolTip('Choose output format to create.  May set default from plugin configuration.')
+        self.fileform.setToolTip(_('Choose output format to create.  May set default from plugin configuration.'))
         self.fileform.activated.connect(self.set_collisions)
         label.setBuddy(self.fileform)
         gbl.addWidget(self.fileform)
         
-        label = QLabel('Update Mode:')
+        label = QLabel(_('Update Mode:'))
         gbl.addWidget(label)
         self.collision = QComboBox(self)
-        self.collision.setToolTip("What sort of update to perform.  May set default from plugin configuration.")
+        self.collision.setToolTip(_("What sort of update to perform.  May set default from plugin configuration."))
         # add collision options
         self.set_collisions()
-        i = self.collision.findText(prefs['collision'])
+        i = self.collision.findText(save_collisions[prefs['collision']])
         if i > -1:
             self.collision.setCurrentIndex(i)
         label.setBuddy(self.collision)
         gbl.addWidget(self.collision)
 
-        self.updatemeta = QCheckBox('Update Calibre &Metadata?',self)
-        self.updatemeta.setToolTip("Update metadata for existing stories in Calibre from web site?\n(Columns set to 'New Only' in the column tabs will only be set for new books.)")
+        self.updatemeta = QCheckBox(_('Update Calibre &Metadata?'),self)
+        self.updatemeta.setToolTip(_("Update metadata for existing stories in Calibre from web site?\n(Columns set to 'New Only' in the column tabs will only be set for new books.)"))
         self.updatemeta.setChecked(prefs['updatemeta'])
         gbl.addWidget(self.updatemeta)
                 
-        self.updateepubcover = QCheckBox('Update EPUB Cover?',self)
-        self.updateepubcover.setToolTip('Update book cover image from site or defaults (if found) <i>inside</i> the EPUB when EPUB is updated.')
+        self.updateepubcover = QCheckBox(_('Update EPUB Cover?'),self)
+        self.updateepubcover.setToolTip(_('Update book cover image from site or defaults (if found) <i>inside</i> the EPUB when EPUB is updated.'))
         self.updateepubcover.setChecked(prefs['updateepubcover'])
         gbl.addWidget(self.updateepubcover)
 
@@ -757,7 +790,7 @@ class StoryListTableWidget(QTableWidget):
         self.clear()
         self.setAlternatingRowColors(True)
         self.setRowCount(len(books))
-        header_labels = ['','Title', 'Author', 'URL', 'Comment']
+        header_labels = ['',_('Title'), _('Author'), 'URL', _('Comment')]
         self.setColumnCount(len(header_labels))
         self.setHorizontalHeaderLabels(header_labels)
         self.horizontalHeader().setStretchLastSection(True)
@@ -825,9 +858,9 @@ class StoryListTableWidget(QTableWidget):
         rows = self.selectionModel().selectedRows()
         if len(rows) == 0:
             return
-        message = '<p>Are you sure you want to remove this book from the list?'
+        message = '<p>'+_('Are you sure you want to remove this book from the list?')
         if len(rows) > 1:
-            message = '<p>Are you sure you want to remove the selected %d books from the list?'%len(rows)
+            message = '<p>'+_('Are you sure you want to remove the selected %d books from the list?')%len(rows)
         if not confirm(message,'fanfictiondownloader_delete_item', self):
             return
         first_sel_row = self.currentRow()
@@ -853,7 +886,7 @@ class RejectListTableWidget(QTableWidget):
         self.clear()
         self.setAlternatingRowColors(True)
         self.setRowCount(len(reject_list))
-        header_labels = ['URL', 'Title', 'Author', 'Note']
+        header_labels = ['URL', _('Title'), _('Author'), _('Note')]
         self.setColumnCount(len(header_labels))
         self.setHorizontalHeaderLabels(header_labels)
         self.horizontalHeader().setStretchLastSection(True)
@@ -895,7 +928,7 @@ class RejectListTableWidget(QTableWidget):
         note_cell.update_items_cache(items)
         note_cell.show_initial_value(rej.note)
         note_cell.set_separator(None)
-        note_cell.setToolTip('Select or Edit Reject Note.')
+        note_cell.setToolTip(_('Select or Edit Reject Note.'))
         self.setCellWidget(row, 3, note_cell)
         
     def remove_selected_rows(self):
@@ -903,9 +936,9 @@ class RejectListTableWidget(QTableWidget):
         rows = self.selectionModel().selectedRows()
         if len(rows) == 0:
             return
-        message = '<p>Are you sure you want to remove this URL from the list?'
+        message = '<p>'+_('Are you sure you want to remove this URL from the list?')
         if len(rows) > 1:
-            message = '<p>Are you sure you want to remove the %d selected URLs from the list?'%len(rows)
+            message = '<p>'+_('Are you sure you want to remove the %d selected URLs from the list?')%len(rows)
         if not confirm(message,'ffdl_rejectlist_delete_item_again', self):
             return
         first_sel_row = self.currentRow()
@@ -923,7 +956,7 @@ class RejectListTableWidget(QTableWidget):
 class RejectListDialog(SizePersistedDialog):
     def __init__(self, gui, reject_list,
                  rejectreasons=[],
-                 header="List of Books to Reject",
+                 header=_("List of Books to Reject"),
                  icon='rotate-right.png',
                  show_delete=True,
                  show_all_reasons=True,
@@ -936,7 +969,7 @@ class RejectListDialog(SizePersistedDialog):
         layout = QVBoxLayout(self)
         self.setLayout(layout)
         title_layout = ImageTitleLayout(self, icon, header,
-                                        '<i></i>FFDL will remember these URLs and display the note and offer to reject them if you try to download them again later.')
+                                        '<i></i>'+_('FFDL will remember these URLs and display the note and offer to reject them if you try to download them again later.'))
         layout.addLayout(title_layout)
         rejects_layout = QHBoxLayout()
         layout.addLayout(rejects_layout)
@@ -950,7 +983,7 @@ class RejectListDialog(SizePersistedDialog):
         button_layout.addItem(spacerItem)
         
         self.remove_button = QtGui.QToolButton(self)
-        self.remove_button.setToolTip('Remove selected URL(s) from the list')
+        self.remove_button.setToolTip(_('Remove selected URL(s) from the list'))
         self.remove_button.setIcon(get_icon('list_remove.png'))
         self.remove_button.clicked.connect(self.remove_from_list)
         button_layout.addWidget(self.remove_button)
@@ -968,11 +1001,11 @@ class RejectListDialog(SizePersistedDialog):
             self.reason_edit.update_items_cache(items)
             self.reason_edit.show_initial_value('')
             self.reason_edit.set_separator(None)
-            self.reason_edit.setToolTip("This will be added to whatever note you've set for each URL above.")
+            self.reason_edit.setToolTip(_("This will be added to whatever note you've set for each URL above."))
             
             horz = QHBoxLayout()
-            label = QLabel("Add this reason to all URLs added:")
-            label.setToolTip("This will be added to whatever note you've set for each URL above.")
+            label = QLabel(_("Add this reason to all URLs added:"))
+            label.setToolTip(_("This will be added to whatever note you've set for each URL above."))
             horz.addWidget(label)
             horz.addWidget(self.reason_edit)
             horz.insertStretch(-1)
@@ -981,8 +1014,8 @@ class RejectListDialog(SizePersistedDialog):
         options_layout = QHBoxLayout()
 
         if show_delete:
-            self.deletebooks = QCheckBox('Delete Books (including books without FanFiction URLs)?',self)
-            self.deletebooks.setToolTip("Delete the selected books after adding them to the Rejected URLs list.")
+            self.deletebooks = QCheckBox(_('Delete Books (including books without FanFiction URLs)?'),self)
+            self.deletebooks.setToolTip(_("Delete the selected books after adding them to the Rejected URLs list."))
             self.deletebooks.setChecked(True)
             options_layout.addWidget(self.deletebooks)
         

calibre-plugin/jobs.py

@@ -93,16 +93,13 @@ def do_download_worker(book_list, options,
         logger.info('Logfile for book ID %s (%s)'%(book_id, job._book['title']))
         logger.info(job.details)
 
+
+
         if count >= total:
-            # All done!  Output some lists for convenience of some users.
-            logger.info("Successfully downloaded:")
-            for book in book_list:
-                if book['good']:
-                    logger.info("%s %s"%(book['title'],book['url']))
-            logger.info("\nUnsuccessful:")
-            for book in book_list:
-                if not book['good']:
-                    logger.info("%s %s"%(book['title'],book['url']))
+            logger.info("\nSuccessful:\n%s\n"%("\n".join([book['url'] for book in 
+                                                      filter(lambda x: x['good'], book_list) ] ) ) )
+            logger.info("\nUnsuccessful:\n%s\n"%("\n".join([book['url'] for book in 
+                                                        filter(lambda x: not x['good'], book_list) ] ) ) )
             break
 
     server.close()
@@ -182,7 +179,7 @@ def do_download_for_worker(book,options,notification=lambda x,y:x):
 
             # update now handled by pre-populating the old images and
             # chapters in the adapter rather than merging epubs.
-            urlchaptercount = int(story.getMetadata('numChapters'))
+            urlchaptercount = int(story.getMetadata('numChapters').replace(',',''))
             (url,
              chaptercount,
              adapter.oldchapters,

calibre-plugin/prefs.py

@@ -34,6 +34,7 @@ default_prefs['suppressauthorsort'] = False
 default_prefs['suppresstitlesort'] = False
 default_prefs['mark'] = False
 default_prefs['showmarked'] = False
+default_prefs['autoconvert'] = False
 default_prefs['urlsfromclip'] = True
 default_prefs['updatedefault'] = True
 default_prefs['fileform'] = 'epub'
@@ -56,6 +57,7 @@ default_prefs['addtolistsonread'] = False
 default_prefs['gcnewonly'] = False
 default_prefs['gc_site_settings'] = {}
 default_prefs['allow_gc_from_ini'] = True
+default_prefs['gc_polish_cover'] = False
 
 default_prefs['countpagesstats'] = []
 

downloader.py

@@ -23,6 +23,7 @@ import getpass
 import string
 import ConfigParser
 from subprocess import call
+import pprint
 
 import logging
 if sys.version_info >= (2, 7):
@@ -270,31 +271,29 @@ def main(argv,
            elif chaptercount == 0:
                print "%s doesn't contain any recognizable chapters, probably from a different source.  Not updating." % (output_filename)
            else:
-               if not options.metaonly:
+               # update now handled by pre-populating the old
+               # images and chapters in the adapter rather than
+               # merging epubs.
+               (url,
+                chaptercount,
+                adapter.oldchapters,
+                adapter.oldimgs,
+                adapter.oldcover,
+                adapter.calibrebookmark,
+                adapter.logfile) = get_update_data(output_filename)
 
-                   # update now handled by pre-populating the old
-                   # images and chapters in the adapter rather than
-                   # merging epubs.
-                   (url,
-                    chaptercount,
-                    adapter.oldchapters,
-                    adapter.oldimgs,
-                    adapter.oldcover,
-                    adapter.calibrebookmark,
-                    adapter.logfile) = get_update_data(output_filename)
+               print "Do update - epub(%d) vs url(%d)" % (chaptercount, urlchaptercount)
 
-                   print "Do update - epub(%d) vs url(%d)" % (chaptercount, urlchaptercount)
+               if not (options.update and chaptercount == urlchaptercount) \
+                       and adapter.getConfig("do_update_hook"):
+                   chaptercount = adapter.hookForUpdates(chaptercount)
 
-                   if not (options.update and chaptercount == urlchaptercount) \
-                           and adapter.getConfig("do_update_hook"):
-                       chaptercount = adapter.hookForUpdates(chaptercount)
-
-                   writeStory(configuration,adapter,"epub")
+               writeStory(configuration,adapter,"epub")
                    
        else:
            # regular download
            if options.metaonly:
-               print adapter.getStoryMetadataOnly()
+               pprint.pprint(adapter.getStoryMetadataOnly().getAllMetadata())
            
            output_filename=writeStory(configuration,adapter,options.format,options.metaonly)
        

fanficdownloader/HtmlTagStack.py

@@ -0,0 +1,57 @@
+# coding: utf-8
+
+import re
+import codecs
+
+stack = []
+
+def get_end_tag(tag):
+    if len(tag) > 0 and tag.find(u'<') > -1 and tag.rfind(u'>') > -1:
+        return re.sub(r'.*<([^\ >]+).*', r'</\1>', tag)
+    return u''
+
+def get_tag_name(tag):
+    if len(tag) > 0 and tag.find(u'<') > -1 and tag.rfind(u'>') > -1:
+        return re.sub(r'</*([^\ >]+).*', r'\1', tag)
+    return u''
+
+def push(tag):
+    if len(tag) > 0 and tag.find(u'<') > -1 and tag.rfind(u'>') > -1:
+        stack.append(tag)
+
+def pop():
+    if len(stack) > 0:
+        return stack.pop()
+    return u''
+
+def pop_end_tag():
+    return unicode(get_end_tag(pop()))
+
+def spool_end():
+    html = u''
+    for tag in reversed(stack):
+        html += get_end_tag(tag)
+    return html
+
+def spool_start():
+    html = u''
+    for item in stack:
+        html += item
+    return html
+
+def has_elements():
+    return len(stack) > 0
+
+def get_last():
+    # t = pop()
+    # push(t)
+    # return t
+    if len(stack) > 0:
+        return stack[len(stack)-1]
+    return u''
+
+def flush():
+    del stack[:]
+
+def get_stack():
+    return stack

fanficdownloader/adapters/__init__.py

@@ -121,6 +121,11 @@ import adapter_simplyundeniablecom
 import adapter_scarheadnet
 import adapter_fictionpadcom
 import adapter_storiesonlinenet
+import adapter_trekiverseorg
+import adapter_literotica
+import adapter_voracity2eficcom
+import adapter_spikeluvercom
+import adapter_bloodshedversecom
 
 ## This bit of complexity allows adapters to be added by just adding
 ## importing.  It eliminates the long if/else clauses we used to need
@@ -144,6 +149,13 @@ for x in imports():
             __domain_map[site]=cls
 
 def getNormalStoryURL(url):
+    r = getNormalStoryURLSite(url)
+    if r:
+        return r[0]
+    else:
+        return None
+
+def getNormalStoryURLSite(url):
     if not getNormalStoryURL.__dummyconfig:
         getNormalStoryURL.__dummyconfig = Configuration("test1.com","EPUB")
     # pulling up an adapter is pretty low over-head.  If
@@ -151,10 +163,11 @@ def getNormalStoryURL(url):
     try:
         adapter = getAdapter(getNormalStoryURL.__dummyconfig,url)
         url = adapter.url
+        site = adapter.getSiteDomain()
         del adapter
-        return url
+        return (url,site)
     except:
-        return None;
+        return None
 
 # kludgey function static/singleton
 getNormalStoryURL.__dummyconfig = None
@@ -191,12 +204,12 @@ def getConfigSectionFor(url):
 
 def getClassFor(url):
     ## fix up leading protocol.
-    fixedurl = re.sub(r"(?i)^[htps]+[:/]+","http://",url.strip())
+    fixedurl = re.sub(r"(?i)^[htp]+(s?)[:/]+",r"http\1://",url.strip())
     if not fixedurl.startswith("http"):
         fixedurl = "http://%s"%url
     ## remove any trailing '#' locations.
     fixedurl = re.sub(r"#.*$","",fixedurl)
-    
+
     parsedUrl = up.urlparse(fixedurl)
     domain = parsedUrl.netloc.lower()
     if( domain != parsedUrl.netloc ):

fanficdownloader/adapters/adapter_archiveofourownorg.py

@@ -77,14 +77,14 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
         # The site domain.  Does have www here, if it uses it.
         return 'archiveofourown.org'
 
-    @classmethod
+    @classmethod
     def getSiteExampleURLs(self):
         return "http://"+self.getSiteDomain()+"/works/123456 http://"+self.getSiteDomain()+"/collections/Some_Archive/works/123456 http://"+self.getSiteDomain()+"/works/123456/chapters/78901"
 
     def getSiteURLPattern(self):
         # http://archiveofourown.org/collections/Smallville_Slash_Archive/works/159770
         # Discard leading zeros from story ID numbers--AO3 doesn't use them in it's own chapter URLs.
-        return re.escape("http://")+re.escape(self.getSiteDomain())+r"(/collections/[^/]+)?/works/0*(?P<id>\d+)"
+        return r"https?://"+re.escape(self.getSiteDomain())+r"(/collections/[^/]+)?/works/0*(?P<id>\d+)"
         
     ## Login
     def needToLoginCheck(self, data):
@@ -145,10 +145,13 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
             
         except urllib2.HTTPError, e:
             if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.meta)
+                raise exceptions.StoryDoesNotExist(self.url)
             else:
                 raise e
-                
+
+        if "Sorry, we couldn&#x27;t find the work you were looking for." in data:
+            raise exceptions.StoryDoesNotExist(self.url)
+            
         if self.needToLoginCheck(data):
             # need to log in for this one.
             self.performLogin(url,data)
@@ -177,19 +180,28 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
                 self.story.addToList('authorUrl','http://'+self.host+a['href'])
                 self.story.addToList('author',a.text)
 
+        newestChapter = None
+        self.newestChapterNum = None # save for comparing during update.
+        # Scan all chapters to find the oldest and newest, on AO3 it's
+        # possible for authors to insert new chapters out-of-order or
+        # change the dates of earlier ones by editing them--That WILL
+        # break epub update.
         # Find the chapters:
         chapters=soup.findAll('a', href=re.compile(r'/works/'+self.story.getMetadata('storyId')+"/chapters/\d+$"))
         self.story.setMetadata('numChapters',len(chapters))
         logger.debug("numChapters: (%s)"%self.story.getMetadata('numChapters'))
-        for x in range(0,len(chapters)):
-            # just in case there's tags, like <i> in chapter titles.
-            chapter=chapters[x]
-            if len(chapters)==1:
-                self.chapterUrls.append((self.story.getMetadata('title'),'http://'+self.host+chapter['href']+addurl))
-            else:
+        if len(chapters)==1:
+            self.chapterUrls.append((self.story.getMetadata('title'),'http://'+self.host+chapters[0]['href']+addurl))
+        else:
+            for index, chapter in enumerate(chapters):
+                # strip just in case there's tags, like <i> in chapter titles.
                 self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+chapter['href']+addurl))
-
-
+                # (2013-09-21)
+                date = stripHTML(chapter.findNext('span'))[1:-1]
+                chapterDate = makeDate(date,self.dateformat)
+                if newestChapter == None or chapterDate > newestChapter:
+                    newestChapter = chapterDate
+                    self.newestChapterNum = index
 
         a = metasoup.find('blockquote',{'class':'userstuff'})
         if a != None:
@@ -299,6 +311,12 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
                     self.setSeries(series_name, series_index)
                     self.story.setMetadata('seriesUrl',series_url)
 
+    def hookForUpdates(self,chaptercount):
+        if self.oldchapters and len(self.oldchapters) > self.newestChapterNum:
+            print("Existing epub has %s chapters\nNewest chapter is %s.  Discarding old chapters from there on."%(len(self.oldchapters), self.newestChapterNum+1))
+            self.oldchapters = self.oldchapters[:self.newestChapterNum]
+        return len(self.oldchapters)
+            
     # grab the text for an individual chapter.
     def getChapterText(self, url):
         logger.debug('Getting chapter text from: %s' % url)
@@ -306,25 +324,31 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
         chapter=bs.BeautifulSoup('<div class="story"></div>').find('div')
         data = self._fetchUrl(url)
         soup = bs.BeautifulSoup(data,selfClosingTags=('br','hr'))
-		
-        headnotes = soup.find('div', {'class' : "preface group"}).find('div', {'class' : "notes module"})
-        if headnotes != None:
-            headnotes = headnotes.find('blockquote', {'class' : "userstuff"})
+
+        exclude_notes=self.getConfigList('exclude_notes')
+
+        if 'authorheadnotes' not in exclude_notes:
+            headnotes = soup.find('div', {'class' : "preface group"}).find('div', {'class' : "notes module"})
             if headnotes != None:
-                chapter.append("<b>Author's Note:</b>")
-                chapter.append(headnotes)
+                headnotes = headnotes.find('blockquote', {'class' : "userstuff"})
+                if headnotes != None:
+                    chapter.append("<b>Author's Note:</b>")
+                    chapter.append(headnotes)
         
-        chapsumm = soup.find('div', {'id' : "summary"})
-        if chapsumm != None:
-            chapsumm = chapsumm.find('blockquote')
-            chapter.append("<b>Summary for the Chapter:</b>")
-            chapter.append(chapsumm)
-        chapnotes = soup.find('div', {'id' : "notes"})
-        if chapnotes != None:
-            chapnotes = chapnotes.find('blockquote')
+        if 'chaptersummary' not in exclude_notes:
+            chapsumm = soup.find('div', {'id' : "summary"})
+            if chapsumm != None:
+                chapsumm = chapsumm.find('blockquote')
+                chapter.append("<b>Summary for the Chapter:</b>")
+                chapter.append(chapsumm)
+                
+        if 'chapterheadnotes' not in exclude_notes:
+            chapnotes = soup.find('div', {'id' : "notes"})
             if chapnotes != None:
-                chapter.append("<b>Notes for the Chapter:</b>")
-                chapter.append(chapnotes)
+                chapnotes = chapnotes.find('blockquote')
+                if chapnotes != None:
+                    chapter.append("<b>Notes for the Chapter:</b>")
+                    chapter.append(chapnotes)
 		
         text = soup.find('div', {'class' : "userstuff module"})
         chtext = text.find('h3', {'class' : "landmark heading"})
@@ -332,17 +356,19 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
             chtext.extract()
         chapter.append(text)
 		
-        chapfoot = soup.find('div', {'class' : "end notes module", 'role' : "complementary"})
-        if chapfoot != None:
-            chapfoot = chapfoot.find('blockquote')
-            chapter.append("<b>Notes for the Chapter:</b>")
-            chapter.append(chapfoot)
+        if 'chapterfootnotes' not in exclude_notes:
+            chapfoot = soup.find('div', {'class' : "end notes module", 'role' : "complementary"})
+            if chapfoot != None:
+                chapfoot = chapfoot.find('blockquote')
+                chapter.append("<b>Notes for the Chapter:</b>")
+                chapter.append(chapfoot)
 		
-        footnotes = soup.find('div', {'id' : "work_endnotes"})
-        if footnotes != None:
-            footnotes = footnotes.find('blockquote')
-            chapter.append("<b>Author's Note:</b>")
-            chapter.append(footnotes)
+        if 'authorfootnotes' not in exclude_notes:
+            footnotes = soup.find('div', {'id' : "work_endnotes"})
+            if footnotes != None:
+                footnotes = footnotes.find('blockquote')
+                chapter.append("<b>Author's Note:</b>")
+                chapter.append(footnotes)
 			
         if None == soup:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)

fanficdownloader/adapters/adapter_bloodshedversecom.py

@@ -0,0 +1,187 @@
+from datetime import timedelta
+import re
+import urllib2
+import urlparse
+
+from .. import BeautifulSoup
+
+from base_adapter import BaseSiteAdapter, makeDate
+from .. import exceptions
+
+
+def getClass():
+    return BloodshedverseComAdapter
+
+
+def _get_query_data(url):
+    components = urlparse.urlparse(url)
+    query_data = urlparse.parse_qs(components.query)
+    return dict((key, data[0]) for key, data in query_data.items())
+
+
+class BloodshedverseComAdapter(BaseSiteAdapter):
+    SITE_ABBREVIATION = 'bvc'
+    SITE_DOMAIN = 'bloodshedverse.com'
+
+    BASE_URL = 'http://' + SITE_DOMAIN + '/'
+    READ_URL_TEMPLATE = BASE_URL + 'stories.php?go=read&no=%s'
+
+    STARTED_DATETIME_FORMAT = '%m/%d/%Y'
+    UPDATED_DATETIME_FORMAT = '%m/%d/%Y %I:%M'
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        query_data = urlparse.parse_qs(self.parsedUrl.query)
+        story_no = query_data['no'][0]
+
+        self.story.setMetadata('storyId', story_no)
+        self._setURL(self.READ_URL_TEMPLATE % story_no)
+        self.story.setMetadata('siteabbrev', self.SITE_ABBREVIATION)
+
+    def _customized_fetch_url(self, url, exception, parameters=None):
+        try:
+            data = self._fetchUrl(url, parameters)
+        except urllib2.HTTPError:
+            raise exception(self.url)
+
+        return BeautifulSoup.BeautifulSoup(data)
+
+    @staticmethod
+    def getSiteDomain():
+        return BloodshedverseComAdapter.SITE_DOMAIN
+
+    @classmethod
+    def getSiteExampleURLs(cls):
+        return cls.READ_URL_TEMPLATE % 1234
+
+    def getSiteURLPattern(self):
+        return re.escape(self.BASE_URL + 'stories.php?go=') + r'(read|chapters)\&no=\d+$'
+
+    # Override stripURLParameters so the "no" parameter won't get stripped
+    @classmethod
+    def stripURLParameters(cls, url):
+        return url
+
+    def extractChapterUrlsAndMetadata(self):
+        # Use FailedToDownload exception because that's the only error that can
+        # occur here, missing stories don't return a 404 header
+        soup = self._customized_fetch_url(self.url, exceptions.FailedToDownload)
+
+        # Since no 404 error code we have to raise the exception ourselves.
+        # A title that is just 'by' indicates that there is no author name
+        # and no story title available.
+        if soup.title.string.strip() == 'by':
+            raise exceptions.StoryDoesNotExist(self.url)
+
+        for option in soup.find('select', {'name': 'chapter'}):
+            title = option.string.strip()
+            url = self.READ_URL_TEMPLATE % option['value']
+            self.chapterUrls.append((title, url))
+
+        # Get the URL to the author's page and find the correct story entry to
+        # scrape the metadata
+        author_url = urlparse.urljoin(self.url, soup.find('a', {'class': 'headline'})['href'])
+        soup = self._customized_fetch_url(author_url, exceptions.FailedToDownload)
+
+        story_no = self.story.getMetadata('storyId')
+        # Ignore first list_box div, it only contains the author information
+        for list_box in soup('div', {'class': 'list_box'})[1:]:
+            url = list_box.find('a', {'class': 'fictitle'})['href']
+            query_data = _get_query_data(url)
+
+            # Found the div containing the story's metadata; break the loop and
+            # parse the element
+            if query_data['no'] == story_no:
+                break
+
+        title_anchor = list_box.find('a', {'class': 'fictitle'})
+        self.story.setMetadata('title', title_anchor.string.strip())
+
+        author_anchor = title_anchor.findNextSibling('a')
+        self.story.setMetadata('author', author_anchor.string.strip())
+        self.story.setMetadata('authorId', _get_query_data(author_anchor['href'])['who'])
+        self.story.setMetadata('authorUrl', urlparse.urljoin(self.url, author_anchor['href']))
+
+        list_review = list_box.find('div', {'class': 'list_review'})
+        reviews = list_review.a.string.strip().split(' ', 1)[0]
+        self.story.setMetadata('reviews', reviews)
+
+        summary_div = list_box.find('div', {'class': 'list_summary'})
+        if not self.getConfig('keep_summary_html'):
+            summary = ''.join(summary_div(text=True))
+        else:
+            summary = self.utf8FromSoup(author_url, summary_div)
+
+        self.story.setMetadata('description', summary)
+
+        # I'm assuming this to be the category, not sure what else it could be
+        first_listinfo = list_box.find('div', {'class': 'list_info'})
+        self.story.addToList('category', first_listinfo.a.string.strip())
+
+        for list_info in first_listinfo.findNextSiblings('div', {'class': 'list_info'}):
+            for b_tag in list_info('b'):
+                key = b_tag.string.strip(': ')
+                # Strip colons from the beginning, superfluous spaces and minus
+                # characters from the end, and possibly trailing commas from
+                # the warnings if only one is present
+                value = b_tag.nextSibling.string.strip(': -,')
+
+                if key == 'Genre':
+                    for genre in value.split(', '):
+                        # Ignore the "none" genre
+                        if not genre == 'none':
+                            self.story.addToList('genre', genre)
+
+                elif key == 'Rating':
+                    self.story.setMetadata('rating', value)
+
+                elif key == 'Complete':
+                    self.story.setMetadata('status', 'Completed' if value == 'Yes' else 'In-Progress')
+
+                elif key == 'Warning':
+                    for warning in value.split(', '):
+                        # The string here starts with ", " before the actual list
+                        # of values sometimes, so check for an empty warning
+                        # and ignore the "none" warning.
+                        if not warning or warning == 'none':
+                            continue
+
+                        self.story.addToList('warnings', warning)
+
+                elif key == 'Chapters':
+                    self.story.setMetadata('numChapters', int(value))
+
+                elif key == 'Words':
+                    # Apparently only numChapters need to be an integer for
+                    # some strange reason. Remove possible ',' characters as to
+                    # not confuse the codebase down the line
+                    self.story.setMetadata('numWords', value.replace(',', ''))
+
+                elif key == 'Started':
+                    self.story.setMetadata('datePublished', makeDate(value, self.STARTED_DATETIME_FORMAT))
+
+                elif key == 'Updated':
+                    date_string, period = value.rsplit(' ', 1)
+                    date = makeDate(date_string, self.UPDATED_DATETIME_FORMAT)
+
+                    # Rather ugly hack to work around Calibre's changing of
+                    # Python's locale setting, causing am/pm to not be properly
+                    # parsed by strptime() when using a non-english locale
+                    if period == 'pm':
+                        date += timedelta(hours=12)
+                    self.story.setMetadata('dateUpdated', date)
+
+        if self.story.getMetadata('rating') == 'NC-17' and not (self.is_adult or self.getConfig('is_adult')):
+            raise exceptions.AdultCheckRequired(self.url)
+
+    def getChapterText(self, url):
+        soup = self._customized_fetch_url(url, exceptions.FailedToDownload)
+        storytext_div = soup.find('div', {'class': 'storytext'})
+
+        if self.getConfig('strip_text_links'):
+            for anchor in storytext_div('a', {'class': 'FAtxtL'}):
+                navigable_string = BeautifulSoup.NavigableString(anchor.string)
+                anchor.replaceWith(navigable_string)
+
+        return self.utf8FromSoup(url, storytext_div)

fanficdownloader/adapters/adapter_buffynfaithnet.py

@@ -87,7 +87,7 @@ class BuffyNFaithNetAdapter(BaseSiteAdapter):
     
     def setHeader(self):
         "buffynfaith.net wants a Referer for images.  Used both above and below(after cookieproc added)"
-        self.opener.addheaders = [('Referer', 'http://'+self.getSiteDomain()+'/')]
+        self.opener.addheaders.append(('Referer', 'http://'+self.getSiteDomain()+'/'))
         
     @classmethod
     def getSiteExampleURLs(self):

fanficdownloader/adapters/adapter_darksolaceorg.py

@@ -182,6 +182,11 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
 
         # first a tag in pagetitle is title
         self.story.setMetadata('title',stripHTML(div.find('a')))
+        div.find('a').extract()
+        # only thing left in div(pagetitle) now should be 'by' and rating.
+        rating = stripHTML(div)
+        if '[' in rating:
+            self.story.setMetadata('rating', rating[rating.index('[')+1:-1])
 
         for chapa in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+
                                                        self.story.getMetadata('storyId')+'&chapter=\d+')):
@@ -234,31 +239,28 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
                 self.setDescription(url,svalue)
                 #self.story.setMetadata('description',stripHTML(svalue))
 
-            if 'Rated' in label:
-                self.story.setMetadata('rating', value[:len(value)-2])
-
             if 'Word count' in label:
                 self.story.setMetadata('numWords', value)
 
             if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'categories.php\?catid=\d+'))
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
                 for cat in cats:
                     self.story.addToList('category',cat.string)
 
             if 'Characters' in label:
-                for char in value.string.split(', '):
-                    if not 'None' in char:
-                        self.story.addToList('characters',char)
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                for char in chars:
+                    self.story.addToList('characters',char.string)
 
             if 'Genre' in label:
-                for genre in value.string.split(', '):
-                    if not 'None' in genre:
-                        self.story.addToList('genre',genre)
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
+                for genre in genres:
+                    self.story.addToList('genre',genre.string)
 
             if 'Warnings' in label:
-                for warning in value.string.split(', '):
-                    if not 'None' in warning:
-                        self.story.addToList('warnings',warning)
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
+                for warning in warnings:
+                    self.story.addToList('warnings',warning.string)
 
             if 'Completed' in label:
                 if 'Yes' in value:

fanficdownloader/adapters/adapter_fanfictionnet.py

@@ -16,6 +16,7 @@
 #
 
 import time
+from datetime import datetime
 import logging
 logger = logging.getLogger(__name__)
 import re
@@ -43,7 +44,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
 
         # normalized story URL.
-        self._setURL("http://"+self.getSiteDomain()\
+        self._setURL("https://"+self.getSiteDomain()\
                          +"/s/"+self.story.getMetadata('storyId')+"/1/")
 
         # ffnet update emails have the latest chapter URL.
@@ -52,9 +53,11 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         # chapter list doesn't get the latest.  So save and use the
         # original URL given to pull chapter list & metadata.
         self.origurl = url
-        if "http://m." in self.origurl:
+        if "https://m." in self.origurl:
             ## accept m(mobile)url, but use www.
-            self.origurl = self.origurl.replace("http://m.","http://www.")
+            self.origurl = self.origurl.replace("https://m.","https://www.")
+
+        self.opener.addheaders.append(('Referer',self.origurl))
 
     @staticmethod
     def getSiteDomain():
@@ -66,10 +69,17 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
 
     @classmethod
     def getSiteExampleURLs(self):
-        return "http://www.fanfiction.net/s/1234/1/ http://www.fanfiction.net/s/1234/12/ http://www.fanfiction.net/s/1234/1/Story_Title http://m.fanfiction.net/s/1234/1/"
+        return "https://www.fanfiction.net/s/1234/1/ https://www.fanfiction.net/s/1234/12/ http://www.fanfiction.net/s/1234/1/Story_Title http://m.fanfiction.net/s/1234/1/"
 
     def getSiteURLPattern(self):
-        return r"http://(www|m)?\.fanfiction\.net/s/\d+(/\d+)?(/|/[^/]+)?/?$"
+        return r"https?://(www|m)?\.fanfiction\.net/s/\d+(/\d+)?(/|/[^/]+)?/?$"
+
+    def _fetchUrl(self,url):
+        time.sleep(1.0) ## ffnet(and, I assume, fpcom) tends to fail
+                        ## more if hit too fast.  This is in
+                        ## additional to what ever the
+                        ## slow_down_sleep_time setting is.
+        return BaseSiteAdapter._fetchUrl(self,url)
 
     def extractChapterUrlsAndMetadata(self):
 
@@ -97,32 +107,35 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         if "not found. Please check to see you are not using an outdated url." in data:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  'Chapter not found. Please check to see you are not using an outdated url.'" % url)
 
-        try:
-            # rather nasty way to check for a newer chapter.  ffnet has a
-            # tendency to send out update notices in email before all
-            # their servers are showing the update on the first chapter.
+        if self.getConfig('check_next_chapter'):
             try:
-                chapcount = len(soup.find('select', { 'name' : 'chapter' } ).findAll('option'))
-            # get chapter part of url.
+                ## ffnet used to have a tendency to send out update
+                ## notices in email before all their servers were
+                ## showing the update on the first chapter.  It
+                ## generates another server request and doesn't seem
+                ## to be needed lately, so now default it to off.
+                try:
+                    chapcount = len(soup.find('select', { 'name' : 'chapter' } ).findAll('option'))
+                # get chapter part of url.
+                except:
+                    chapcount = 1
+                chapter = url.split('/',)[5]
+                tryurl = "https://%s/s/%s/%d/"%(self.getSiteDomain(),
+                                                self.story.getMetadata('storyId'),
+                                                chapcount+1)
+                logger.debug('=Trying newer chapter: %s' % tryurl)
+                newdata = self._fetchUrl(tryurl)
+                if "not found. Please check to see you are not using an outdated url." \
+                        not in newdata:
+                    logger.debug('=======Found newer chapter: %s' % tryurl)
+                    soup = bs.BeautifulSoup(newdata)
             except:
-                chapcount = 1
-            chapter = url.split('/',)[5]
-            tryurl = "http://%s/s/%s/%d/"%(self.getSiteDomain(),
-                                           self.story.getMetadata('storyId'),
-                                           chapcount+1)
-            logger.debug('=Trying newer chapter: %s' % tryurl)
-            newdata = self._fetchUrl(tryurl)
-            if "not found. Please check to see you are not using an outdated url." \
-                    not in newdata:
-                logger.debug('=======Found newer chapter: %s' % tryurl)
-                soup = bs.BeautifulSoup(newdata)
-        except:
-            pass
-        
+                pass
+            
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"^/u/\d+"))
         self.story.setMetadata('authorId',a['href'].split('/')[2])
-        self.story.setMetadata('authorUrl','http://'+self.host+a['href'])
+        self.story.setMetadata('authorUrl','https://'+self.host+a['href'])
         self.story.setMetadata('author',a.string)
 
         ## Pull some additional data from html.
@@ -141,9 +154,9 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
             # of Book, Movie, etc.
             self.story.addToList('category',stripHTML(categories[1]))
         elif 'Crossover' in categories[0]['href']:
-            caturl = "http://%s%s"%(self.getSiteDomain(),categories[0]['href'])
+            caturl = "https://%s%s"%(self.getSiteDomain(),categories[0]['href'])
             catsoup = bs.BeautifulSoup(self._fetchUrl(caturl))
-            for a in catsoup.findAll('a',href=re.compile(r"^/crossovers/")):
+            for a in catsoup.findAll('a',href=re.compile(r"^/crossovers/.+?/\d+/")):
                 self.story.addToList('category',stripHTML(a))
             else:
                 # Fall back.  I ran across a story with a Crossver
@@ -156,7 +169,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
                 
                 
             
-        a = soup.find('a', href='http://www.fictionratings.com/')
+        a = soup.find('a', href=re.compile(r'https?://www\.fictionratings\.com/'))
         rating = a.string
         if 'Fiction' in rating: # if rating has 'Fiction ', strip that out for consistency with past.
             rating = rating[8:]
@@ -165,7 +178,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
 
         # after Rating, the same bit of text containing id:123456 contains
         # Complete--if completed.
-        gui_table1i = soup.find('table',{'cellpadding':'5'})
+        gui_table1i = soup.find('div',{'id':'content_wrapper_inner'})
 
         self.story.setMetadata('title', stripHTML(gui_table1i.find('b'))) # title appears to be only(or at least first) bold tag in gui_table1i
 
@@ -205,9 +218,17 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
             self.story.extendList('genre',genrelist)
             metalist=metalist[1:]
 
+        # Updated: <span data-xutime='1368059198'>5/8</span> - Published: <span data-xutime='1278984264'>7/12/2010</span>
+        # Published: <span data-xutime='1384358726'>8m ago</span>
+        dates = soup.findAll('span',{'data-xutime':re.compile(r'^\d+$')})
+        if len(dates) > 1 :
+            # updated get set to the same as published upstream if not found.
+            self.story.setMetadata('dateUpdated',datetime.fromtimestamp(float(dates[0]['data-xutime'])))
+        self.story.setMetadata('datePublished',datetime.fromtimestamp(float(dates[-1]['data-xutime'])))
+            
         donechars = False
         while len(metalist) > 0:
-            if  metalist[0].startswith('Chapters') or metalist[0].startswith('Status') or metalist[0].startswith('id:'):
+            if  metalist[0].startswith('Chapters') or metalist[0].startswith('Status') or metalist[0].startswith('id:') or metalist[0].startswith('Updated:') or metalist[0].startswith('Published:'):
                 pass
             elif  metalist[0].startswith('Reviews'):
                 self.story.setMetadata('reviews',metalist[0].split(':')[1].strip())
@@ -215,10 +236,6 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
                 self.story.setMetadata('favs',metalist[0].split(':')[1].strip())
             elif  metalist[0].startswith('Follows:'):
                 self.story.setMetadata('follows',metalist[0].split(':')[1].strip())
-            elif  metalist[0].startswith('Updated'):
-                self.story.setMetadata('dateUpdated',makeDate(metalist[0].split(':')[1].strip(), '%m-%d-%y'))
-            elif  metalist[0].startswith('Published'):
-                self.story.setMetadata('datePublished',makeDate(metalist[0].split(':')[1].strip(), '%m-%d-%y'))
             elif  metalist[0].startswith('Words'):
                 self.story.setMetadata('numWords',metalist[0].split(':')[1].strip())
             elif not donechars:
@@ -257,9 +274,9 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         else:
             allOptions = select.findAll('option')
             for o in allOptions:
-                url = u'http://%s/s/%s/%s/' % ( self.getSiteDomain(),
-                                            self.story.getMetadata('storyId'),
-                                            o['value'])
+                url = u'https://%s/s/%s/%s/' % ( self.getSiteDomain(),
+                                                 self.story.getMetadata('storyId'),
+                                                 o['value'])
                 # just in case there's tags, like <i> in chapter titles.
                 title = u"%s" % o
                 title = re.sub(r'<[^>]+>','',title)
@@ -270,7 +287,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         return
 
     def getChapterText(self, url):
-        time.sleep(5.0) ## ffnet(and, I assume, fpcom) tends to fail
+        time.sleep(4.0) ## ffnet(and, I assume, fpcom) tends to fail
                         ## more if hit too fast.  This is in
                         ## additional to what ever the
                         ## slow_down_sleep_time setting is.

fanficdownloader/adapters/adapter_fictionpadcom.py

@@ -133,6 +133,7 @@ class FictionPadSiteAdapter(BaseSiteAdapter):
         author = tables['users'][0]
         story = tables['stories'][0]
         story_ver = tables['story_versions'][0]
+        print("story:%s"%story)
         
         self.story.setMetadata('authorId',author['id'])
         self.story.setMetadata('author',author['display_name'])
@@ -151,7 +152,8 @@ class FictionPadSiteAdapter(BaseSiteAdapter):
         self.story.setMetadata('comments',story['comments_count'])
         self.story.setMetadata('views',story['views_count'])
         self.story.setMetadata('likes',int(story['likes'])) # no idea why they floated these.
-        self.story.setMetadata('dislikes',int(story['dislikes']))
+        if 'dislikes' in story:
+            self.story.setMetadata('dislikes',int(story['dislikes']))
 
         if story_ver['is_complete']:
             self.story.setMetadata('status', 'Completed')

fanficdownloader/adapters/adapter_fictionpresscom.py

@@ -41,10 +41,10 @@ class FictionPressComSiteAdapter(FanFictionNetSiteAdapter):
 
     @classmethod
     def getSiteExampleURLs(self):
-        return "http://www.fictionpress.com/s/1234/1/ http://www.fictionpress.com/s/1234/12/ http://www.fictionpress.com/s/1234/1/Story_Title http://m.fictionpress.com/s/1234/1/"
+        return "https://www.fictionpress.com/s/1234/1/ https://www.fictionpress.com/s/1234/12/ http://www.fictionpress.com/s/1234/1/Story_Title http://m.fictionpress.com/s/1234/1/"
 
     def getSiteURLPattern(self):
-        return r"http://(www|m)?\.fictionpress\.com/s/\d+(/\d+)?(/|/[a-zA-Z0-9_-]+)?/?$"
+        return r"https?://(www|m)?\.fictionpress\.com/s/\d+(/\d+)?(/|/[a-zA-Z0-9_-]+)?/?$"
 
 def getClass():
     return FictionPressComSiteAdapter

fanficdownloader/adapters/adapter_ficwadcom.py

@@ -85,7 +85,11 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
 
         # use BeautifulSoup HTML parser to make everything easier to find.
         try:
-            soup = bs.BeautifulSoup(self._fetchUrl(url))
+            data = self._fetchUrl(url)
+            # non-existent/removed story urls get thrown to the front page.
+            if "<h2>Welcome to FicWad</h2>" in data:
+                raise exceptions.StoryDoesNotExist(self.url)
+            soup = bs.BeautifulSoup(data)
         except urllib2.HTTPError, e:
             if e.code == 404:
                 raise exceptions.StoryDoesNotExist(self.url)

fanficdownloader/adapters/adapter_fimfictionnet.py

@@ -59,7 +59,7 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
         return "http://www.fimfiction.net/story/1234/story-title-here http://www.fimfiction.net/story/1234/ http://www.fimfiction.com/story/1234/1/ http://mobile.fimfiction.net/story/1234/1/story-title-here/chapter-title-here"
 
     def getSiteURLPattern(self):
-        return r"http://(www|mobile)\.fimfiction\.(net|com)/story/\d+/?.*"
+        return r"https?://(www|mobile)\.fimfiction\.(net|com)/story/\d+/?.*"
         
     def extractChapterUrlsAndMetadata(self):
         
@@ -85,7 +85,7 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
             
             # Unfortunately, we still need to load the story index
             # page to parse the characters.  And chapters, now, too.
-            data = self._fetchUrl(self.url)
+            data = self.do_fix_blockquotes(self._fetchUrl(self.url))
             soup = bs.BeautifulSoup(data)
         except urllib2.HTTPError, e:
             if e.code == 404:
@@ -101,7 +101,7 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
         # if "/images/missing_story.png" in data:
         #     raise exceptions.StoryDoesNotExist(self.url)
         
-        if "This story has been marked as having adult content." in data:
+        if "This story has been marked as having adult content. Please click below to confirm you are of legal age to view adult material in your country." in data:
             raise exceptions.AdultCheckRequired(self.url)
         
         if self.password:
@@ -249,22 +249,55 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
                     value = unicode(value)
                 self.story.setMetadata(metakey, value)
 
-        rawGroupList = soup.find('ul', {'id':'story_group_list'})
-        if rawGroupList is not None:
-            for groupName in rawGroupList.findAll('a', {'href':re.compile('^/group/')}):
+        ## Groups and sequels code from FaceDeer
+        allGroupLists = soup.findAll('ul', {'id':'story_group_list'})
+        for groupList in allGroupLists:
+            for groupName in groupList.findAll('a', {'href':re.compile('^/group/')}):
+                self.story.addToList("groupsUrl", 'http://'+self.host+groupName["href"]) 
                 self.story.addToList("groups",stripHTML(groupName).replace(',', ';'))
-            
+
+        sequelStoryHeader = soup.find('h1', {'class':'header-stories'}, text="Sequels")
+        if not sequelStoryHeader == None:
+            sequelContainer = sequelStoryHeader.parent.parent
+            for sequel in sequelContainer.findAll('a', {'class':'story_link'}):
+                self.story.addToList("sequelsUrl", 'http://'+self.host+sequel["href"]) 
+                self.story.addToList("sequels", stripHTML(sequel).replace(',', ';'))
+                
+        #The link to the prequel is embedded in the description text, so erring
+        #on the side of caution and wrapping this whole thing in a try block.
+        #If anything goes wrong this probably wasn't a valid prequel link.
+        try:
+            description = soup.find('div', {'class':'description'})
+            firstHR = description.find("hr")
+            nextSib = firstHR.nextSibling
+            if "This story is a sequel to" in nextSib.string:
+                link = nextSib.nextSibling
+                if link.name == "a":
+                    self.story.setMetadata("prequelUrl", 'http://'+self.host+link["href"])
+                    self.story.setMetadata("prequel", stripHTML(link))
+        except:
+            pass
+        
     def hookForUpdates(self,chaptercount):
         if self.oldchapters and len(self.oldchapters) > self.newestChapterNum:
             print("Existing epub has %s chapters\nNewest chapter is %s.  Discarding old chapters from there on."%(len(self.oldchapters), self.newestChapterNum+1))
             self.oldchapters = self.oldchapters[:self.newestChapterNum]
         return len(self.oldchapters)
-            
+
+    def do_fix_blockquotes(self,data):
+        if self.getConfig('fix_fimf_blockquotes'):
+            # <p class="double"><blockquote>
+            # </blockquote></p>
+            # include > in re groups so there's always something in the group.
+            data = re.sub(r'<p([^>]*>\s*)<blockquote([^>]*>)',r'<blockquote\2<p\1',data)
+            data = re.sub(r'</blockquote(>\s*)</p>',r'</p\1</blockquote>',data)
+        return data
         
     def getChapterText(self, url):
         logger.debug('Getting chapter text from: %s' % url)
-        
-        soup = bs.BeautifulSoup(self._fetchUrl(url),selfClosingTags=('br','hr')).find('div', {'class' : 'chapter_content'})
+
+        data = self.do_fix_blockquotes(self._fetchUrl(url))
+        soup = bs.BeautifulSoup(data,selfClosingTags=('br','hr')).find('div', {'class' : 'chapter_content'})
         if soup == None:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
         return self.utf8FromSoup(url,soup)

fanficdownloader/adapters/adapter_finestoriescom.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-# Copyright 2012 Fanficdownloader team
+# Copyright 2013 Fanficdownloader team
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -37,11 +37,6 @@ class FineStoriesComAdapter(BaseSiteAdapter):
     def __init__(self, config, url):
         BaseSiteAdapter.__init__(self, config, url)
 
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
         self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
         self.password = ""
         self.is_adult=False
@@ -51,7 +46,6 @@ class FineStoriesComAdapter(BaseSiteAdapter):
         if 'storyInfo' in self.story.getMetadata('storyId'):
             self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
         
-        
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/s/storyInfo.php?id='+self.story.getMetadata('storyId'))
         

fanficdownloader/adapters/adapter_hpfanficarchivecom.py

@@ -128,23 +128,22 @@ class HPFanficArchiveComAdapter(BaseSiteAdapter):
         # <span class="label">Rated:</span> NC-17<br /> etc
         labels = soup.findAll('span',{'class':'label'})
         for labelspan in labels:
-            value = labelspan.nextSibling
+            val = labelspan.nextSibling
+            value = unicode('')
+            while val and not defaultGetattr(val,'class') == 'label':
+                value += unicode(val)
+                val = val.nextSibling
             label = labelspan.string
+            #print("label:%s\nvalue:%s"%(label,value))
 
             if 'Summary' in label:
-                ## Everything until the next span class='label'
-                svalue = ""
-                while not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
-                    value = value.nextSibling
-                self.setDescription(url,svalue)
-                #self.story.setMetadata('description',stripHTML(svalue))
+                self.setDescription(url,value)
 
             if 'Rated' in label:
-                self.story.setMetadata('rating', value)
+                self.story.setMetadata('rating', stripHTML(value))
 
             if 'Word count' in label:
-                self.story.setMetadata('numWords', value)
+                self.story.setMetadata('numWords', stripHTML(value))
 
             if 'Categories' in label:
                 cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
@@ -161,13 +160,18 @@ class HPFanficArchiveComAdapter(BaseSiteAdapter):
                 for genre in genres:
                     self.story.addToList('genre',genre.string)
 
+            if 'Pairing' in label:
+                ships = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=4'))
+                for ship in ships:
+                    self.story.addToList('ships',ship.string)
+
             if 'Warnings' in label:
                 warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2')) # XXX
                 for warning in warnings:
                     self.story.addToList('warnings',warning.string)
 
             if 'Completed' in label:
-                if 'Yes' in value:
+                if 'Yes' in stripHTML(value):
                     self.story.setMetadata('status', 'Completed')
                 else:
                     self.story.setMetadata('status', 'In-Progress')

fanficdownloader/adapters/adapter_literotica.py

@@ -0,0 +1,247 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+import urlparse
+import time
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter, makeDate
+
+class LiteroticaSiteAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["utf8",
+                       "Windows-1252"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        
+        self.story.setMetadata('siteabbrev','litero')
+
+        # normalize to first chapter.  Not sure if they ever have more than 2 digits.
+        storyid = self.parsedUrl.path.split('/',)[2]
+        if re.match(r'-ch\d\d$',storyid):
+            storyid = storyid[:-2]+'01'
+        self.story.setMetadata('storyId',storyid)
+        
+        self.origurl = url
+        if "//www.i." in self.origurl:
+            ## accept m(mobile)url, but use www.
+            self.origurl = self.origurl.replace("//www.i.","//www.")
+
+        # normalized story URL.
+        self._setURL(url[:url.index('//')+2]+self.getSiteDomain()\
+                         +"/s/"+self.story.getMetadata('storyId'))
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = '%m/%d/%y'
+        
+    @staticmethod
+    def getSiteDomain():
+        return 'www.literotica.com'
+
+    @classmethod
+    def getAcceptDomains(cls):
+        return ['www.literotica.com', 'www.i.literotica.com']
+
+    @classmethod
+    def getSiteExampleURLs(self):
+        #return "http://www.literotica.com/s/story-title http://www.literotica.com/stories/showstory.php?id=1234 http://www.i.literotica.com/stories/showstory.php?id=1234"
+        return "http://www.literotica.com/s/story-title https://www.literotica.com/s/story-title"
+
+    def getSiteURLPattern(self):
+        return r"https?://www(\.i)?\.literotica\.com/s/([a-zA-Z0-9_-]+)"
+
+    def extractChapterUrlsAndMetadata(self):
+
+        if not (self.is_adult or self.getConfig("is_adult")):
+            raise exceptions.AdultCheckRequired(self.url)
+        
+        url1 = self.origurl
+        logger.debug("first page URL: "+url1)
+
+        try:
+            data1 = self._fetchUrl(url1)
+            soup1 = bs.BeautifulSoup(data1)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(url1)
+            else:
+                raise e
+
+        #strip comments from soup
+        [comment.extract() for comment in soup1.findAll(text=lambda text:isinstance(text, bs.Comment))]
+
+        # author
+        a = soup1.find("span", "b-story-user-y")
+        self.story.setMetadata('authorId', urlparse.parse_qs(a.a['href'].split('?')[1])['uid'])
+        authorurl = a.a['href']
+        if authorurl.startswith('//'):
+            authorurl = self.parsedUrl.scheme+':'+authorurl
+        self.story.setMetadata('authorUrl', authorurl)
+        self.story.setMetadata('author', a.text)
+
+        # get the author page
+        try:
+            dataAuth = self._fetchUrl(authorurl)
+            soupAuth = bs.BeautifulSoup(dataAuth)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(authorurl)
+            else:
+                raise e
+
+        ## site has started using //domain.name/asdf urls remove https?: from front
+        storyLink = soupAuth.find('a', href=url1[url1.index(':')+1:])
+
+        if storyLink is not None:
+            # pull the published date from the author page
+            # default values from single link.  Updated below if multiple chapter.
+            date = storyLink.parent.parent.findAll('td')[-1].text
+            self.story.setMetadata('datePublished', makeDate(date, self.dateformat))
+            self.story.setMetadata('dateUpdated',makeDate(date, self.dateformat))
+
+        # find num of pages
+        # find a "3 Pages:" string on the page and parse it
+        pgs = soup1.find("span", "b-pager-caption-t r-d45").string.split(' ')[0]
+
+        # If there are multiple pages, find and request the last page
+        if "1" != pgs:
+            logger.debug("last page number: "+pgs)
+            try:
+                data2 = self._fetchUrl(url1, {'page': pgs})
+                soup2 = bs.BeautifulSoup(data2)
+                [comment.extract() for comment in soup2.findAll(text=lambda text:isinstance(text, bs.Comment))]
+            except urllib2.HTTPError, e:
+                if e.code == 404:
+                    # TODO: Probably should reformat this
+                    raise exceptions.StoryDoesNotExist(url1, {'page': pgs})
+                else:
+                    raise e
+        else:
+            #If we're already on the last page, copy the soup
+            soup2 = soup1
+
+        # parse out the list of chapters
+        chaps = soup2.find('div', id='b-series')
+        if chaps:  # may be one post only
+            #self.chapterUrls = [(ch.a.text, ch.a['href']) for ch in chaps.findAll('li')]
+
+            # if there are chapters, lets pull them and title from the
+            # author page because *this* chapter is omitted from the
+            # list on the last page.
+            row = storyLink.parent.parent.previousSibling
+            while row['class'] != 'ser-ttl':
+                row = row.previousSibling
+
+            seriesTitle = stripHTML(row)
+            if seriesTitle:
+                # this regex is deliberately greedy. We want to get the biggest match before a ':'
+                self.story.setMetadata('title', re.match('(.*):[^:]*$', seriesTitle).group(1))
+            else:
+                self.story.setMetadata('title', soup1.h1.string)
+
+            # now chapter list.  Assumed oldest to newest.
+            self.chapterUrls = []
+            row = row.nextSibling
+                
+            self.story.setMetadata('datePublished',makeDate(stripHTML(row.find('td',{'class':'dt'})), self.dateformat))
+            while row['class'] == 'sl':
+                # pages include full URLs.
+                chapurl = row.a['href']
+                if chapurl.startswith('//'):
+                    chapurl = self.parsedUrl.scheme+':'+chapurl
+                self.chapterUrls.append((row.a.string,chapurl))
+                if not row.nextSibling:
+                    break
+                row = row.nextSibling
+
+            row = row.previousSibling
+            self.story.setMetadata('dateUpdated',makeDate(stripHTML(row.find('td',{'class':'dt'})), self.dateformat))
+                
+        else:  # if one post only
+            self.chapterUrls = [(soup1.h1.string, url1)]
+            self.story.setMetadata('title', soup1.h1.string)
+
+        # normalize on first chapter URL.
+        self._setURL(self.chapterUrls[0][1])
+
+        # reset storyId to first chapter.
+        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
+
+        self.story.setMetadata('numChapters', len(self.chapterUrls))
+
+        self.story.setMetadata('category', soup1.find('div', 'b-breadcrumbs').findAll('a')[1].string)
+        self.story.setMetadata('description', soup1.find('meta', {'name': 'description'})['content'])
+
+        return
+
+    def getChapterText(self, url):
+        logger.debug('Getting chapter text from: %s' % url)
+        time.sleep(0.5)
+        data1 = self._fetchUrl(url)
+        soup1 = bs.BeautifulSoup(data1)
+
+        #strip comments from soup
+        [comment.extract() for comment in soup1.findAll(text=lambda text:isinstance(text, bs.Comment))]
+
+        # get story text
+        story1 = soup1.find('div', 'b-story-body-x').p
+        story1.name='div'
+        story1.append('<br />')
+        storytext = self.utf8FromSoup(url,story1)
+
+        # find num pages
+        pgs = int(soup1.find("span", "b-pager-caption-t r-d45").string.split(' ')[0])
+        logger.debug("pages: "+str(pgs))
+
+        # get all the pages
+        for i in xrange(2, pgs+1):
+            try:
+                logger.debug("fetching page "+str(i))
+                time.sleep(0.5)
+                data2 = self._fetchUrl(url, {'page': i})
+                soup2 = bs.BeautifulSoup(data2)
+                [comment.extract() for comment in soup2.findAll(text=lambda text:isinstance(text, bs.Comment))]
+                story2 = soup2.find('div', 'b-story-body-x').p
+                story2.name='div'
+                story2.append('<br />')
+                storytext += self.utf8FromSoup(url,story2)
+            except urllib2.HTTPError, e:
+                if e.code == 404:
+                    raise exceptions.StoryDoesNotExist(url)
+                else:
+                    raise e
+        return storytext
+
+
+def getClass():
+    return LiteroticaSiteAdapter
+
+
+

fanficdownloader/adapters/adapter_onedirectionfanfictioncom.py

@@ -192,7 +192,7 @@ class OneDirectionFanfictionComAdapter(BaseSiteAdapter):
             if 'Summary' in label:
                 ## Everything until the next span class='label'
                 svalue = ""
-                while not defaultGetattr(value,'class') == 'label':
+                while value and not defaultGetattr(value,'class') == 'label':
                     svalue += str(value)
                     value = value.nextSibling
                 self.setDescription(url,svalue)

fanficdownloader/adapters/adapter_potionsandsnitchesnet.py

@@ -163,12 +163,11 @@ class PotionsAndSnitchesNetSiteAdapter(BaseSiteAdapter):
                     self.story.setMetadata('status', 'In-Progress')
 
             if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), "%d %b %Y"))
+                # limit date values, there's some extra chars.
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value[:12]), "%d %b %Y"))
             
             if 'Updated' in label:
-                # there's a stray [ at the end.
-                #value = value[0:-1]
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), "%d %b %Y"))
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value[:12]), "%d %b %Y"))
 
         try:
             # Find Series name from series URL.

fanficdownloader/adapters/adapter_spikeluvercom.py

@@ -0,0 +1,186 @@
+import re
+import urllib2
+import urlparse
+
+from .. import BeautifulSoup
+
+from base_adapter import BaseSiteAdapter, makeDate
+from .. import exceptions
+
+
+def getClass():
+    return SpikeluverComAdapter
+
+
+class SpikeluverComAdapter(BaseSiteAdapter):
+    SITE_ABBREVIATION = 'slc'
+    SITE_DOMAIN = 'spikeluver.com'
+
+    BASE_URL = 'http://' + SITE_DOMAIN + '/SpuffyRealm/'
+    LOGIN_URL = BASE_URL + 'user.php?action=login'
+    VIEW_STORY_URL_TEMPLATE = BASE_URL + 'viewstory.php?sid=%d'
+    METADATA_URL_SUFFIX = '&index=1'
+    AGE_CONSENT_URL_SUFFIX = '&ageconsent=ok&warning=5'
+
+    DATETIME_FORMAT = '%m/%d/%Y'
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        query_data = urlparse.parse_qs(self.parsedUrl.query)
+        story_id = query_data['sid'][0]
+
+        self.story.setMetadata('storyId', story_id)
+        self._setURL(self.VIEW_STORY_URL_TEMPLATE % int(story_id))
+        self.story.setMetadata('siteabbrev', self.SITE_ABBREVIATION)
+
+    def _customized_fetch_url(self, url, exception, parameters=None):
+        try:
+            data = self._fetchUrl(url, parameters)
+        except urllib2.HTTPError:
+            raise exception(self.url)
+
+        return BeautifulSoup.BeautifulSoup(data)
+
+    @staticmethod
+    def getSiteDomain():
+        return SpikeluverComAdapter.SITE_DOMAIN
+
+    @classmethod
+    def getSiteExampleURLs(cls):
+        return cls.VIEW_STORY_URL_TEMPLATE % 1234
+
+    def getSiteURLPattern(self):
+        return re.escape(self.VIEW_STORY_URL_TEMPLATE[:-2]) + r'\d+$'
+
+    def extractChapterUrlsAndMetadata(self):
+        soup = self._customized_fetch_url(self.url + self.METADATA_URL_SUFFIX, exceptions.StoryDoesNotExist)
+
+        # No additional login is required, just check for adult
+        pagetitle_div = soup.find('div', id='pagetitle')
+        if pagetitle_div.a['href'].startswith('javascript:'):
+            if not(self.is_adult or self.getConfig('is_adult')):
+                raise exceptions.AdultCheckRequired(self.url)
+
+        url = ''.join([self.url, self.METADATA_URL_SUFFIX, self.AGE_CONSENT_URL_SUFFIX])
+        soup = self._customized_fetch_url(url, exceptions.StoryDoesNotExist)
+
+        pagetitle_div = soup.find('div', id='pagetitle')
+        self.story.setMetadata('title', pagetitle_div.a.string.strip())
+
+        author_anchor = pagetitle_div.a.findNextSibling('a')
+        url = urlparse.urljoin(self.BASE_URL, author_anchor['href'])
+        components = urlparse.urlparse(url)
+        query_data = urlparse.parse_qs(components.query)
+
+        self.story.setMetadata('author', author_anchor.string.strip())
+        self.story.setMetadata('authorId', query_data['uid'])
+        self.story.setMetadata('authorUrl', url)
+
+        sort_div = soup.find('div', id='sort')
+        self.story.setMetadata('reviews', sort_div('a')[1].string.strip())
+
+        listbox_tag = soup.find('div', {'class': 'listbox'})
+        for span_tag in listbox_tag('span'):
+            key = span_tag.string.strip(' :')
+            try:
+                value = span_tag.nextSibling.string.strip()
+            # This can happen with some fancy markup in the summary. Just
+            # ignore this error and set value to None, the summary parsing
+            # takes care of this
+            except AttributeError:
+                value = None
+
+            if key == 'Summary':
+                contents = []
+                keep_summary_html = self.getConfig('keep_summary_html')
+
+                # For some reason span_tag.findNextSiblings() only returns tags
+                # instead both NavigableString objects _and_ Tag objects as
+                # documented, so navigate manually
+                sibling = span_tag.nextSibling
+                while sibling:
+                    if isinstance(sibling, BeautifulSoup.Tag):
+                        # Encountered next label, break. Not as bad as other
+                        # e-fiction sites, let's hope this is enough for proper
+                        # parsing.
+                        if sibling.name == 'span' and sibling.get('class', None) == 'label':
+                            break
+
+                        if keep_summary_html:
+                            contents.append(self.utf8FromSoup(self.url, sibling))
+                        else:
+                            contents.append(''.join(sibling(text=True)))
+                    else:
+                        contents.append(sibling)
+                    sibling = sibling.nextSibling
+
+                # Remove the preceding break line tag and other crud
+                contents.pop()
+                contents.pop()
+                self.story.setMetadata('description', ''.join(contents))
+
+            elif key == 'Rated':
+                self.story.setMetadata('rating', value)
+
+            elif key == 'Categories':
+                for sibling in span_tag.findNextSiblings(['a', 'br']):
+                    if sibling.name == 'br':
+                        break
+
+                    self.story.addToList('category', sibling.string.strip())
+
+            # Seems to be always "None" for some reason
+            elif key == 'Characters':
+                for sibling in span_tag.findNextSiblings(['a', 'br']):
+                    if sibling.name == 'br':
+                        break
+                    self.story.addToList('characters', sibling.string.strip())
+
+            elif key == 'Genres':
+                for sibling in span_tag.findNextSiblings(['a', 'br']):
+                    if sibling.name == 'br':
+                        break
+
+                    self.story.addToList('genre', sibling.string.strip())
+
+            elif key == 'Warnings':
+                for sibling in span_tag.findNextSiblings(['a', 'br']):
+                    if sibling.name == 'br':
+                        break
+                    self.story.addToList('warnings', sibling.string.strip())
+
+            # Challenges
+
+            elif key == 'Series':
+                a = span_tag.findNextSibling('a')
+                if not a:
+                    continue
+                self.story.setMetadata('series', a.string.strip())
+                self.story.setMetadata('seriesUrl', urlparse.urljoin(self.BASE_URL, a['href']))
+
+            elif key == 'Chapters':
+                self.story.setMetadata('numChapters', int(value))
+
+            elif key == 'Completed':
+                self.story.setMetadata('status', 'Completed' if value == 'Yes' else 'In-Progress')
+
+            elif key == 'Word count':
+                self.story.setMetadata('numWords', value)
+
+            elif key == 'Published':
+                self.story.setMetadata('datePublished', makeDate(value, self.DATETIME_FORMAT))
+
+            elif key == 'Updated':
+                self.story.setMetadata('dateUpdated', makeDate(value, self.DATETIME_FORMAT))
+
+        for p_tag in listbox_tag.findNextSiblings('p'):
+            chapter_anchor = p_tag.find('a')
+            title = chapter_anchor.string.strip()
+            url = urlparse.urljoin(self.BASE_URL, chapter_anchor['href'])
+            self.chapterUrls.append((title, url))
+
+    def getChapterText(self, url):
+        url += self.AGE_CONSENT_URL_SUFFIX
+        soup = self._customized_fetch_url(url, exceptions.FailedToDownload)
+        return self.utf8FromSoup(url, soup.find('div', id='story'))

fanficdownloader/adapters/adapter_squidgeorgpeja.py

@@ -62,7 +62,7 @@ class SquidgeOrgPejaAdapter(BaseSiteAdapter):
         
         
         # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/peja/cgi-bin/viewstory.php?sid='+self.story.getMetadata('storyId'))
+        self._setURL('https://' + self.getSiteDomain() + '/peja/cgi-bin/viewstory.php?sid='+self.story.getMetadata('storyId'))
         
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','wwomb')
@@ -83,10 +83,10 @@ class SquidgeOrgPejaAdapter(BaseSiteAdapter):
     
     @classmethod
     def getSiteExampleURLs(self):
-        return "http://"+self.getSiteDomain()+"/peja/cgi-bin/viewstory.php?sid=1234"
+        return "https://"+self.getSiteDomain()+"/peja/cgi-bin/viewstory.php?sid=1234"
 
     def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/")+"~?"+re.escape("peja/cgi-bin/viewstory.php?sid=")+r"\d+$"        
+        return r"https?"+re.escape("://"+self.getSiteDomain()+"/")+r"~?"+re.escape("peja/cgi-bin/viewstory.php?sid=")+r"\d+$"        
 
     ## Getting the chapter list and the meta data, plus 'is adult' checking.
     def extractChapterUrlsAndMetadata(self):
@@ -103,6 +103,9 @@ class SquidgeOrgPejaAdapter(BaseSiteAdapter):
                 raise e
 
         data = self._fetchUrl(url)
+
+        if "fatal MySQL error was encountered" in data:
+            raise exceptions.FailedToDownload("Site SQL Error--bad story")
             
         # use BeautifulSoup HTML parser to make everything easier to find.
         soup = bs.BeautifulSoup(data)
@@ -113,7 +116,7 @@ class SquidgeOrgPejaAdapter(BaseSiteAdapter):
         # Find authorid and URL from... author url.
         author = soup.find('div', {'id':"pagetitle"}).find('a')
         self.story.setMetadata('authorId',author['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/peja/cgi-bin/'+author['href'])
+        self.story.setMetadata('authorUrl','https://'+self.host+'/peja/cgi-bin/'+author['href'])
         self.story.setMetadata('author',author.string)
 		
         authorSoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
@@ -128,7 +131,7 @@ class SquidgeOrgPejaAdapter(BaseSiteAdapter):
         chapterselect=soup.find('select',{'name':'chapter'})
         if chapterselect:
             for ch in chapterselect.findAll('option'):
-                self.chapterUrls.append((stripHTML(ch),'http://'+self.host+'/peja/cgi-bin/viewstory.php?sid='+self.story.getMetadata('storyId')+'&chapter='+ch['value']))
+                self.chapterUrls.append((stripHTML(ch),'https://'+self.host+'/peja/cgi-bin/viewstory.php?sid='+self.story.getMetadata('storyId')+'&chapter='+ch['value']))
         else:
             self.chapterUrls.append((title,url))
 		
@@ -204,7 +207,7 @@ class SquidgeOrgPejaAdapter(BaseSiteAdapter):
             # http://www.squidge.org/peja/cgi-bin/series.php?seriesid=254
             a = titleblock.find('a', href=re.compile(r"series.php\?seriesid=\d+"))
             series_name = a.string
-            series_url = 'http://'+self.host+'/peja/cgi-bin/'+a['href']
+            series_url = 'https://'+self.host+'/peja/cgi-bin/'+a['href']
     
             # use BeautifulSoup HTML parser to make everything easier to find.
             seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))

fanficdownloader/adapters/adapter_storiesonlinenet.py

@@ -37,11 +37,6 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
     def __init__(self, config, url):
         BaseSiteAdapter.__init__(self, config, url)
 
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
         self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
         self.password = ""
         self.is_adult=False
@@ -71,7 +66,7 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
         return "http://"+self.getSiteDomain()+"/s/1234 http://"+self.getSiteDomain()+"/s/1234:4010"
 
     def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain())+r"/s/\d+((:\d+)?(;\d+)?$|(:i)?$)"
+        return re.escape("http://"+self.getSiteDomain())+r"/s/\d+((:\d+)?(;\d+)?$|(:i)?$)?"
 
     ## Login seems to be reasonably standard across eFiction sites.
     def needToLoginCheck(self, data):
@@ -147,6 +142,10 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
         a = soup.find('h1')
         self.story.setMetadata('title',stripHTML(a))
         
+        notice = soup.find('div', {'class' : 'notice'})
+        if notice:
+            self.story.setMetadata('notice',unicode(notice))
+
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"/a/\w+"))
         self.story.setMetadata('authorId',a['href'].split('/')[2])
@@ -172,7 +171,7 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
 
             a = asoup.findAll('td', {'class' : 'lc2'})
             for lc2 in a:
-                if lc2.find('a')['href'] == '/s/'+self.story.getMetadata('storyId'):
+                if lc2.find('a', href=re.compile(r'^/s/'+self.story.getMetadata('storyId'))):
                     i=1
                     break
                 if a[len(a)-1] == lc2:
@@ -185,23 +184,74 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
         self.story.setMetadata('size', lc2.findNext('td', {'class' : 'num'}).text)
         
         lc4 = lc2.findNext('td', {'class' : 'lc4'})        
-        
+        desc = lc4.contents[0]
+
         try:
             a = lc4.find('a', href=re.compile(r"/library/show_series.php\?id=\d+"))
-            i = a.parent.text.split('(')[1].split(')')[0]
-            self.setSeries(stripHTML(a), i)
-            self.story.setMetadata('seriesUrl','http://'+self.host+a['href'])
+            if a:
+                # if there's a number after the series name, series_contents is a two element list:
+                # [<a href="...">Title</a>, u' (2)']
+                series_contents = a.parent.contents
+                i = 0 if len(series_contents) == 1 else series_contents[1].strip(' ()')
+                seriesUrl = 'http://'+self.host+a['href']
+                self.story.setMetadata('seriesUrl',seriesUrl)
+                series_name = stripHTML(a)
+                logger.debug("Series name= %s" % series_name)
+                series_soup = bs.BeautifulSoup(self._fetchUrl(seriesUrl))
+                if series_soup:
+                    logger.debug("Retrieving Series - looking for name")
+                    series_name = series_soup.find('span', {'id' : 'ptitle'}).text.partition(' — ')[0]
+                    logger.debug("Series name: '{0}'".format(series_name))
+                self.setSeries(series_name, i)
+                desc = lc4.contents[2]
+                # Check if series is in a universe
+                universes_soup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')  + "&type=uni")) 
+#                logger.debug("Universe page=", universes_soup)
+                if universes_soup:
+                    universes = universes_soup.findAll('div', {'class' : 'ser-box'})
+                    logger.debug("Number of Universes: %d" % len(universes))
+                    for universe in universes:
+                        logger.debug("universe.find('a')={0}".format(universe.find('a')))
+                        # The universe id is in an "a" tag that has an id but nothing else. It is the first tag.
+                        # The id is prefixed with the letter "u".
+                        universe_id = universe.find('a')['id'][1:]
+                        logger.debug("universe_id='%s'" % universe_id)
+                        universe_name = universe.find('div', {'class' : 'ser-name'}).text.partition(' ')[2]
+                        logger.debug("universe_name='%s'" % universe_name)
+                        # If there is link to the story, we have the right universe
+                        story_a = universe.find('a', {'href' : '/s/'+self.story.getMetadata('storyId')})
+                        if story_a:
+                            logger.debug("Story is in a series that is in a universe! The universe is '%s'" % universe_name)
+                            self.story.setMetadata("universe", universe_name)
+                            self.story.setMetadata('universeUrl','http://'+self.host+ '/library/universe.php?id=' + universe_id)
+                            break
         except:
             pass
         try:
             a = lc4.find('a', href=re.compile(r"/library/universe.php\?id=\d+"))
             if a:
                 self.story.setMetadata("universe",stripHTML(a))
+                desc = lc4.contents[2]
+                # Assumed only one universe, but it does have a URL--use universeHTML
+                universe_name = stripHTML(a)
+                universeUrl = 'http://'+self.host+a['href']
+                logger.debug("Retrieving Universe - about to get page")
+                universe_soup = bs.BeautifulSoup(self._fetchUrl(universeUrl))
+                logger.debug("Retrieving Universe - have page")
+                if universe_soup:
+                    logger.debug("Retrieving Universe - looking for name")
+                    universe_name = universe_soup.find('span', {'id' : 'ptitle'}).text.partition(' &mdash;')[0]
+                    logger.debug("Universes name: '{0}'".format(universe_name))
+
+                self.story.setMetadata('universeUrl',universeUrl)
+                logger.debug("Setting universe name: '{0}'".format(universe_name))
+                self.story.setMetadata('universe',universe_name)
+                if self.getConfig("universe_as_series"):
+                    self.setSeries(universe_name, 0)
+                    self.story.setMetadata('seriesUrl',universeUrl)
         except:
             pass
-            
 
-        desc = lc4.contents[0]
         self.setDescription('http://'+self.host+'/s/'+self.story.getMetadata('storyId'),desc)
             
         for b in lc4.findAll('b'):
@@ -255,8 +305,18 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
         # some big chapters are split over several pages
         pager = div.find('span', {'class' : 'pager'})
         if pager != None:
+            a = pager.previousSibling
+            while a != None:
+                logger.debug("before pager: {0}".format(a))
+                b = a.previousSibling
+                a.extract()
+                a = b
+
             urls=pager.findAll('a')
             urls=urls[:len(urls)-1]
+            pager.extract()
+            div.contents = div.contents[2:]
+#            logger.debug(div)
                         
             for ur in urls:
                 soup = bs.BeautifulSoup(self._fetchUrl("http://"+self.getSiteDomain()+ur['href']),
@@ -264,38 +324,76 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
         
                 div1 = soup.find('div', {'id' : 'story'})
                 
-                # appending next section
-                last=div.findAll('p')
-                next=div1.find('span', {'class' : 'conTag'}).nextSibling
-            
-                last[len(last)-1]=last[len(last)-1].append(next)
-                div.append(div1)
-            
-        # removing all the left-over stuff    
-        for a in div.findAll('span'):
-            a.extract() 
-            
-        for a in div.findAll('h1'):
-            a.extract()
-        for a in div.findAll('h2'):
-            a.extract()
-        for a in div.findAll('h3'):
-            a.extract()
-        for a in div.findAll('h4'):
-            a.extract()
-        for a in div.findAll('br'):
-            a.extract()
-        for a in div.findAll('div', {'class' : 'date'}):
-            a.extract()
-            
-        a = div.find('form')
-        if a != None:
-            b = a.nextSibling
-            while b != None:
+                # Find the "Continues" marker on the current page and remove everything after that. 
+                continues = div.find('span', {'class' : 'conTag'})
+                if continues != None:
+                    while continues != None:
+#                        logger.debug("removing end: {0}".format(continues))
+                        b = continues.nextSibling
+                        continues.extract()
+                        continues = b
+
+                # Find the "Continued" marker and delete everything before that
+                continued = div1.find('span', {'class' : 'conTag'})
+                if continued != None:
+                    a = continued.previousSibling
+                    while a != None:
+#                        logger.debug("before conTag: {0}".format(a))
+                        b = a.previousSibling
+                        a.extract()
+                        a = b
+                # Remove the pager from the end if this is the last page
+                endPager = div1.find('span', {'class' : 'pager'})
+                if endPager != None:
+                    b = endPager.nextSibling
+                    while endPager != None:
+                        logger.debug("removing end: {0}".format(endPager))
+                        b = endPager.nextSibling
+                        endPager.extract()
+                        endPager = b
+                    div1.contents = div1.contents[:len(div1) - 2]
+#                logger.debug("after removing pager: {0}".format(div1))
+                for tag in div1.contents[2:]:
+                    div.append(tag)
+
+        # If it is a chapter, there are dates at the start for when it was posted or modified. These plus 
+        # everything before them can be discarded. 
+        postedDates = div.findAll('div', {'class' : 'date'})
+        if postedDates:
+            a = postedDates[0].previousSibling
+            while a != None:
+#                logger.debug("before dates: {0}".format(a))
+                b = a.previousSibling
                 a.extract()
-                a=b
-                b=b.nextSibling
-        
+                a = b
+            for a in div.findAll('div', {'class' : 'date'}):
+                a.extract()
+
+        # For single chapter stories, there is a copyright statement. Remove this and everything
+        # before it.
+        copy = div.find('h4', {'class': 'copy'})
+        while copy != None:
+#            logger.debug("before copyright: {0}".format(copy))
+            b = copy.previousSibling
+            copy.extract()
+            copy = b
+
+        # For a story or the last chapter, remove voting form and the in library box
+        a = div.find('div', {'id' : 'vote-form'})
+        if a != None:
+            a.extract()
+        a = div.find('div', {'id' : 'b-man-div'})
+        if a != None:
+            a.extract()
+
+        # Kill the "The End" header and everything after it.
+        a = div.find(['h2', 'h3'], {'class' : 'end'})
+        logger.debug("Chapter end= '{0}'".format(a))
+        while a != None:
+            b = a.nextSibling
+            a.extract()
+            a=b
+
 
         if None == div:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)

fanficdownloader/adapters/adapter_test1.py

@@ -50,6 +50,7 @@ class TestSiteAdapter(BaseSiteAdapter):
     def extractChapterUrlsAndMetadata(self):
         idstr = self.story.getMetadata('storyId')
         idnum = int(idstr)
+        self.do_sleep()
 
         if idnum >= 1000:
             logger.warn("storyId:%s - Custom INI data will be used."%idstr)
@@ -296,6 +297,7 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
 
     def getChapterText(self, url):
         logger.debug('Getting chapter text from: %s' % url)
+        self.do_sleep()
         if self.story.getMetadata('storyId').startswith('670') or \
                 self.story.getMetadata('storyId').startswith('672'):
             time.sleep(1.0)

fanficdownloader/adapters/adapter_trekiverseorg.py

@@ -0,0 +1,325 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return TrekiverseOrgAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class TrekiverseOrgAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["iso-8859-1",
+                       "Windows-1252"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        
+        # normalized story URL.
+        self._setURL("http://"+self.getSiteDomain()\
+                         +"/efiction/viewstory.php?sid="+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','trkvs')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%d/%m/%Y"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.
+        return 'trekiverse.org'
+
+    @classmethod
+    def getAcceptDomains(cls):
+        return ['trekiverse.org','efiction.trekiverse.org']
+
+    @classmethod
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/efiction/viewstory.php?sid=1234 http://efiction."+self.getSiteDomain()+"/viewstory.php?sid=1234"
+
+    def getSiteURLPattern(self):
+        return r'(http://trekiverse\.org/efiction/viewstory\.php\?sid=\d+|http://efiction\.trekiverse\.org/viewstory\.php\?sid=\d+)'
+       
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Registered Users Only' in data \
+                or 'There is no such account on our website' in data \
+                or "That password doesn't match the one in our database" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['penname'] = self.username
+            params['password'] = self.password
+        else:
+            params['penname'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['cookiecheck'] = '1'
+        params['submit'] = 'Submit'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/efiction/user.php?action=login'
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['penname']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "Member Account" not in d : #Member Account
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['penname']))
+            raise exceptions.FailedToLogin(url,params['penname'])
+            return False
+        else:
+            return True
+
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        if self.is_adult or self.getConfig("is_adult"):
+            # Weirdly, different sites use different warning numbers.
+            # If the title search below fails, there's a good chance
+            # you need a different number.  print data at that point
+            # and see what the 'click here to continue' url says.
+            addurl = "&index=1&ageconsent=ok&warning=5"
+        else:
+            addurl="&index=1"
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url+addurl
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url)
+            
+        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
+        if m != None:
+            if self.is_adult or self.getConfig("is_adult"):
+                # We tried the default and still got a warning, so
+                # let's pull the warning number from the 'continue'
+                # link and reload data.
+                addurl = m.group(1)
+                # correct stupid & error in url.
+                addurl = addurl.replace("&","&")
+                url = self.url+'&index=1'+addurl
+                logger.debug("URL 2nd try: "+url)
+
+                try:
+                    data = self._fetchUrl(url)
+                except urllib2.HTTPError, e:
+                    if e.code == 404:
+                        raise exceptions.StoryDoesNotExist(self.url)
+                    else:
+                        raise e    
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+            
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title and author
+        a = soup.find('div', {'id' : 'pagetitle'})
+        aut = a.find('a', href=re.compile(r"^viewuser\.php\?uid="))
+        self.story.setMetadata('authorId',aut['href'].split('=')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+'/efiction/'+aut['href'])
+        self.story.setMetadata('author',aut.string)
+        
+        ttl = a.find('a', href=re.compile(r'^viewstory.php\?sid=%s$'%self.story.getMetadata('storyId')))
+        self.story.setMetadata('title',ttl.string)
+
+        # Find the chapters:
+        outputdiv = soup.find('div', {'id':'output'})
+        # (amp;)? because it should be &, but is escaped to & in URL.
+        # viewstory.php?sid=35&chapter=3
+        chapters=outputdiv.findAll('a', href=re.compile(r'^viewstory.php\?sid=%s&(amp;)?chapter=\d+$'%self.story.getMetadata('storyId')))
+        if len(chapters)==0:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: No php/html chapters found.")
+        if len(chapters)==1:
+            self.chapterUrls.append((self.story.getMetadata('title'),'http://'+self.host+'/efiction/'+chapters[0]['href']))
+        else:
+            for chapter in chapters:
+                # just in case there's tags, like <i> in chapter titles.
+                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/efiction/'+chapter['href']))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # eFiction sites don't help us out a lot with their meta data
+        # formating, so it's a little ugly.
+
+        def defaultGetattr(d,k):
+            try:
+                return d[k]
+            except:
+                return ""
+        
+        # <span class="label">Rated:</span> NC-17<br /> etc
+        labels = soup.findAll('span',{'class':'label'})
+        for labelspan in labels:
+            value = labelspan.nextSibling
+            label = labelspan.string
+
+            if 'Summary' in label:
+                ## Everything until the next span class='label'
+                svalue = ''
+                while value and not defaultGetattr(value,'class') == 'label':
+                    svalue += str(value)
+                    value = value.nextSibling
+                # sometimes poorly formated desc (<p> w/o </p>) leads
+                # to all labels being included.
+                svalue=svalue[:svalue.find('<span class="label">')]
+                self.setDescription(url,svalue)
+                #self.story.setMetadata('description',stripHTML(svalue))
+
+            if 'Rated' in label:
+                self.story.setMetadata('rating', value)
+
+            if 'Word count' in label:
+                self.story.setMetadata('numWords', value)
+
+            if 'Categories' in label:
+                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
+                catstext = [cat.string for cat in cats]
+                for cat in catstext:
+                    self.story.addToList('category',cat.string)
+
+            if 'Characters' in label:
+                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
+                charstext = [char.string for char in chars]
+                for char in charstext:
+                    self.story.addToList('characters',char.string)
+
+            if 'Genre' in label:
+                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=9'))
+                genrestext = [genre.string for genre in genres]
+                self.genre = ', '.join(genrestext)
+                for genre in genrestext:
+                    self.story.addToList('genre',genre.string)
+
+            if 'Awards' in label:
+                awards = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=12'))
+                awardstext = [award.string for award in awards]
+                self.award = ', '.join(awardstext)
+                for award in awardstext:
+                    self.story.addToList('awards',award.string)
+
+            if 'Pairing' in label:
+                ships = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=10'))
+                shipstext = [ship.string for ship in ships]
+                self.ship = ', '.join(shipstext)
+                for ship in shipstext:
+                    self.story.addToList('ships',ship.string)
+
+            if 'Warnings' in label:
+                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=11'))
+                warningstext = [warning.string for warning in warnings]
+                self.warning = ', '.join(warningstext)
+                for warning in warningstext:
+                    self.story.addToList('warnings',warning.string)
+
+            if 'Completed' in label:
+                if 'Yes' in value:
+                    self.story.setMetadata('status', 'Completed')
+                else:
+                    self.story.setMetadata('status', 'In-Progress')
+
+            if 'Published' in label:
+                self.story.setMetadata('datePublished', makeDate(value.strip(), "%d %b %Y"))
+            
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(value.strip(), "%d %b %Y"))
+                    
+        try:
+            # Find Series name from series URL.
+            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
+            series_name = a.string
+            series_url = 'http://'+self.host+'/efiction/'+a['href']
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
+            i=1
+            for a in storyas:
+                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
+                    self.setSeries(series_name, i)
+                    self.story.setMetadata('seriesUrl',series_url)
+                    break
+                i+=1
+            
+        except:
+            # I find it hard to care if the series parsing fails
+            pass
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulSoup(self._fetchUrl(url)) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+
+        notesdiv = soup.find('div', {'class':'noteinfo'})
+        if notesdiv != None:
+            div.insert(0,"<hr>")
+            div.insert(0,notesdiv)
+            div.insert(0,"<hr>")
+        
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_tthfanficorg.py

@@ -134,18 +134,18 @@ class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter):
         if "<h2>Story Not Found</h2>" in data:
             raise exceptions.StoryDoesNotExist(url)
         
+        if self.is_adult or self.getConfig("is_adult"):
+            form = soup.find('form', {'id':'sitemaxratingform'})
+            params={'ctkn':form.find('input', {'name':'ctkn'})['value'],
+                    'sitemaxrating':'5'}
+            logger.info("Attempting to get rating cookie for %s" % url)
+            data = self._postUrl("http://"+self.getSiteDomain()+'/setmaxrating.php',params)
+            # refetch story page.
+            data = self._fetchUrl(url)
+            soup = bs.BeautifulSoup(data)
+
         if "NOTE: This story is rated FR21 which is above your chosen filter level." in data:
-            if self.is_adult or self.getConfig("is_adult"):
-                form = soup.find('form', {'id':'sitemaxratingform'})
-                params={'ctkn':form.find('input', {'name':'ctkn'})['value'],
-                        'sitemaxrating':'5'}
-                logger.info("Attempting to get rating cookie for %s" % url)
-                data = self._postUrl("http://"+self.getSiteDomain()+'/setmaxrating.php',params)
-                # refetch story page.
-                data = self._fetchUrl(url)
-                soup = bs.BeautifulSoup(data)
-            else:
-                raise exceptions.AdultCheckRequired(self.url)
+            raise exceptions.AdultCheckRequired(self.url)
 
         # http://www.tthfanfic.org/AuthorStories-3449/Greywizard.htm
         # Find authorid and URL from... author url.
@@ -163,7 +163,14 @@ class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter):
             authorsoup = bs.BeautifulSoup(authordata)
             # author can have several pages, scan until we find it.
             while( not authorsoup.find('a', href=re.compile(r"^/Story-"+self.story.getMetadata('storyId'))) ):
-                nextpage = 'http://'+self.host+authorsoup.find('a', {'class':'arrowf'})['href']
+                nextarrow = authorsoup.find('a', {'class':'arrowf'})
+                if not nextarrow:
+                    ## if rating is set lower than story, it won't be
+                    ## visible on author lists unless.  The *story* is
+                    ## visible via the url, just not the entry on
+                    ## author list.
+                    raise exceptions.AdultCheckRequired(self.url)
+                nextpage = 'http://'+self.host+nextarrow['href']
                 logger.debug("**AUTHOR** nextpage URL: "+nextpage)
                 authordata = self._fetchUrl(nextpage)
                 descurl=nextpage

fanficdownloader/adapters/adapter_voracity2eficcom.py

@@ -0,0 +1,222 @@
+import re
+import urllib2
+import urlparse
+
+from .. import BeautifulSoup
+
+from base_adapter import BaseSiteAdapter, makeDate
+from .. import exceptions
+
+
+def getClass():
+    return Voracity2EficComAdapter
+
+
+class Voracity2EficComAdapter(BaseSiteAdapter):
+    SITE_ABBREVIATION = 'voe'
+    SITE_DOMAIN = 'voracity2.e-fic.com'
+
+    BASE_URL = 'http://' + SITE_DOMAIN + '/'
+    LOGIN_URL = BASE_URL + 'user.php?action=login'
+    VIEW_STORY_URL_TEMPLATE = BASE_URL + 'viewstory.php?sid=%d'
+    METADATA_URL_SUFFIX = '&index=1'
+    AGE_CONSENT_URL_SUFFIX = '&ageconsent=ok&warning=4'
+
+    DATETIME_FORMAT = '%m/%d/%Y'
+    REQUIRED_SKIN = 'Simple Elegance'
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        query_data = urlparse.parse_qs(self.parsedUrl.query)
+        story_id = query_data['sid'][0]
+
+        self.story.setMetadata('storyId', story_id)
+        self._setURL(self.VIEW_STORY_URL_TEMPLATE % int(story_id))
+        self.story.setMetadata('siteabbrev', self.SITE_ABBREVIATION)
+
+        self.is_logged_in = False
+
+    def _login(self):
+        # Apparently self.password is only set when login fails, i.e.
+        # the FailedToLogin exception is raised, so the adapter gets new
+        # login data and tries again
+        if self.password:
+            password = self.password
+            username = self.username
+        else:
+            username = self.getConfig('username')
+            password = self.getConfig('password')
+
+        parameters = {
+            'penname': username,
+            'password': password,
+            'submit': 'Submit'}
+
+        class CustomizedFailedToLogin(exceptions.FailedToLogin):
+            def __init__(self, url, passwdonly=False):
+                # Use username variable from outer scope
+                exceptions.FailedToLogin.__init__(self, url, username, passwdonly)
+
+        soup = self._customized_fetch_url(self.LOGIN_URL, CustomizedFailedToLogin, parameters)
+        div = soup.find('div', id='useropts')
+        if not div:
+            raise CustomizedFailedToLogin(self.LOGIN_URL)
+
+        self.is_logged_in = True
+
+    def _customized_fetch_url(self, url, exception, parameters=None):
+        try:
+            data = self._fetchUrl(url, parameters)
+        except urllib2.HTTPError:
+            raise exception(self.url)
+
+        return BeautifulSoup.BeautifulSoup(data)
+
+    @staticmethod
+    def getSiteDomain():
+        return Voracity2EficComAdapter.SITE_DOMAIN
+
+    @classmethod
+    def getSiteExampleURLs(cls):
+        return cls.VIEW_STORY_URL_TEMPLATE % 1234
+
+    def getSiteURLPattern(self):
+        return re.escape(self.VIEW_STORY_URL_TEMPLATE[:-2]) + r'\d+$'
+
+    def extractChapterUrlsAndMetadata(self):
+        soup = self._customized_fetch_url(self.url + self.METADATA_URL_SUFFIX, urllib2.URLError)
+
+        # Check if the story is for "Registered Users Only", i.e. has adult
+        # content. Based on the "is_adult" attributes either login or raise an
+        # error.
+        errortext_div = soup.find('div', {'class': 'errortext'})
+        if errortext_div:
+            error_text = ''.join(errortext_div(text=True)).strip()
+            if error_text == 'Registered Users Only':
+                if not (self.is_adult or self.getConfig('is_adult')):
+                    raise exceptions.AdultCheckRequired(self.url)
+                self._login()
+            else:
+                # This case usually occurs when the story doesn't exist, but
+                # might potentially be something else, so just raise
+                # FailedToDownload exception with the found error text.
+                raise exceptions.FailedToDownload(error_text)
+
+        url = ''.join([self.url, self.METADATA_URL_SUFFIX, self.AGE_CONSENT_URL_SUFFIX])
+        soup = self._customized_fetch_url(url, urllib2.URLError)
+
+        # If logged in and the skin doesn't match the required skin throw an
+        # error
+        if self.is_logged_in:
+            skin = soup.find('select', {'name': 'skin'}).find('option', selected=True)['value']
+            if skin != self.REQUIRED_SKIN:
+                raise exceptions.FailedToDownload('Required skin "%s" must be set in preferences' % self.REQUIRED_SKIN)
+
+        pagetitle_div = soup.find('div', id='pagetitle')
+        self.story.setMetadata('title', pagetitle_div.a.string)
+
+        author_anchor = pagetitle_div.a.findNextSibling('a')
+        url = urlparse.urljoin(self.BASE_URL, author_anchor['href'])
+        components = urlparse.urlparse(url)
+        query_data = urlparse.parse_qs(components.query)
+
+        self.story.setMetadata('author', author_anchor.string)
+        self.story.setMetadata('authorId', query_data['uid'])
+        self.story.setMetadata('authorUrl', url)
+
+        sort_div = soup.find('div', id='sort')
+        self.story.setMetadata('reviews', sort_div('a')[1].string)
+
+        for b_tag in soup.find('div', {'class': 'listbox'})('b'):
+            key = b_tag.string.strip(' :')
+            try:
+                value = b_tag.nextSibling.string.strip()
+            # This can happen with some fancy markup in the summary. Just
+            # ignore this error and set value to None, the summary parsing
+            # takes care of this
+            except AttributeError:
+                value = None
+
+            if key == 'Summary':
+                contents = []
+                keep_summary_html = self.getConfig('keep_summary_html')
+
+                # For some reason span_tag.findNextSiblings() only returns tags
+                # instead both NavigableString objects _and_ Tag objects as
+                # documented, so navigate manually
+                sibling = b_tag.nextSibling
+                while sibling:
+                    if isinstance(sibling, BeautifulSoup.Tag):
+                        # Encountered next label, break. This method is the
+                        # safest and most reliable I could think of. Blame
+                        # e-fiction sites that allow their users to include
+                        # arbitrary markup into their summaries and the
+                        # horrible HTML markup.
+                        if sibling.name == 'b' and sibling.findPreviousSibling().name == 'br':
+                            break
+
+                        if keep_summary_html:
+                            contents.append(self.utf8FromSoup(self.url, sibling))
+                        else:
+                            contents.append(''.join(sibling(text=True)))
+                    else:
+                        contents.append(sibling)
+                    sibling = sibling.nextSibling
+
+                # Remove the preceding break line tag and other crud
+                contents.pop()
+                contents.pop()
+                self.story.setMetadata('description', ''.join(contents))
+
+            elif key == 'Rating':
+                self.story.setMetadata('rating', value)
+
+            elif key == 'Category':
+                for sibling in b_tag.findNextSiblings(['a', 'br']):
+                    if sibling.name == 'br':
+                        break
+                    self.story.addToList('category', sibling.string)
+
+            # Seems to be always "None" for some reason
+            elif key == 'Characters':
+                for sibling in b_tag.findNextSiblings(['a', 'br']):
+                    if sibling.name == 'br':
+                        break
+                    self.story.addToList('characters', sibling.string)
+
+            elif key == 'Series':
+                a = b_tag.findNextSibling('a')
+                if not a:
+                    continue
+                self.story.setMetadata('series', a.string)
+                self.story.setMetadata('seriesUrl', urlparse.urljoin(self.BASE_URL, a['href']))
+
+            elif key == 'Chapter':
+                self.story.setMetadata('numChapters', int(value))
+
+            elif key == 'Completed':
+                self.story.setMetadata('status', 'Completed' if value == 'Yes' else 'In-Progress')
+
+            elif key == 'Words':
+                self.story.setMetadata('numWords', value)
+
+            elif key == 'Read':
+                self.story.setMetadata('readings', value)
+
+            elif key == 'Published':
+                self.story.setMetadata('datePublished', makeDate(value, self.DATETIME_FORMAT))
+
+            elif key == 'Updated':
+                self.story.setMetadata('dateUpdated', makeDate(value, self.DATETIME_FORMAT))
+
+        for b_tag in soup.find('div', id='output').findNextSiblings('b'):
+            chapter_anchor = b_tag.a
+            title = chapter_anchor.string
+            url = urlparse.urljoin(self.BASE_URL, chapter_anchor['href'])
+            self.chapterUrls.append((title, url))
+
+    def getChapterText(self, url):
+        url += self.AGE_CONSENT_URL_SUFFIX
+        soup = self._customized_fetch_url(url, urllib2.URLError)
+        return self.utf8FromSoup(url, soup.find('div', id='story'))

fanficdownloader/adapters/base_adapter.py

@@ -76,6 +76,8 @@ class BaseSiteAdapter(Configurable):
         self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
         self.password = ""
         self.is_adult=False
+
+        self.override_sleep = None
         
         self.opener = u2.build_opener(u2.HTTPCookieProcessor(),GZipProcessor())
         ## Specific UA because too many sites are blocking the default python UA.
@@ -147,8 +149,7 @@ class BaseSiteAdapter(Configurable):
 
     # Assumes application/x-www-form-urlencoded.  parameters, headers are dict()s
     def _postUrl(self, url, parameters={}, headers={}):
-        if self.getConfig('slow_down_sleep_time'):
-            time.sleep(float(self.getConfig('slow_down_sleep_time')))
+        self.do_sleep()
 
         ## u2.Request assumes POST when data!=None.  Also assumes data
         ## is application/x-www-form-urlencoded.
@@ -159,24 +160,38 @@ class BaseSiteAdapter(Configurable):
         req = u2.Request(url,
                          data=urllib.urlencode(parameters),
                          headers=headers)
-        return self._decode(self.opener.open(req).read())
+        return self._decode(self.opener.open(req,None,float(self.getConfig('connect_timeout',30.0))).read())
 
     def _fetchUrlRaw(self, url, parameters=None):
         if parameters != None:
-            return self.opener.open(url.replace(' ','%20'),urllib.urlencode(parameters)).read()
+            return self.opener.open(url.replace(' ','%20'),urllib.urlencode(parameters),float(self.getConfig('connect_timeout',30.0))).read()
         else:
-            return self.opener.open(url.replace(' ','%20')).read()
+            return self.opener.open(url.replace(' ','%20'),None,float(self.getConfig('connect_timeout',30.0))).read()
+
+    def set_sleep(self,val):
+        print("\n===========\n set sleep time %s\n==========="%val)
+        self.override_sleep = val
     
+    def do_sleep(self):
+        if self.override_sleep:
+            time.sleep(float(self.override_sleep))
+        elif self.getConfig('slow_down_sleep_time'):
+            time.sleep(float(self.getConfig('slow_down_sleep_time')))
+        
     # parameters is a dict()
     def _fetchUrl(self, url, parameters=None):
-        if self.getConfig('slow_down_sleep_time'):
-            time.sleep(float(self.getConfig('slow_down_sleep_time')))
+        self.do_sleep()
 
         excpt=None
         for sleeptime in [0, 0.5, 4, 9]:
             time.sleep(sleeptime)	
             try:
                 return self._decode(self._fetchUrlRaw(url,parameters))
+            except u2.HTTPError, he:
+                excpt=he
+                if he.code == 404:
+                    logger.warn("Caught an exception reading URL: %s  Exception %s."%(unicode(url),unicode(he)))
+                    break # break out on 404
             except Exception, e:
                 excpt=e
                 logger.warn("Caught an exception reading URL: %s  Exception %s."%(unicode(url),unicode(e)))
@@ -358,7 +373,7 @@ class BaseSiteAdapter(Configurable):
 
         retval = soup.__str__('utf8').decode('utf-8')
 
-        if self.getConfig('nook_img_fix'):
+        if self.getConfig('nook_img_fix') and not self.getConfig('replace_br_with_p'):
             # if the <img> tag doesn't have a div or a p around it,
             # nook gets confused and displays it on every page after
             # that under the text for the rest of the chapter.
@@ -367,7 +382,7 @@ class BaseSiteAdapter(Configurable):
             
         # Don't want body tags in chapter html--writers add them.
         # This is primarily for epub updates.
-        retval = re.sub(r"</?body>\r?\n?","",retval)
+        retval = re.sub(r"</?body[^>]*>\r?\n?","",retval)
         
         if self.getConfig("replace_br_with_p"):
             # Apply heuristic processing to replace <br> paragraph

fanficdownloader/htmlheuristics.py

@@ -18,6 +18,9 @@
 import logging
 logger = logging.getLogger(__name__)
 import re
+import codecs
+import BeautifulSoup as bs
+import HtmlTagStack as stack
 
 from . import exceptions as exceptions
 
@@ -27,24 +30,29 @@ def replace_br_with_p(body):
     # However, Python Regex does not recognize it as a whitespace, so we'll be changing it to a reagular space.
     body = body.replace(u'\xa0', u' ')
 
-    if body.find('>') == -1 or body.rfind("<") == -1:
+    if body.find('>') == -1 or body.rfind('<') == -1:
         return body
 
     # logger.debug(u'BODY start.: ' + body[:250])
     # logger.debug(u'BODY end...: ' + body[-250:])
     # logger.debug(u'BODY.......: ' + body)
 
+    # clean breaks (<br />), removing whitespaces between them.
+    body = re.sub(r'\s*<br[^>]*>\s*', r'<br />', body)
+
     # change surrounding div to a p and remove attrs Top surrounding
     # tag in all cases now should be div, to just strip the first and
     # last tags.
-    body = body[body.index('>')+1:body.rindex("<")]
+    if is_valid_block(body) and body.find('<div') == 0:
+        body = body[body.index('>')+1:body.rindex('<')]
 
-    # Need to look at BeautifulSoup to see if it'll even return breaks that aren't properly formatted (<br />).
-    body = re.sub(r'\s*<br[^>]*>\s*', r'<br />', body)
+    body = soup_up_div(u'<div>' + body + u'</div>')
+
+    body = body[body.index('>')+1:body.rindex('<')]
 
     # Find all bexisting blocks with p, pre and blockquote tags, we need to shields break tags inside those.
     # This is for "lenient" mode, however it is also used to clear break tags before and after the block elements.
-    blocksRegex = re.compile(r'(\s*<br\ */*>\s*)*\s*<(pre|p|blockquote)([^>]*)>(.+?)</\2>\s*(\s*<br\ */*>\s*)*', re.DOTALL)
+    blocksRegex = re.compile(r'(\s*<br\ />\s*)*\s*<(pre|p|blockquote|table)([^>]*)>(.+?)</\2>\s*(\s*<br\ />\s*)*', re.DOTALL)
     body = blocksRegex.sub(r'\n<\2\3>\4</\2>\n', body)
 
     # if aggressive mode = true
@@ -66,7 +74,7 @@ def replace_br_with_p(body):
     # change surrounding div to a p and remove attrs Top surrounding
     # tag in all cases now should be div, to just strip the first and
     # last tags.
-    body = u'<p>' + body + u'</p>'
+    # body = u'<p>' + body + u'</p>'
 
     # Nuke div tags surrounding a HR tag.
     body = re.sub(r'<div[^>]+>\s*<hr[^>]+>\s*</div>', r'\n<hr />\n', body)
@@ -129,32 +137,35 @@ def replace_br_with_p(body):
             contentLinesSum += lineLen
             if lineLen > longestLineLength:
                 longestLineLength = lineLen
+                
+    if contentLines == 0:
+        contentLines = 1
 
     averageLineLength = contentLinesSum/contentLines
 
     logger.debug(u'---')
-    logger.debug(u'Lines.............: ' + str(len(lines)))
-    logger.debug(u'contentLines......: ' + str(contentLines))
-    logger.debug(u'contentLinesSum...: ' + str(contentLinesSum))
-    logger.debug(u'longestLineLength.: ' + str(longestLineLength))
-    logger.debug(u'averageLineLength.: ' + str(averageLineLength))
+    logger.debug(u'Lines.............: ' + unicode(len(lines)))
+    logger.debug(u'contentLines......: ' + unicode(contentLines))
+    logger.debug(u'contentLinesSum...: ' + unicode(contentLinesSum))
+    logger.debug(u'longestLineLength.: ' + unicode(longestLineLength))
+    logger.debug(u'averageLineLength.: ' + unicode(averageLineLength))
 
     if breaksMaxIndex == len(breaksCount)-1 and breaksMax < 2:
         breaksMaxIndex = 0
         breaksMax = breaksCount[0]
 
     logger.debug(u'---')
-    logger.debug(u'breaks 1: ' + str(breaksCount[0]))
-    logger.debug(u'breaks 2: ' + str(breaksCount[1]))
-    logger.debug(u'breaks 3: ' + str(breaksCount[2]))
-    logger.debug(u'breaks 4: ' + str(breaksCount[3]))
-    logger.debug(u'breaks 5: ' + str(breaksCount[4]))
-    logger.debug(u'breaks 6: ' + str(breaksCount[5]))
-    logger.debug(u'breaks 7: ' + str(breaksCount[6]))
-    logger.debug(u'breaks 8: ' + str(breaksCount[7]))
+    logger.debug(u'breaks 1: ' + unicode(breaksCount[0]))
+    logger.debug(u'breaks 2: ' + unicode(breaksCount[1]))
+    logger.debug(u'breaks 3: ' + unicode(breaksCount[2]))
+    logger.debug(u'breaks 4: ' + unicode(breaksCount[3]))
+    logger.debug(u'breaks 5: ' + unicode(breaksCount[4]))
+    logger.debug(u'breaks 6: ' + unicode(breaksCount[5]))
+    logger.debug(u'breaks 7: ' + unicode(breaksCount[6]))
+    logger.debug(u'breaks 8: ' + unicode(breaksCount[7]))
     logger.debug(u'----')
-    logger.debug(u'max found: ' + str(breaksMax))
-    logger.debug(u'max Index: ' + str(breaksMaxIndex))
+    logger.debug(u'max found: ' + unicode(breaksMax))
+    logger.debug(u'max Index: ' + unicode(breaksMaxIndex))
     logger.debug(u'----')
 
     if breaksMaxIndex > 0 and breaksCount[0] > breaksMax and averageLineLength < 90:
@@ -163,14 +174,15 @@ def replace_br_with_p(body):
     # Find all instances of consecutive breaks less than otr equal to the max count use most often
     #  replase those tags to inverted p tag pairs, those with more connsecutive breaks are replaced them with a horisontal line
     for i in range(len(breaksCount)):
+        # if i > 0 or breaksMaxIndex == 0:
         if i <= breaksMaxIndex:
-            logger.debug(str(i) + u' <= breaksMaxIndex (' + str(breaksMaxIndex) + u')')
+            logger.debug(unicode(i) + u' <= breaksMaxIndex (' + unicode(breaksMaxIndex) + u')')
             body = breaksRegexp[i].sub(r'\1</p>\n<p>\3', body)
         elif i == breaksMaxIndex+1:
-            logger.debug(str(i) + u' == breaksMaxIndex+1 (' + str(breaksMaxIndex+1) + u')')
+            logger.debug(unicode(i) + u' == breaksMaxIndex+1 (' + unicode(breaksMaxIndex+1) + u')')
             body = breaksRegexp[i].sub(r'\1</p>\n<p><br/></p>\n<p>\3', body)
         else:
-            logger.debug(str(i) + u' > breaksMaxIndex+1 (' + str(breaksMaxIndex+1) + u')')
+            logger.debug(unicode(i) + u' > breaksMaxIndex+1 (' + unicode(breaksMaxIndex+1) + u')')
             body = breaksRegexp[i].sub(r'\1</p>\n<hr />\n<p>\3', body)
 
     body = breaksRegexp[8].sub(r'</p>\n<hr />\n<p>', body)
@@ -181,6 +193,9 @@ def replace_br_with_p(body):
     body = body.replace(u'&squareBracketStart;', u'[')
     body = body.replace(u'&squareBracketEnd;', u']')
 
+    body = body.replace(u'{p}', u'<p>')
+    body = body.replace(u'{/p}', u'</p>')
+
     # If for some reason, a third break makes its way inside the paragraph, preplace that with the empty paragraph for the additional linespaing.
     body = re.sub(r'<p>\s*(<br\ \/>)+', r'<p><br /></p>\n<p>', body)
 
@@ -209,9 +224,125 @@ def replace_br_with_p(body):
     body = re.sub(r'\s*<(\S+)[^>]*>\s*</\1>', r'', body)
 
     body = body.replace(u'{br /}', u'<br />')
-    
+    body = body.strip()
+
     # re-wrap in div tag.
     body = u'<div>\n' + body + u'</div>\n'
 
-    return body 
+    # return body
+    return tag_sanitizer(body)
 
+def is_valid_block(block):
+    return unicode(block).find('<') == 0 and unicode(block).find('<!') != 0
+
+def soup_up_div(body):
+    blockTags = ['address', 'blockquote', 'del', 'div', 'dl', 'fieldset', 'form', 'ins', 'noscript', 'ol', 'p', 'pre', 'table', 'ul']
+    recurseTags = ['blockquote', 'div', 'noscript']
+
+    tag = body[:body.index('>')+1]
+    tagend = body[body.rindex('<'):]
+
+    body = body.replace(u'<br />', u'[br /]')
+
+    soup = bs.BeautifulSoup(body)
+
+    body = u''
+    lastElement = 1 # 1 = block, 2 = nested, 3 = invalid
+
+    for i in soup.contents[0]:
+        if unicode(i).strip().__len__() > 0:
+            s = unicode(i)
+            if  type(i) == bs.Tag:
+                if  i.name in blockTags:
+                    if lastElement > 1:
+                        body = body.strip(r'\s*(\[br\ \/\]\s*)*\s*')
+                        body += u'{/p}'
+
+                    lastElement = 1
+
+                    if i.name in recurseTags:
+                        s = soup_up_div(s)
+
+                    body += s.strip() + '\n'
+                else:
+                    if lastElement == 1:
+                        body = body.strip(r'\s*(\[br\ \/\]\s*)*\s*')
+                        body += u'{p}'
+
+                    lastElement = 2
+                    body += s
+            elif type(i) == bs.Comment:
+                body += s
+            else:
+                if lastElement == 1:
+                    body = body.strip(r'\s*(\[br\ \/\]\s*)*\s*')
+                    body += u'{p}'
+
+                lastElement = 3
+                body += s
+
+    if lastElement > 1:
+        body = body.strip(r'\s*(\[br\ \/\]\s*)*\s*')
+        body += u'{/p}'
+
+    body = body.replace(u'[br /]', u'<br />')
+
+    return tag + body + tagend
+
+
+def is_end_tag(tag):
+    return re.match(r'</([^\ >]+)>', tag) != None
+
+def is_comment_tag(tag):
+    return re.match(r'<\!\-\-([^>]+)>', tag) != None
+
+def is_closed_tag(tag):
+    return re.match(r'<(.+?)/>', tag) != None
+
+def tag_sanitizer(html):
+    blockTags = ['address', 'blockquote', 'del', 'div', 'dl', 'fieldset', 'form', 'ins', 'noscript', 'ol', 'pre', 'table', 'ul']
+
+    body = u''
+    tags = re.findall(r'(<[^>]+>)([^<]*)', html)
+
+    for rTag in tags:
+        name = stack.get_tag_name(rTag[0])
+        is_end = is_end_tag(rTag[0])
+        is_closed = is_closed_tag(rTag[0]) or is_comment_tag(rTag[0])
+
+        # is_comment = is_comment_tag(rTag[0])
+        # logger.debug(u'%s >  isEnd: %s >  isClosed: %s >  isComment: %s'%(name, unicode(is_end), unicode(is_closed), unicode(is_comment)))
+        # logger.debug(u'> %s%s\n'%(rTag[0], rTag[1]))
+
+        if name in blockTags:
+            body += rTag[0]
+            body += rTag[1]
+        elif name == u'p':
+            if is_end:
+                body += stack.spool_end()
+                body += rTag[0]
+                body += rTag[1]
+            elif is_closed:
+                body += rTag[0]
+                body += rTag[1]
+            else:
+                body += rTag[0]
+                body += stack.spool_start()
+                body += rTag[1]
+        else:
+            if is_end:
+                t = stack.get_last()
+                tn = stack.get_tag_name(t)
+                rTn = stack.get_tag_name(rTag[0])
+                if tn == rTn:
+                    body += rTag[0]
+                    stack.pop()
+            elif not is_closed:
+                stack.push(rTag[0])
+                body += rTag[0]
+            else:
+                body += rTag[0]
+
+            body += rTag[1]
+    stack.flush()
+    return body

fanficdownloader/story.py

@@ -30,7 +30,7 @@ from configurable import Configurable
 
 # Create convert_image method depending on which graphics lib we can
 # load.  Preferred: calibre, PIL, none
-        
+
 imagetypes = {
     'jpg':'image/jpeg',
     'jpeg':'image/jpeg',
@@ -48,14 +48,14 @@ try:
         export = False
         img = Image()
         img.load(data)
-        
+
         owidth, oheight = img.size
         nwidth, nheight = sizes
         scaled, nwidth, nheight = fit_image(owidth, oheight, nwidth, nheight)
         if scaled:
             img.size = (nwidth, nheight)
             export = True
-            
+
         if normalize_format_name(img.format) != imgtype:
             export = True
 
@@ -65,7 +65,7 @@ try:
             canvas.compose(img)
             img = canvas
             export = True
-            
+
         if grayscale and img.type != "GrayscaleType":
             img.type = "GrayscaleType"
             export = True
@@ -75,7 +75,7 @@ try:
         else:
             logger.debug("image used unchanged")
             return (data,imgtype,imagetypes[imgtype])
-        
+
 except:
 
     # No calibre routines, try for PIL for CLI.
@@ -87,14 +87,14 @@ except:
                           removetrans,imgtype="jpg",background='#ffffff'):
             export = False
             img = Image.open(StringIO(data))
-            
+
             owidth, oheight = img.size
             nwidth, nheight = sizes
             scaled, nwidth, nheight = fit_image(owidth, oheight, nwidth, nheight)
             if scaled:
                 img = img.resize((nwidth, nheight),Image.ANTIALIAS)
                 export = True
-                
+
             if normalize_format_name(img.format) != imgtype:
                 if img.mode == "P":
                     # convert pallete gifs to RGB so jpg save doesn't fail.
@@ -119,7 +119,7 @@ except:
             else:
                 logger.debug("image used unchanged")
                 return (data,imgtype,imagetypes[imgtype])
-        
+
     except:
         # No calibre or PIL, simple pass through with mimetype.
         def convert_image(url,data,sizes,grayscale,
@@ -129,16 +129,16 @@ except:
 ## also used for explicit no image processing.
 def no_convert_image(url,data):
     parsedUrl = up.urlparse(url)
-    
+
     ext=parsedUrl.path[parsedUrl.path.rfind('.')+1:].lower()
-    
+
     if ext not in imagetypes:
         logger.debug("no_convert_image url:%s - no known extension"%url)
         # doesn't have extension? use jpg.
         ext='jpg'
-        
+
     return (data,ext,imagetypes[ext])
-        
+
 def normalize_format_name(fmt):
     if fmt:
         fmt = fmt.lower()
@@ -221,8 +221,67 @@ langs = {
     "Devanagari":"hi",
     }
 
+class InExMatch:
+    keys = []
+    regex = None
+    match = None
+    negate = False
+
+    def  __init__(self,line):
+        if "=~" in line:
+            (self.keys,self.match) = line.split("=~")
+            self.match = self.match.replace('\s',' ')
+            self.regex = re.compile(self.match)
+        elif "!~" in line:
+            (self.keys,self.match) = line.split("!~")
+            self.match = self.match.replace('\s',' ')
+            self.regex = re.compile(self.match)
+            self.negate = True
+        elif "==" in line:
+            (self.keys,self.match) = line.split("==")
+            self.match = self.match.replace('\s',' ')
+        elif "!=" in line:
+            (self.keys,self.match) = line.split("!=")
+            self.match = self.match.replace('\s',' ')
+            self.negate = True
+        self.keys = map( lambda x: x.strip(), self.keys.split(",") )
+
+    # For conditional, only one key
+    def is_key(self,key):
+        return key == self.keys[0]
+
+    # For conditional, only one key
+    def key(self):
+        return self.keys[0]
+
+    def in_keys(self,key):
+        return key in self.keys
+
+    def is_match(self,value):
+        retval = False
+        if self.regex:
+            if self.regex.search(value):
+                retval = True
+            #print(">>>>>>>>>>>>>%s=~%s r: %s,%s=%s"%(self.match,value,self.negate,retval,self.negate != retval))
+        else:
+            retval = self.match == value
+            #print(">>>>>>>>>>>>>%s==%s r: %s,%s=%s"%(self.match,value,self.negate,retval, self.negate != retval))
+            
+        return self.negate != retval
+
+    def __str__(self):
+        if self.negate:
+            f='!'
+        else:
+            f='='
+        if self.regex:
+            s='~'
+        else:
+            s='='
+        return u'InExMatch(%s %s%s %s)'%(self.keys,f,s,self.match)
+
 class Story(Configurable):
-    
+
     def __init__(self, configuration):
         Configurable.__init__(self, configuration)
         try:
@@ -231,10 +290,11 @@ class Story(Configurable):
         except:
             self.metadata = {'version':'4.4'}
         self.replacements = []
+        self.in_ex_cludes = {}
         self.chapters = [] # chapters will be tuples of (title,html)
         self.imgurls = []
         self.imgtuples = []
-        
+
         self.cover=None # *href* of new cover image--need to create html.
         self.oldcover=None # (oldcoverhtmlhref,oldcoverhtmltype,oldcoverhtmldata,oldcoverimghref,oldcoverimgtype,oldcoverimgdata)
         self.calibrebookmark=None # cheesy way to carry calibre bookmark file forward across update.
@@ -251,7 +311,16 @@ class Story(Configurable):
                 self.addToList(metadata,val)
 
         self.setReplace(self.getConfig('replace_metadata'))
-        
+
+        in_ex_clude_list = ['include_metadata_pre','exclude_metadata_pre',
+                            'include_metadata_post','exclude_metadata_post']
+        for ie in in_ex_clude_list:
+            ies = self.getConfig(ie)
+            # print("%s %s"%(ie,ies))
+            if ies:
+                iel = []
+                self.in_ex_cludes[ie] = self.set_in_ex_clude(ies)
+
     def setMetadata(self, key, value, condremoveentities=True):
         ## still keeps < < and &
         if condremoveentities:
@@ -269,6 +338,56 @@ class Story(Configurable):
             self.addToList('lastupdate',value.strftime("Last Update Year/Month: %Y/%m"))
             self.addToList('lastupdate',value.strftime("Last Update: %Y/%m/%d"))
 
+        
+    ## metakey[,metakey]=~pattern
+    ## metakey[,metakey]==string
+    ## *for* part lines.  Effect only when trailing conditional key=~regexp matches
+    ## metakey[,metakey]=~pattern[&&metakey=~regexp]
+    ## metakey[,metakey]==string[&&metakey=~regexp]
+    ## metakey[,metakey]=~pattern[&&metakey==string]
+    ## metakey[,metakey]==string[&&metakey==string]
+    def set_in_ex_clude(self,setting):
+        dest = []
+        # print("set_in_ex_clude:"+setting)
+        for line in setting.splitlines():
+            if line:
+                (match,condmatch)=(None,None)
+                if "&&" in line:
+                    (line,conditional) = line.split("&&")
+                    condmatch = InExMatch(conditional)
+                match = InExMatch(line)
+                dest.append([match,condmatch])
+        return dest
+              
+    def do_in_ex_clude(self,which,value,key):
+        if value and which in self.in_ex_cludes:
+            include = 'include' in which
+            keyfound = False
+            found = False
+            for (match,condmatch) in self.in_ex_cludes[which]:
+                keyfndnow = False
+                if match.in_keys(key):
+                    # key in keys and either no conditional, or conditional matched
+                    if condmatch == None or condmatch.is_key(key):
+                        keyfndnow = True
+                    else:
+                        condval = self.getMetadata(condmatch.key())
+                        keyfndnow = condmatch.is_match(condval)
+                    keyfound |= keyfndnow
+                        # print("match:%s %s\ncondmatch:%s %s\n\tkeyfound:%s\n\tfound:%s"%(
+                        #         match,value,condmatch,condval,keyfound,found))
+                    if keyfndnow:
+                        found = isinstance(value,basestring) and match.is_match(value)
+                    if found:
+                        # print("match:%s %s\n\tkeyfndnow:%s\n\tfound:%s"%(
+                        #         match,value,keyfndnow,found))
+                        if not include:
+                            value = None
+                        break
+            if include and keyfound and not found:
+                value = None
+        return value
+        
 
     ## Two or three part lines.  Two part effect everything.
     ## Three part effect only those key(s) lists.
@@ -297,10 +416,14 @@ class Story(Configurable):
                 # A way to explicitly include spaces in the
                 # replacement string.  The .ini parser eats any
                 # trailing spaces.
-                replacement=replacement.replace('\s',' ') 
+                replacement=replacement.replace('\s',' ')
                 self.replacements.append([metakeys,regexp,replacement,condkey,condregexp])
-    
+
     def doReplacements(self,value,key):
+        value = self.do_in_ex_clude('include_metadata_pre',value,key)
+        value = self.do_in_ex_clude('exclude_metadata_pre',value,key)
+        # if key=='characters' and value=='Bilbo':
+        #     value = ''
         for (metakeys,regexp,replacement,condkey,condregexp) in self.replacements:
             if (metakeys == None or key in metakeys) \
                     and isinstance(value,basestring) \
@@ -309,11 +432,13 @@ class Story(Configurable):
                 if condkey and condkey != key: # prevent infinite recursion.
                     condval = self.getMetadata(condkey)
                     doreplace = condval != None and condregexp.search(condval)
-                    
+
                 if doreplace:
                     value = regexp.sub(replacement,value)
+        value = self.do_in_ex_clude('include_metadata_post',value,key)
+        value = self.do_in_ex_clude('exclude_metadata_post',value,key)
         return value
-        
+
     def getMetadataRaw(self,key):
         if self.isValidMetaEntry(key) and self.metadata.has_key(key):
             return self.metadata[key]
@@ -326,7 +451,7 @@ class Story(Configurable):
             return value
 
         if self.isList(key):
-            join_string = self.getConfig("join_string_"+key,u", ").replace('\s',' ') 
+            join_string = self.getConfig("join_string_"+key,u", ").replace('\s',' ')
             value = join_string.join(self.getList(key, removeallentities, doreplacements=True))
             if doreplacements:
                 value = self.doReplacements(value,key+"_LIST")
@@ -351,7 +476,7 @@ class Story(Configurable):
                 return value
         else: #if self.getConfig("default_value_"+key):
             return self.getConfig("default_value_"+key)
-        
+
     def getAllMetadata(self,
                        removeallentities=False,
                        doreplacements=True,
@@ -360,13 +485,13 @@ class Story(Configurable):
         All single value *and* list value metadata as strings (unless keeplists=True, then keep lists).
         '''
         allmetadata = {}
-        
+
         # special handling for authors/authorUrls
         linkhtml="<a class='%slink' href='%s'>%s</a>"
         if self.isList('author'): # more than one author, assume multiple authorUrl too.
             htmllist=[]
             for i, v in enumerate(self.getList('author')):
-                aurl = self.getList('authorUrl')[i]                    
+                aurl = self.getList('authorUrl')[i]
                 auth = v
                 # make sure doreplacements & removeallentities are honored.
                 if doreplacements:
@@ -375,9 +500,9 @@ class Story(Configurable):
                 if removeallentities:
                     aurl=removeAllEntities(aurl)
                     auth=removeAllEntities(auth)
-                
+
                 htmllist.append(linkhtml%('author',aurl,auth))
-            join_string = self.getConfig("join_string_authorHTML",u", ").replace('\s',' ') 
+            join_string = self.getConfig("join_string_authorHTML",u", ").replace('\s',' ')
             self.setMetadata('authorHTML',join_string.join(htmllist))
         else:
             self.setMetadata('authorHTML',linkhtml%('author',self.getMetadata('authorUrl', removeallentities, doreplacements),
@@ -388,20 +513,43 @@ class Story(Configurable):
                                                     self.getMetadata('series', removeallentities, doreplacements)))
         elif self.getMetadataRaw('series') != None:
             self.setMetadata('seriesHTML',self.getMetadataRaw('series'))
-            
+
+        # logger.debug("make_linkhtml_entries:%s"%self.getConfig('make_linkhtml_entries'))
+        for k in self.getConfigList('make_linkhtml_entries'):
+            # Assuming list, because it has to be site specific and
+            # they are all lists.  Bail if kUrl list not the same
+            # length.
+            # logger.debug("\nk:%s\nlist:%s\nlistURL:%s"%(k,self.getList(k),self.getList(k+'Url')))
+            if len(self.getList(k+'Url')) != len(self.getList(k)):
+                continue
+            htmllist=[]
+            for i, v in enumerate(self.getList(k)):
+                url = self.getList(k+'Url')[i]
+                # make sure doreplacements & removeallentities are honored.
+                if doreplacements:
+                    url=self.doReplacements(url,k+'Url')
+                    v=self.doReplacements(v,k)
+                if removeallentities:
+                    url=removeAllEntities(url)
+                    v=removeAllEntities(v)
+
+                htmllist.append(linkhtml%(k,url,v))
+            join_string = self.getConfig("join_string_"+k+"HTML",u", ").replace('\s',' ')
+            self.setMetadata(k+'HTML',join_string.join(htmllist))
+
         for k in self.getValidMetaList():
             if self.isList(k) and keeplists:
                 allmetadata[k] = self.getList(k, removeallentities, doreplacements)
             else:
                 allmetadata[k] = self.getMetadata(k, removeallentities, doreplacements)
-                
+
         return allmetadata
 
     # just for less clutter in adapters.
     def extendList(self,listname,l):
         for v in l:
             self.addToList(listname,v.strip())
-    
+
     def addToList(self,listname,value):
         if value==None:
             return
@@ -421,24 +569,24 @@ class Story(Configurable):
         return self.hasConfig("include_in_"+listname) or \
             ( self.isValidMetaEntry(listname) and self.metadata.has_key(listname) \
                   and isinstance(self.metadata[listname],list) )
-    
+
     def getList(self,listname,
                 removeallentities=False,
                 doreplacements=True,
                 includelist=[]):
         #print("getList(%s,%s)"%(listname,includelist))
         retlist = []
-        
+
         if not self.isValidMetaEntry(listname):
             return retlist
-        
+
         # includelist prevents infinite recursion of include_in_'s
         if self.hasConfig("include_in_"+listname) and listname not in includelist:
             for k in self.getConfigList("include_in_"+listname):
                 retlist.extend(self.getList(k,removeallentities=False,
                                             doreplacements=doreplacements,includelist=includelist+[listname]))
         else:
-        
+
             if not self.isList(listname):
                 retlist = [self.getMetadata(listname,removeallentities=False,
                                             doreplacements=doreplacements)]
@@ -458,7 +606,7 @@ class Story(Configurable):
         # ships=>[ ]*(/|&amp;|&)[ ]*=>/
         if listname == 'ships' and self.getConfig('sort_ships'):
             retlist = [ '/'.join(sorted(x.split('/'))) for x in retlist ]
-                
+
         if retlist:
             if listname in ('author','authorUrl','authorId') or self.getConfig('keep_in_order_'+listname):
                 # need to retain order for author & authorUrl so the
@@ -473,9 +621,9 @@ class Story(Configurable):
     def getSubjectTags(self, removeallentities=False):
         # set to avoid duplicates subject tags.
         subjectset = set()
-        
+
         tags_list = self.getConfigList("include_subject_tags") + self.getConfigList("extra_subject_tags")
-            
+
         # metadata all go into dc:subject tags, but only if they are configured.
         for (name,value) in self.getAllMetadata(removeallentities=removeallentities,keeplists=True).iteritems():
             if name in tags_list:
@@ -491,7 +639,7 @@ class Story(Configurable):
             subjectset.remove('')
 
         return list(subjectset | set(self.getConfigList("extratags")))
-            
+
     def addChapter(self, url, title, html):
         if self.getConfig('strip_chapter_numbers') and \
                 self.getConfig('chapter_title_strip_pattern'):
@@ -512,7 +660,7 @@ class Story(Configurable):
                                 html) )
         else:
             retval = self.chapters
-            
+
         return retval
 
     def formatFileName(self,template,allowunsafefilename=True):
@@ -526,7 +674,7 @@ class Story(Configurable):
             pattern = re.compile(self.getConfig("output_filename_safepattern",r"[^a-zA-Z0-9_\. \[\]\(\)&'-]+"))
             for k in origvalues.keys():
                 values[k]=re.sub(pattern,'_', removeAllEntities(self.getMetadata(k)))
-    
+
         return string.Template(template).substitute(values).encode('utf8')
 
     # pass fetch in from adapter in case we need the cookies collected
@@ -537,7 +685,7 @@ class Story(Configurable):
         # isn't used anywhere.
         if cover and self.getConfig('never_make_cover'):
             return
-        
+
         url = url.strip() # ran across an image with a space in the
                           # src. Browser handled it, so we'd better, too.
 
@@ -545,12 +693,18 @@ class Story(Configurable):
         # gets too big too fast and breaks things.
         if is_appengine:
             return
-        
+
         if url.startswith("http") or url.startswith("file") or parenturl == None:
             imgurl = url
         else:
             parsedUrl = urlparse.urlparse(parenturl)
-            if url.startswith("/") :
+            if url.startswith("//") :
+                imgurl = urlparse.urlunparse(
+                    (parsedUrl.scheme,
+                     '',
+                     url,
+                     '','',''))
+            elif url.startswith("/") :
                 imgurl = urlparse.urlunparse(
                     (parsedUrl.scheme,
                      parsedUrl.netloc,
@@ -598,7 +752,7 @@ class Story(Configurable):
             except Exception, e:
                 logger.info("Failed to load or convert image, skipping:\n%s\nException: %s"%(imgurl,e))
                 return "failedtoload"
-            
+
             # explicit cover, make the first image.
             if cover and not self.getConfig('never_make_cover'):
                 if len(self.imgtuples) > 0 and 'cover' in self.imgtuples[0]['newsrc']:
@@ -624,19 +778,19 @@ class Story(Configurable):
                     self.cover=newsrc
                     self.imgtuples.append({'newsrc':newsrc,'mime':mime,'data':data})
                     self.imgurls.append(imgurl)
-            
+
                 newsrc = "images/%s-%s.%s"%(
                     prefix,
                     self.imgurls.index(imgurl),
                     ext)
                 self.imgtuples.append({'newsrc':newsrc,'mime':mime,'data':data})
-                
+
             #logger.debug("\nimgurl:%s\nnewsrc:%s\nimage size:%d\n"%(imgurl,newsrc,len(data)))
         else:
             newsrc = self.imgtuples[self.imgurls.index(imgurl)]['newsrc']
-            
+
         #print("===============\n%s\nimg url:%s\n============"%(newsrc,self.imgurls[-1]))
-        
+
         return newsrc
 
     def getImgUrls(self):
@@ -645,9 +799,9 @@ class Story(Configurable):
             #parsedUrl = urlparse.urlparse(url)
             retlist.append(self.imgtuples[i])
         return retlist
-    
+
     def __str__(self):
-        return "Metadata: " +str(self.metadata) 
+        return "Metadata: " +str(self.metadata)
 
 def commaGroups(s):
     groups = []

fanficdownloader/writers/writer_epub.py

@@ -361,7 +361,7 @@ div { margin: 0pt; padding: 0pt; }
 
         metadata.appendChild(newTag(contentdom,"dc:contributor",text="fanficdownloader [http://fanficdownloader.googlecode.com]",attrs={"opf:role":"bkp"}))
         metadata.appendChild(newTag(contentdom,"dc:rights",text=""))
-        if self.story.getMetadata('langcode') != None:
+        if self.story.getMetadata('langcode'):
             metadata.appendChild(newTag(contentdom,"dc:language",text=self.story.getMetadata('langcode')))
         else:
             metadata.appendChild(newTag(contentdom,"dc:language",text='en'))

index.html

@@ -46,17 +46,6 @@
       {{yourfile}}
       <!--			</div> -->
 
-          <h3>fanfiction.net / fimfiction.net</h3>
-          <p>
-            Fanfiction.net appears to be blocking access from Google
-            App Engine, which prevents this web service.  There's
-            nothing I can do about it.  At the time of writing, the
-            latest CLI and calibre plugin versions worked.
-          </p>
-          <p>It appears that FimFiction.net is also blocking access from Google
-            App Engine now.
-          </p>
-	  
       {% if authorized %}
       <form action="/fdown" method="post">
 	<div id='urlbox'>
@@ -68,11 +57,11 @@
 	  <h3>Changes:</h3>
 	  <p>
 	    <ul>
-              <li>Fixes for storiesonline.net, thanks davidfor</li>
-              <li>Additional fixes/enhancements for replace_br_with_p heuristic processing feature.  Turn on in personal User Config.  Thanks, Asbjørn Grandt.</li>
+              <li>New site: spikeluver.com -- Thanks, cryzed!</li>
+              <li>New site: bloodshedverse.com -- Thanks, cryzed!</li>
+              <li>Updates for site: voracity2.e-fic.com -- Thanks, cryzed!</li>
 	    </ul>
 	  </p>
-          
 	  <p>
 	    Questions?  Check out our 
 	    <a href="http://code.google.com/p/fanficdownloader/wiki/FanFictionDownloaderFAQs">FAQs</a>.
@@ -81,7 +70,7 @@
 	    If you have any problems with this application, please
 	    report them in
 	    the <a href="http://groups.google.com/group/fanfic-downloader">FanFictionDownLoader Google Group</a>.  The
-	    <a href="http://4-4-80.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
+	    <a href="http://4-5-02.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
 	  </p>
 	  <div id='error'>
 	    {{ error_message }}

makeplugin.py

@@ -24,14 +24,14 @@ from makezip import createZipFile
 
 if __name__=="__main__":
     filename="FanFictionDownLoader.zip"
-    exclude=['*.pyc','*~','*.xcf','*[0-9].png']
+    exclude=['*.pyc','*~','*.xcf','*[0-9].png','*.po','*.pot','*default.mo']
     # from top dir. 'w' for overwrite
     createZipFile(filename,"w",
                   ['plugin-defaults.ini','plugin-example.ini','fanficdownloader','downloader.py','defaults.ini'],
                   exclude=exclude)
     #from calibre-plugin dir. 'a' for append
     os.chdir('calibre-plugin')
-    files=['about.txt','images',]
+    files=['about.txt','images','translations']
     files.extend(glob('*.py'))
     files.extend(glob('plugin-import-name-*.txt'))
     createZipFile("../"+filename,"a",

status.html

@@ -51,16 +51,6 @@
 	      by {{ fic.author }} ({{ fic.format }})
 	  {% endif %}
 	  {% if fic.failure %}
-          <h3>fanfiction.net / fimfiction.net</h3>
-          <p>
-            FYI, fanfiction.net appears to be blocking access from Google
-            App Engine, which prevents this web service.  There's
-            nothing I can do about it.  At the time of writing, the
-            latest CLI and calibre plugin versions worked.
-          </p>
-          <p>It appears that FimFiction.net is also blocking access from Google
-            App Engine now.
-          </p>
 	  <span id='error'>{{ fic.failure }}</span>
 	  {% endif %}
 	  {% if  not fic.completed and not fic.failure %}