Bump version to v2.2.8

Prevent duplicated story URLs even without normalization

README.md

@@ -10,8 +10,10 @@ FFDL, AKA fanficdownloader) project previously hosted as a
 This program is available as a calibre plugin, a command-line
 interface, and a web service.
 
-*Currently, the project is the middle of being moved from code.google
-to GitHub and being renamed.  For now, continue to use
-FanFictionDownLoader(FFDL).*
+FanFicFare has now been launched.  New versions, features and updates
+will all be in FanFicFare.
 
-[code.google project]: https://code.google.com/p/fanficdownloader/
+There's additional info in the project [wiki] pages.
+
+[code.google project]: http://google-opensource.blogspot.com/2015/03/farewell-to-google-code.html
+[wiki]: https://github.com/JimmXinu/FanFicFare/wiki

calibre-plugin/__init__.py

@@ -42,7 +42,7 @@ class FanFicFareBase(InterfaceActionBase):
     description         = _('UI plugin to download FanFiction stories from various sites.')
     supported_platforms = ['windows', 'osx', 'linux']
     author              = 'Jim Miller'
-    version             = (2, 2, 2)
+    version             = (2, 2, 8)
     minimum_calibre_version = (1, 48, 0)
 
     #: This field defines the GUI plugin class that contains all the code

calibre-plugin/about.html

@@ -0,0 +1,32 @@
+<hr />
+
+<p>Plugin created by Jim Miller, borrowing heavily from Grant Drake's
+  '<a href="http://www.mobileread.com/forums/showthread.php?t=134856">Reading List</a>',
+  '<a href="http://www.mobileread.com/forums/showthread.php?t=126727">Extract ISBN</a>' and
+  '<a href="http://www.mobileread.com/forums/showthread.php?t=134000">Count Pages</a>'
+  plugins.</p>
+
+<p>
+  Calibre officially distributes plugins from the mobileread.com forum site.
+  The official distro channel for this plugin is there: <a href="http://www.mobileread.com/forums/showthread.php?t=259221">FanFicFare</a>
+</p>
+
+<p> I also monitor the
+  <a href="http://groups.google.com/group/fanfic-downloader">general users
+    group</a> for the downloader.  That covers the web application and CLI, too.
+</p>
+
+<p>
+  The source for this plugin is available at it's 
+  <a href="https://github.com/JimmXinu/FanFicFare">project home</a>.
+</p>
+
+<hr />
+
+<p>
+  See the <a href="https://github.com/JimmXinu/FanFicFare/wiki/Supportedsites">list of supported sites</a>.
+</p>
+
+<p>
+  Read the <a href="https://github.com/JimmXinu/FanFicFare/wiki/FAQs">FAQs</a>.
+</p>

calibre-plugin/about.txt

@@ -1,28 +0,0 @@
-<hr />
-
-<p>Plugin created by Jim Miller, borrowing heavily from Grant Drake's
-'<a href="http://www.mobileread.com/forums/showthread.php?t=134856">Reading List</a>',
-'<a href="http://www.mobileread.com/forums/showthread.php?t=126727">Extract ISBN</a>' and
-'<a href="http://www.mobileread.com/forums/showthread.php?t=134000">Count Pages</a>'
-plugins.</p>
-
-<p>
-Calibre officially distributes plugins from the mobileread.com forum site.
-The official distro channel for this plugin is there: <a href="http://www.mobileread.com/forums/showthread.php?t=163261">FanFictionDownLoader</a>
-</p>
-
-<p> I also monitor the
-<a href="http://groups.google.com/group/fanfic-downloader">general users
-group</a> for the downloader.  That covers the web application and CLI, too.
-</p>
-
-The source for this plugin is available at it's 
-<a href="https://github.com/JimmXinu/FanFicFare">project home</a>.
-<hr />
-
-<p>
-See the <a href="https://github.com/JimmXinu/FanFicFare/wiki/Supportedsites">list of supported sites</a>.
-</p>
-<p>
-Read the <a href="https://github.com/JimmXinu/FanFicFare/wiki/FAQs">FAQs</a>.
-</p>

calibre-plugin/config.py

@@ -17,12 +17,12 @@ try:
     from PyQt5.Qt import (QDialog, QWidget, QVBoxLayout, QHBoxLayout, QGridLayout,
                           QLabel, QLineEdit, QFont, QWidget, QTextEdit, QComboBox,
                           QCheckBox, QPushButton, QTabWidget, QScrollArea,
-                          QDialogButtonBox, QGroupBox, Qt )
+                          QDialogButtonBox, QGroupBox, QButtonGroup, QRadioButton, Qt)
 except ImportError as e:
     from PyQt4.Qt import (QDialog, QWidget, QVBoxLayout, QHBoxLayout, QGridLayout,
                           QLabel, QLineEdit, QFont, QWidget, QTextEdit, QComboBox,
                           QCheckBox, QPushButton, QTabWidget, QScrollArea,
-                          QDialogButtonBox, QGroupBox, Qt )
+                          QDialogButtonBox, QGroupBox, QButtonGroup, QRadioButton, Qt)
 try:
     from calibre.gui2 import QVariant
     del QVariant
@@ -39,9 +39,15 @@ else:
             return [convert_qvariant(i) for i in x.toList()]
         return x.toPyObject()
 
-from calibre.gui2.ui import get_gui
 from calibre.gui2 import dynamic, info_dialog, question_dialog
-from calibre.constants import numeric_version as calibre_version
+from calibre.gui2.ui import get_gui
+from calibre.gui2.complete2 import EditWithComplete
+
+try:
+    from calibre.ebooks.covers import generate_cover as cal_generate_cover
+    HAS_CALGC=True
+except:
+    HAS_CALGC=False
 
 # pulls in translation files for _() strings
 try:
@@ -49,12 +55,14 @@ try:
 except NameError:
     pass # load_translations() added in calibre 1.9
 
+from calibre.library.field_metadata import FieldMetadata
+field_metadata = FieldMetadata()
+
 # There are a number of things used several times that shouldn't be
 # translated.  This is just a way to make that easier by keeping them
 # out of the _() strings.
 # I'm tempted to override _() to include them...
 no_trans = { 'pini':'personal.ini',
-             'imgset':'\n\n[epub]\ninclude_images:true\nkeep_summary_html:true\nmake_firstimage_cover:true\n\n',
              'gcset':'generate_cover_settings',
              'ccset':'custom_columns_settings',
              'gc':'Generate Cover',
@@ -70,7 +78,12 @@ no_trans = { 'pini':'personal.ini',
              'p':'password',
              }
 
-from calibre_plugins.fanficfare_plugin.prefs import prefs, PREFS_NAMESPACE
+STD_COLS_SKIP = ['size','cover','news','ondevice','path','series_sort','sort']
+
+from calibre_plugins.fanficfare_plugin.prefs \
+    import (prefs, PREFS_NAMESPACE, updatecalcover_order, calcover_save_options,
+            gencalcover_order, SAVE_YES, SAVE_NO)
+
 from calibre_plugins.fanficfare_plugin.dialogs \
     import (UPDATE, UPDATEALWAYS, collision_order, save_collisions, RejectListDialog,
             EditTextDialog, IniTextDialog, RejectUrlEntry)
@@ -84,8 +97,6 @@ from calibre_plugins.fanficfare_plugin.common_utils \
 from calibre_plugins.fanficfare_plugin.fff_util \
     import (test_config)
 
-from calibre.gui2.complete2 import EditWithComplete  #MultiCompleteLineEdit
-
 class RejectURLList:
     def __init__(self,prefs):
         self.prefs = prefs
@@ -205,10 +216,8 @@ class ConfigWidget(QWidget):
         if 'Reading List' not in plugin_action.gui.iactions:
             self.readinglist_tab.setEnabled(False)
 
-        self.generatecover_tab = GenerateCoverTab(self, plugin_action)
-        tab_widget.addTab(self.generatecover_tab, 'Generate Cover')
-        if 'Generate Cover' not in plugin_action.gui.iactions:
-            self.generatecover_tab.setEnabled(False)
+        self.calibrecover_tab = CalibreCoverTab(self, plugin_action)
+        tab_widget.addTab(self.calibrecover_tab, _('Calibre Cover'))
 
         self.countpages_tab = CountPagesTab(self, plugin_action)
         tab_widget.addTab(self.countpages_tab, 'Count Pages')
@@ -234,7 +243,6 @@ class ConfigWidget(QWidget):
         prefs['fileform'] = unicode(self.basic_tab.fileform.currentText())
         prefs['collision'] = save_collisions[unicode(self.basic_tab.collision.currentText())]
         prefs['updatemeta'] = self.basic_tab.updatemeta.isChecked()
-        prefs['updatecover'] = self.basic_tab.updatecover.isChecked()
         prefs['updateepubcover'] = self.basic_tab.updateepubcover.isChecked()
         prefs['keeptags'] = self.basic_tab.keeptags.isChecked()
         prefs['suppressauthorsort'] = self.basic_tab.suppressauthorsort.isChecked()
@@ -247,7 +255,6 @@ class ConfigWidget(QWidget):
         prefs['updatedefault'] = self.basic_tab.updatedefault.isChecked()
         prefs['deleteotherforms'] = self.basic_tab.deleteotherforms.isChecked()
         prefs['adddialogstaysontop'] = self.basic_tab.adddialogstaysontop.isChecked()
-        prefs['includeimages'] = self.basic_tab.includeimages.isChecked()
         prefs['lookforurlinhtml'] = self.basic_tab.lookforurlinhtml.isChecked()
         prefs['checkforseriesurlid'] = self.basic_tab.checkforseriesurlid.isChecked()
         prefs['checkforurlchange'] = self.basic_tab.checkforurlchange.isChecked()
@@ -273,17 +280,23 @@ class ConfigWidget(QWidget):
             # if they've removed everything, reset to default.
             prefs['personal.ini'] = get_resources('plugin-example.ini')
 
-        # Generate Covers tab
-        prefs['gcnewonly'] = self.generatecover_tab.gcnewonly.isChecked()
+        # Covers tab
+        prefs['updatecalcover'] = calcover_save_options[unicode(self.calibrecover_tab.updatecalcover.currentText())]
+        # for backward compatibility:
+        prefs['updatecover'] = prefs['updatecalcover'] == SAVE_YES
+        prefs['gencalcover'] = calcover_save_options[unicode(self.calibrecover_tab.gencalcover.currentText())]
+        prefs['calibre_gen_cover'] = self.calibrecover_tab.calibre_gen_cover.isChecked()
+        prefs['plugin_gen_cover'] = self.calibrecover_tab.plugin_gen_cover.isChecked()
+        prefs['gcnewonly'] = self.calibrecover_tab.gcnewonly.isChecked()
         gc_site_settings = {}
-        for (site,combo) in self.generatecover_tab.gc_dropdowns.iteritems():
+        for (site,combo) in self.calibrecover_tab.gc_dropdowns.iteritems():
             val = unicode(convert_qvariant(combo.itemData(combo.currentIndex())))
             if val != 'none':
                 gc_site_settings[site] = val
                 #print("gc_site_settings[%s]:%s"%(site,gc_site_settings[site]))
         prefs['gc_site_settings'] = gc_site_settings
-        prefs['allow_gc_from_ini'] = self.generatecover_tab.allow_gc_from_ini.isChecked()
-        prefs['gc_polish_cover'] = self.generatecover_tab.gc_polish_cover.isChecked()
+        prefs['allow_gc_from_ini'] = self.calibrecover_tab.allow_gc_from_ini.isChecked()
+        prefs['gc_polish_cover'] = self.calibrecover_tab.gc_polish_cover.isChecked()
 
         # Count Pages tab
         countpagesstats = []
@@ -312,6 +325,9 @@ class ConfigWidget(QWidget):
         # error column
         prefs['errorcol'] = unicode(convert_qvariant(self.cust_columns_tab.errorcol.itemData(self.cust_columns_tab.errorcol.currentIndex())))
 
+        # metadata column
+        prefs['savemetacol'] = unicode(convert_qvariant(self.cust_columns_tab.savemetacol.itemData(self.cust_columns_tab.savemetacol.currentIndex())))
+
         # cust cols tab
         colsmap = {}
         for (col,combo) in self.cust_columns_tab.custcol_dropdowns.iteritems():
@@ -334,6 +350,7 @@ class ConfigWidget(QWidget):
         prefs['imapfolder'] = unicode(self.imap_tab.imapfolder.text())
         prefs['imapmarkread'] = self.imap_tab.imapmarkread.isChecked()
         prefs['imapsessionpass'] = self.imap_tab.imapsessionpass.isChecked()
+        prefs['auto_reject_from_email'] = self.imap_tab.auto_reject_from_email.isChecked()
         
         prefs.save_to_db()
 
@@ -396,21 +413,17 @@ class BasicTab(QWidget):
         horz.addWidget(self.collision)
         self.l.addLayout(horz)
 
+        horz = QHBoxLayout()
         self.updatemeta = QCheckBox(_('Default Update Calibre &Metadata?'),self)
         self.updatemeta.setToolTip(_("On each download, FanFicFare offers an option to update Calibre's metadata (title, author, URL, tags, custom columns, etc) from the web site. <br />This sets whether that will default to on or off. <br />Columns set to 'New Only' in the column tabs will only be set for new books."))
         self.updatemeta.setChecked(prefs['updatemeta'])
-        self.l.addWidget(self.updatemeta)
+        horz.addWidget(self.updatemeta)
 
         self.updateepubcover = QCheckBox(_('Default Update EPUB Cover when Updating EPUB?'),self)
         self.updateepubcover.setToolTip(_("On each download, FanFicFare offers an option to update the book cover image <i>inside</i> the EPUB from the web site when the EPUB is updated.<br />This sets whether that will default to on or off."))
         self.updateepubcover.setChecked(prefs['updateepubcover'])
-        self.l.addWidget(self.updateepubcover)
-
-        self.smarten_punctuation = QCheckBox(_('Smarten Punctuation (EPUB only)'),self)
-        self.smarten_punctuation.setToolTip(_("Run Smarten Punctuation from Calibre's Polish Book feature on each EPUB download and update."))
-        self.smarten_punctuation.setChecked(prefs['smarten_punctuation'])
-        if calibre_version >= (0, 9, 39):
-            self.l.addWidget(self.smarten_punctuation)
+        horz.addWidget(self.updateepubcover)
+        self.l.addLayout(horz)
 
         cali_gb = groupbox = QGroupBox(_("Updating Calibre Options"))
         self.l = QVBoxLayout()
@@ -421,11 +434,6 @@ class BasicTab(QWidget):
         self.deleteotherforms.setChecked(prefs['deleteotherforms'])
         self.l.addWidget(self.deleteotherforms)
         
-        self.updatecover = QCheckBox(_('Update Calibre Cover when Updating Metadata?'),self)
-        self.updatecover.setToolTip(_("Update calibre book cover image from EPUB when metadata is updated.  (EPUB only.)\nDoesn't go looking for new images on 'Update Calibre Metadata Only'."))
-        self.updatecover.setChecked(prefs['updatecover'])
-        self.l.addWidget(self.updatecover)
-
         self.keeptags = QCheckBox(_('Keep Existing Tags when Updating Metadata?'),self)
         self.keeptags.setToolTip(_("Existing tags will be kept and any new tags added.\n%(cmplt)s and %(inprog)s tags will be still be updated, if known.\n%(lul)s tags will be updated if %(lus)s in %(is)s.\n(If Tags is set to 'New Only' in the Standard Columns tab, this has no effect.)")%no_trans)
         self.keeptags.setChecked(prefs['keeptags'])
@@ -466,16 +474,16 @@ class BasicTab(QWidget):
         self.showmarked.setChecked(prefs['showmarked'])
         self.l.addWidget(self.showmarked)
 
+        self.smarten_punctuation = QCheckBox(_('Smarten Punctuation (EPUB only)'),self)
+        self.smarten_punctuation.setToolTip(_("Run Smarten Punctuation from Calibre's Polish Book feature on each EPUB download and update."))
+        self.smarten_punctuation.setChecked(prefs['smarten_punctuation'])
+        self.l.addWidget(self.smarten_punctuation)
+
         self.autoconvert = QCheckBox(_("Automatically Convert new/update books?"),self)
         self.autoconvert.setToolTip(_("Automatically call calibre's Convert for new/update books.\nConverts to the current output format as chosen in calibre's\nPreferences->Behavior settings."))
         self.autoconvert.setChecked(prefs['autoconvert'])
         self.l.addWidget(self.autoconvert)
 
-        self.show_est_time = QCheckBox(_("Show estimated time left?"),self)
-        self.show_est_time.setToolTip(_("When a Progress Bar is shown, show a rough estimate of the time left."))
-        self.show_est_time.setChecked(prefs['show_est_time'])
-        self.l.addWidget(self.show_est_time)
-
         gui_gb = groupbox = QGroupBox(_("GUI Options"))
         self.l = QVBoxLayout()
         groupbox.setLayout(self.l)
@@ -495,16 +503,15 @@ class BasicTab(QWidget):
         self.adddialogstaysontop.setChecked(prefs['adddialogstaysontop'])
         self.l.addWidget(self.adddialogstaysontop)
 
+        self.show_est_time = QCheckBox(_("Show estimated time left?"),self)
+        self.show_est_time.setToolTip(_("When a Progress Bar is shown, show a rough estimate of the time left."))
+        self.show_est_time.setChecked(prefs['show_est_time'])
+        self.l.addWidget(self.show_est_time)
+
         misc_gb = groupbox = QGroupBox(_("Misc Options"))
         self.l = QVBoxLayout()
         groupbox.setLayout(self.l)
 
-        # this is a cheat to make it easier for users to realize there's a new include_images features.
-        self.includeimages = QCheckBox(_("Include images in EPUBs?"),self)
-        self.includeimages.setToolTip(_("Download and include images in EPUB stories.  This is equivalent to adding:%(imgset)s ...to the top of %(pini)s.  Your settings in %(pini)s will override this.")%no_trans)
-        self.includeimages.setChecked(prefs['includeimages'])
-        self.l.addWidget(self.includeimages)
-
         self.injectseries = QCheckBox(_("Inject calibre Series when none found?"),self)
         self.injectseries.setToolTip(_("If no series is found, inject the calibre series (if there is one) so \nit appears on the FanFicFare title page(not cover)."))
         self.injectseries.setChecked(prefs['injectseries'])
@@ -639,15 +646,20 @@ class PersonalIniTab(QWidget):
         self.ini_button.clicked.connect(self.add_ini_button)
         self.l.addWidget(self.ini_button)
         
+        label = QLabel(_("Changes will only be saved if you click 'OK' to leave Customize FanFicFare."))
+        label.setWordWrap(True)
+        self.l.addWidget(label)
+        
         self.defaults = QPushButton(_('View Defaults')+' (plugin-defaults.ini)', self)
         self.defaults.setToolTip(_("View all of the plugin's configurable settings\nand their default settings."))
         self.defaults.clicked.connect(self.show_defaults)
         self.l.addWidget(self.defaults)
 
-        label = QLabel(_("Changes will only be saved if you click 'OK' to leave Customize FanFicFare."))
-        label.setWordWrap(True)
-        self.l.addWidget(label)
-        
+        self.showcalcols = QPushButton(_('Show Calibre Column Names'), self)
+        self.showcalcols.setToolTip(_("FanFicFare passes the Calibre columns into the download/update process.  This will show you the columns available by name."))
+        self.showcalcols.clicked.connect(self.show_showcalcols)
+        self.l.addWidget(self.showcalcols)
+
         self.l.insertStretch(-1)
         # let edit box fill the space.
         
@@ -673,6 +685,27 @@ class PersonalIniTab(QWidget):
         if d.result() == d.Accepted:
             self.personalini = d.get_plain_text()
                 
+    def show_showcalcols(self):
+        lines=[]#[('calibre_std_user_categories',_('User Categories'))]
+        for k,f in field_metadata.iteritems():
+            if f['name'] and k not in STD_COLS_SKIP: # only if it has a human readable name.
+                lines.append(('calibre_std_'+k,f['name']))
+            
+        for k, column in self.plugin_action.gui.library_view.model().custom_columns.iteritems():
+            if k != prefs['savemetacol']:
+                # custom always have name.
+                lines.append(('calibre_cust_'+k[1:],column['name']))
+
+        lines.sort() # sort by key.
+
+        EditTextDialog(self,
+                       '\n'.join(['%s (%s)'%(l,k) for (k,l) in lines]),
+                       icon=self.windowIcon(),
+                       title=_('Calibre Column Entry Names'),
+                       label=_('Label (entry_name)'),
+                       read_only=True,
+                       save_size_name='fff:showcalcols').exec_()
+        
 class ReadingListTab(QWidget):
 
     def __init__(self, parent_dialog, plugin_action):
@@ -708,6 +741,7 @@ class ReadingListTab(QWidget):
         self.send_lists_box.update_items_cache(reading_lists)
         self.send_lists_box.setText(prefs['send_lists'])
         horz.addWidget(self.send_lists_box)
+        self.send_lists_box.setCursorPosition(0)
         self.l.addLayout(horz)
         
         self.addtoreadlists = QCheckBox(_('Add new/updated stories to "To Read" Reading List(s).'),self)
@@ -724,6 +758,7 @@ class ReadingListTab(QWidget):
         self.read_lists_box.update_items_cache(reading_lists)
         self.read_lists_box.setText(prefs['read_lists'])
         horz.addWidget(self.read_lists_box)
+        self.read_lists_box.setCursorPosition(0)
         self.l.addLayout(horz)
         
         self.addtolistsonread = QCheckBox(_('Add stories back to "Send to Device" Reading List(s) when marked "Read".'),self)
@@ -733,14 +768,19 @@ class ReadingListTab(QWidget):
             
         self.l.insertStretch(-1)
         
-class GenerateCoverTab(QWidget):
+class CalibreCoverTab(QWidget):
 
     def __init__(self, parent_dialog, plugin_action):
         self.parent_dialog = parent_dialog
         self.plugin_action = plugin_action
         QWidget.__init__(self)
         
-        self.l = QVBoxLayout()
+        self.gencov_elements=[] ## used to disable/enable when gen
+                                ## cover is off/on.  This is more
+                                ## about being a visual que than real
+                                ## necessary function.
+        
+        topl = self.l = QVBoxLayout()
         self.setLayout(self.l)
 
         try:
@@ -748,8 +788,116 @@ class GenerateCoverTab(QWidget):
             gc_settings = gc_plugin.get_saved_setting_names()
         except KeyError:
             gc_settings= []
+
             
-        label = QLabel(_('The %(gc)s plugin can create cover images for books using various metadata and configurations.  If you have GC installed, FanFicFare can run GC on new downloads and metadata updates.  Pick a GC setting by site or Default.')%no_trans)
+        label = QLabel(_("The Calibre cover image for a downloaded book can come"
+                         " from the story site(if EPUB and images are enabled), or"
+                         " from either Calibre's built-in random cover generator or"
+                         " the %(gc)s plugin.")%no_trans)
+        label.setWordWrap(True)
+        self.l.addWidget(label)
+        self.l.addSpacing(5)
+        
+        tooltip = _("Update Calibre book cover image from EPUB when Calibre metadata is updated.\n"
+                    "Doesn't go looking for new images on 'Update Calibre Metadata Only'.\n"
+                    "Cover in EPUB could be from site or previously injected into the EPUB.\n"
+                    "This comes before Generate Cover so %(gc)s(Plugin) use the image if configured to.")%no_trans
+        horz = QHBoxLayout()
+        label = QLabel(_('Update Calibre Cover (from EPUB):'))
+        label.setToolTip(tooltip)
+        horz.addWidget(label)
+        self.updatecalcover = QComboBox(self)
+        for i in updatecalcover_order:
+            self.updatecalcover.addItem(i)
+        # back compat.  If has own value, use.
+        if prefs['updatecalcover']:
+            self.updatecalcover.setCurrentIndex(self.updatecalcover.findText(calcover_save_options[prefs['updatecalcover']]))
+        elif prefs['updatecover']: # doesn't have own val, set YES if old value set.
+            self.updatecalcover.setCurrentIndex(self.updatecalcover.findText(calcover_save_options[SAVE_YES]))
+        else: # doesn't have own value, old value not set, NO.
+            self.updatecalcover.setCurrentIndex(self.updatecalcover.findText(calcover_save_options[SAVE_NO]))
+        self.updatecalcover.setToolTip(tooltip)
+        label.setBuddy(self.updatecalcover)
+        horz.addWidget(self.updatecalcover)
+        self.l.addLayout(horz)
+
+        tooltip = _("Generate a Calibre book cover image when Calibre metadata is updated.<br />"
+                    "Defaults to 'Yes, Always' for backward compatibility and because %(gc)s(Plugin)"
+                    " will only run if configured for Default or site.")%no_trans
+        horz = QHBoxLayout()
+        label = QLabel(_('Generate Calibre Cover:'))
+        label.setToolTip(tooltip)
+        horz.addWidget(label)
+        self.gencalcover = QComboBox(self)
+        for i in gencalcover_order:
+            self.gencalcover.addItem(i)
+        # back compat.  If has own value, use.
+        # if prefs['gencalcover']:
+        self.gencalcover.setCurrentIndex(self.gencalcover.findText(calcover_save_options[prefs['gencalcover']]))
+        # elif prefs['gencover']: # doesn't have own val, set YES if old value set.
+        #     self.gencalcover.setCurrentIndex(self.gencalcover.findText(calcover_save_options[SAVE_YES]))
+        # else: # doesn't have own value, old value not set, NO.
+        #     self.gencalcover.setCurrentIndex(self.gencalcover.findText(calcover_save_options[SAVE_NO]))
+            
+        self.gencalcover.setToolTip(tooltip)
+        label.setBuddy(self.gencalcover)
+        horz.addWidget(self.gencalcover)
+        self.l.addLayout(horz)
+        self.gencalcover.currentIndexChanged.connect(self.endisable_elements)
+
+        # can't be local or it's destroyed when __init__ is done and
+        # connected things don't fire.
+        self.gencov_rdgrp = QButtonGroup()
+        self.gencov_gb = QGroupBox()
+        horz = QHBoxLayout()
+        self.gencov_gb.setLayout(horz)
+        
+        self.plugin_gen_cover = QRadioButton(_('Plugin %(gc)s')%no_trans,self)
+        self.plugin_gen_cover.setToolTip(_("Use plugin to create covers.  Additional settings are below."))
+        self.gencov_rdgrp.addButton(self.plugin_gen_cover)
+        # always, new only, when no cover from site, inject yes/no...
+        self.plugin_gen_cover.setChecked(prefs['plugin_gen_cover'])
+        horz.addWidget(self.plugin_gen_cover)
+        self.gencov_elements.append(self.plugin_gen_cover)
+
+        self.calibre_gen_cover = QRadioButton(_('Calibre Generate Cover'),self)
+        self.calibre_gen_cover.setToolTip(_("Call Calibre's Edit Metadata Generate cover"
+                                            " feature to create a random cover each time"
+                                            " a story is downloaded or updated.<br />"
+                                            "Right click or long click the 'Generate cover'"
+                                            " button in Calibre's Edit Metadata to customize."))
+        self.gencov_rdgrp.addButton(self.calibre_gen_cover)
+        # always, new only, when no cover from site, inject yes/no...
+        self.calibre_gen_cover.setChecked(prefs['calibre_gen_cover'])
+        horz.addWidget(self.calibre_gen_cover)
+        self.gencov_elements.append(self.calibre_gen_cover)
+
+        #self.l.addLayout(horz)
+        self.l.addWidget(self.gencov_gb)
+
+        self.gcnewonly = QCheckBox(_("Generate Covers Only for New Books")%no_trans,self)
+        self.gcnewonly.setToolTip(_("Default is to generate a cover any time the calibre metadata is"
+                                    " updated.<br \>Used for both Calibre and Plugin generated covers."))
+        self.gcnewonly.setChecked(prefs['gcnewonly'])
+        self.l.addWidget(self.gcnewonly)
+        self.gencov_elements.append(self.gcnewonly)
+
+        self.gc_polish_cover = QCheckBox(_("Inject/update the cover inside EPUB"),self)
+        self.gc_polish_cover.setToolTip(_("Calibre's Polish feature will be used to inject or update the generated"
+                                          " cover into the EPUB ebook file.<br \>Used for both Calibre and Plugin generated covers."))
+        self.gc_polish_cover.setChecked(prefs['gc_polish_cover'])
+        self.l.addWidget(self.gc_polish_cover)
+        self.gencov_elements.append(self.gc_polish_cover)
+
+        self.gcp_gb = QGroupBox(_("%(gc)s(Plugin) Settings")%no_trans)
+        topl.addWidget(self.gcp_gb)
+        self.l = QVBoxLayout()
+        self.gcp_gb.setLayout(self.l)
+        self.gencov_elements.append(self.gcp_gb)
+
+        self.gencov_rdgrp.buttonClicked.connect(self.endisable_elements)
+            
+        label = QLabel(_('The %(gc)s plugin can create cover images for books using various metadata (including existing cover image).  If you have %(gc)s installed, FanFicFare can run %(gc)s on new downloads and metadata updates.  Pick a %(gc)s setting by site and/or one to use by Default.')%no_trans)
         label.setWordWrap(True)
         self.l.addWidget(label)
         self.l.addSpacing(5)
@@ -772,7 +920,7 @@ class GenerateCoverTab(QWidget):
             horz = QHBoxLayout()
             label = QLabel(site)
             if site == _("Default"):
-                s = _("On Metadata update, run %(gc)s with this setting, if not selected for specific site.")%no_trans
+                s = _("On Metadata update, run %(gc)s with this setting, if there isn't a more specific setting below.")%no_trans
             else:
                 no_trans['site']=site # not ideal, but, meh.
                 s = _("On Metadata update, run %(gc)s with this setting for %(site)s stories.")%no_trans
@@ -798,20 +946,34 @@ class GenerateCoverTab(QWidget):
         
         self.sl.insertStretch(-1)
         
-        self.gcnewonly = QCheckBox(_("Run %(gc)s Only on New Books")%no_trans,self)
-        self.gcnewonly.setToolTip(_("Default is to run GC any time the calibre metadata is updated."))
-        self.gcnewonly.setChecked(prefs['gcnewonly'])
-        self.l.addWidget(self.gcnewonly)
-
         self.allow_gc_from_ini = QCheckBox(_('Allow %(gcset)s from %(pini)s to override')%no_trans,self)
-        self.allow_gc_from_ini.setToolTip(_("The %(pini)s parameter %(gcset)s allows you to choose a GC setting based on metadata rather than site, but it's much more complex.<br \>%(gcset)s is ignored when this is off.")%no_trans)
+        self.allow_gc_from_ini.setToolTip(_("The %(pini)s parameter %(gcset)s allows you to choose a %(gc)s setting based on metadata"
+                                            " rather than site, but it's much more complex.<br \>%(gcset)s is ignored when this is off.")%no_trans)
         self.allow_gc_from_ini.setChecked(prefs['allow_gc_from_ini'])
         self.l.addWidget(self.allow_gc_from_ini)
-            
-        self.gc_polish_cover = QCheckBox(_("Use calibre's Polish feature to inject/update the cover"),self)
-        self.gc_polish_cover.setToolTip(_("Calibre's Polish feature will be used to inject or update the generated cover into the ebook, EPUB only."))
-        self.gc_polish_cover.setChecked(prefs['gc_polish_cover'])
-        self.l.addWidget(self.gc_polish_cover)
+
+        # keep at end.
+        self.endisable_elements()
+
+    def endisable_elements(self,button=None):
+        "Clearing house function for setting elements of Calibre"
+        "Cover tab enabled/disabled depending on all factors."
+
+        ## First, cover gen on/off 
+        for e in self.gencov_elements:
+            e.setEnabled(calcover_save_options[unicode(self.gencalcover.currentText())] != SAVE_NO)
+
+        # next, disable plugin settings when using calibre gen cov.
+        if not self.plugin_gen_cover.isChecked():
+            self.gcp_gb.setEnabled(False)
+
+        # disable (but not enable) unsupported options.
+        if not HAS_CALGC:
+            self.calibre_gen_cover.setEnabled(False)
+        if not 'Generate Cover' in self.plugin_action.gui.iactions:
+            self.plugin_gen_cover.setEnabled(False)
+            self.gcp_gb.setEnabled(False)
+        
             
 class CountPagesTab(QWidget):
 
@@ -1080,6 +1242,21 @@ class CustomColumnsTab(QWidget):
         self.errorcol.setCurrentIndex(self.errorcol.findData(prefs['errorcol']))
         horz.addWidget(self.errorcol)
         self.l.addLayout(horz)
+        
+        horz = QHBoxLayout()
+        label = QLabel(_("Saved Metadata Column:"))
+        tooltip=_("If set, FanFicFare will save a copy of all its metadata in this column when the book is downloaded or updated.<br/>The metadata from this column can later be used to update custom columns without having to request the metadata from the server again.<br/>(Long Text columns only.)")
+        label.setToolTip(tooltip)
+        horz.addWidget(label)
+        self.savemetacol = QComboBox(self)
+        self.savemetacol.setToolTip(tooltip)
+        self.savemetacol.addItem('','')
+        for key, column in custom_columns.iteritems():
+            if column['datatype'] in ('comments'):
+                self.savemetacol.addItem(column['name'],key)
+        self.savemetacol.setCurrentIndex(self.savemetacol.findData(prefs['savemetacol']))
+        horz.addWidget(self.savemetacol)
+        self.l.addLayout(horz)
 
         #print("prefs['custom_cols'] %s"%prefs['custom_cols'])
 
@@ -1200,6 +1377,12 @@ class ImapTab(QWidget):
         self.l.addWidget(self.imapmarkread,row,0,1,-1)
         row+=1
             
+        self.auto_reject_from_email = QCheckBox(_('Discard URLs on Reject List'),self)
+        self.auto_reject_from_email.setToolTip(_('If checked, FanFicFare will silently discard story URLs from emails that are on your Reject URL List.<br>Otherwise they will appear and you will see the normal Reject URL dialog.<br>The Emails will still be marked Read if configured to.'))
+        self.auto_reject_from_email.setChecked(prefs['auto_reject_from_email'])
+        self.l.addWidget(self.auto_reject_from_email,row,0,1,-1)
+        row+=1
+        
         label = QLabel(_("<b>It's safest if you create a separate email account that you use only "
                          "for your story update notices.  FanFicFare and calibre cannot guarantee that "
                          "malicious code cannot get your email password once you've entered it. "

calibre-plugin/dialogs.py

@@ -79,14 +79,16 @@ UPDATE=_('Update EPUB if New Chapters')
 UPDATEALWAYS=_('Update EPUB Always')
 OVERWRITE=_('Overwrite if Newer')
 OVERWRITEALWAYS=_('Overwrite Always')
-CALIBREONLY=_('Update Calibre Metadata Only')
+CALIBREONLY=_('Update Calibre Metadata from Web Site')
+CALIBREONLYSAVECOL=_('Update Calibre Metadata from Saved Metadata Column')
 collision_order=[SKIP,
                  ADDNEW,
                  UPDATE,
                  UPDATEALWAYS,
                  OVERWRITE,
                  OVERWRITEALWAYS,
-                 CALIBREONLY,]
+                 CALIBREONLY,
+                 CALIBREONLYSAVECOL,]
 
 # best idea I've had for how to deal with config/pref saving the
 # collision name in english.
@@ -97,6 +99,7 @@ SAVE_UPDATEALWAYS='Update EPUB Always'
 SAVE_OVERWRITE='Overwrite if Newer'
 SAVE_OVERWRITEALWAYS='Overwrite Always'
 SAVE_CALIBREONLY='Update Calibre Metadata Only'
+SAVE_CALIBREONLYSAVECOL='Update Calibre Metadata Only(Saved Column)'
 save_collisions={
     SKIP:SAVE_SKIP,
     ADDNEW:SAVE_ADDNEW,
@@ -112,6 +115,7 @@ save_collisions={
     SAVE_OVERWRITE:OVERWRITE,
     SAVE_OVERWRITEALWAYS:OVERWRITEALWAYS,
     SAVE_CALIBREONLY:CALIBREONLY,
+    SAVE_CALIBREONLYSAVECOL:CALIBREONLYSAVECOL,
     }
     
 anthology_collision_order=[UPDATE,
@@ -122,7 +126,7 @@ gpstyle='QGroupBox {border:0; padding-top:10px; padding-bottom:0px; margin-botto
 
 class RejectUrlEntry:
 
-    matchpat=re.compile(r"^(?P<url>[^,]+)(,(?P<fullnote>(((?P<title>.+) by (?P<auth>.+?)( - (?P<note>.+))?)|.*)))?$")
+    matchpat=re.compile(r"^(?P<url>[^,]+?)(,(?P<fullnote>(((?P<title>.+?) by (?P<auth>.+?)( - (?P<note>.+))?)|.*)))?$")
     
     def __init__(self,url_or_line,note=None,title=None,auth=None,
                  addreasontext=None,fromline=False,book_id=None):
@@ -303,7 +307,7 @@ class AddNewDialog(SizePersistedDialog):
         self.collision.setToolTip("CollisionToolTip")
         # add collision options
         self.set_collisions()
-        i = self.collision.findText(save_collisions[prefs['collision']])
+        i = self.collision.findText(save_collisions[self.prefs['collision']])
         if i > -1:
             self.collision.setCurrentIndex(i)
         self.collisionlabel.setBuddy(self.collision)
@@ -317,14 +321,14 @@ class AddNewDialog(SizePersistedDialog):
         horz = QHBoxLayout()
         self.updatemeta = QCheckBox(_('Update Calibre &Metadata?'),self)
         self.updatemeta.setToolTip(_("Update metadata for existing stories in Calibre from web site?\n(Columns set to 'New Only' in the column tabs will only be set for new books.)"))
-        self.updatemeta.setChecked(prefs['updatemeta'])
+        self.updatemeta.setChecked(self.prefs['updatemeta'])
         horz.addWidget(self.updatemeta)
         self.mergehide.append(self.updatemeta)
         self.mergeupdateshow.append(self.updatemeta)
 
         self.updateepubcover = QCheckBox(_('Update EPUB Cover?'),self)
         self.updateepubcover.setToolTip(_('Update book cover image from site or defaults (if found) <i>inside</i> the EPUB when EPUB is updated.'))
-        self.updateepubcover.setChecked(prefs['updateepubcover'])
+        self.updateepubcover.setChecked(self.prefs['updateepubcover'])
         horz.addWidget(self.updateepubcover)
         self.mergehide.append(self.updateepubcover)
         
@@ -354,7 +358,10 @@ class AddNewDialog(SizePersistedDialog):
                     extrapayload=None):
         # rather than mutex in fff_plugin, just bail here if it's
         # already in use.
-        if self.isVisible(): return
+        if self.isVisible():
+            if url_list_text: # add to open box.
+               self.url.setText( '\n'.join([self.get_urlstext(), url_list_text]) )
+            return
 
         try:
             self.go_signal.disconnect()
@@ -427,12 +434,19 @@ class AddNewDialog(SizePersistedDialog):
         prev=self.collision.currentText()
         self.collision.clear()
         if self.merge:
-            order = anthology_collision_order
+            order = list(anthology_collision_order)
         else:
-            order = collision_order
+            order = list(collision_order)
+            ## Remove options that aren't valid.
+            if self.fileform.currentText() != 'epub':
+                order.remove(UPDATE)
+                order.remove(UPDATEALWAYS)
+            if self.prefs['savemetacol'] == '':
+                order.remove(CALIBREONLYSAVECOL)
+
         for o in order:
-            if self.merge or self.fileform.currentText() == 'epub' or o not in [UPDATE,UPDATEALWAYS]:
-                self.collision.addItem(o)
+            self.collision.addItem(o)
+            
         i = self.collision.findText(prev)
         if i > -1:
             self.collision.setCurrentIndex(i)
@@ -783,7 +797,7 @@ class UpdateExistingDialog(SizePersistedDialog):
         self.fileform.addItem('mobi')
         self.fileform.addItem('html')
         self.fileform.addItem('txt')
-        self.fileform.setCurrentIndex(self.fileform.findText(prefs['fileform']))
+        self.fileform.setCurrentIndex(self.fileform.findText(self.prefs['fileform']))
         self.fileform.setToolTip(_('Choose output format to create.  May set default from plugin configuration.'))
         self.fileform.activated.connect(self.set_collisions)
         label.setBuddy(self.fileform)
@@ -795,7 +809,7 @@ class UpdateExistingDialog(SizePersistedDialog):
         self.collision.setToolTip(_("What sort of update to perform.  May set default from plugin configuration."))
         # add collision options
         self.set_collisions()
-        i = self.collision.findText(save_collisions[prefs['collision']])
+        i = self.collision.findText(save_collisions[self.prefs['collision']])
         if i > -1:
             self.collision.setCurrentIndex(i)
         label.setBuddy(self.collision)
@@ -803,12 +817,12 @@ class UpdateExistingDialog(SizePersistedDialog):
 
         self.updatemeta = QCheckBox(_('Update Calibre &Metadata?'),self)
         self.updatemeta.setToolTip(_("Update metadata for existing stories in Calibre from web site?\n(Columns set to 'New Only' in the column tabs will only be set for new books.)"))
-        self.updatemeta.setChecked(prefs['updatemeta'])
+        self.updatemeta.setChecked(self.prefs['updatemeta'])
         gbl.addWidget(self.updatemeta)
                 
         self.updateepubcover = QCheckBox(_('Update EPUB Cover?'),self)
         self.updateepubcover.setToolTip(_('Update book cover image from site or defaults (if found) <i>inside</i> the EPUB when EPUB is updated.'))
-        self.updateepubcover.setChecked(prefs['updateepubcover'])
+        self.updateepubcover.setChecked(self.prefs['updateepubcover'])
         gbl.addWidget(self.updateepubcover)
 
 
@@ -827,10 +841,18 @@ class UpdateExistingDialog(SizePersistedDialog):
     def set_collisions(self):
         prev=self.collision.currentText()
         self.collision.clear()
-        for o in collision_order:
-            if o not in [ADDNEW,SKIP] and \
-                    (self.fileform.currentText() == 'epub' or o not in [UPDATE,UPDATEALWAYS]):
-                self.collision.addItem(o)        
+        order = list(collision_order)
+        order.remove(ADDNEW)
+        order.remove(SKIP)
+        if self.fileform.currentText() != 'epub':
+            order.remove(UPDATE)
+            order.remove(UPDATEALWAYS)
+        if self.prefs['savemetacol'] == '':
+            order.remove(CALIBREONLYSAVECOL)
+            
+        for o in order:
+            self.collision.addItem(o)
+            
         i = self.collision.findText(prev)
         if i > -1:
             self.collision.setCurrentIndex(i)
@@ -997,6 +1019,7 @@ class RejectListTableWidget(QTableWidget):
         note_cell.set_separator(None)
         note_cell.setToolTip(_('Select or Edit Reject Note.'))
         self.setCellWidget(row, 3, note_cell)
+        note_cell.setCursorPosition(0)
         
     def remove_selected_rows(self):
         self.setFocus()
@@ -1072,6 +1095,7 @@ class RejectListDialog(SizePersistedDialog):
             label.setToolTip(_("This will be added to whatever note you've set for each URL above."))
             horz.addWidget(label)
             horz.addWidget(self.reason_edit)
+            self.reason_edit.setCursorPosition(0)
             horz.insertStretch(-1)
             layout.addLayout(horz)
                     
@@ -1130,6 +1154,7 @@ class EditTextDialog(SizePersistedDialog):
 
     def __init__(self, parent, text,
                  icon=None, title=None, label=None, tooltip=None,
+                 read_only=False,
                  rejectreasons=[],reasonslabel=None,
                  save_size_name='fff:edit text dialog',
                  ):
@@ -1146,6 +1171,7 @@ class EditTextDialog(SizePersistedDialog):
         
         self.textedit = QTextEdit(self)
         self.textedit.setLineWrapMode(QTextEdit.NoWrap)
+        self.textedit.setReadOnly(read_only)
         self.textedit.setText(text)
         self.l.addWidget(self.textedit)
 
@@ -1171,6 +1197,7 @@ class EditTextDialog(SizePersistedDialog):
                 self.l.addLayout(horz)
             else:
                 self.l.addWidget(self.reason_edit)
+            self.reason_edit.setCursorPosition(0)
             
         button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
         button_box.accepted.connect(self.accept)

calibre-plugin/fff_plugin.py

@@ -13,24 +13,24 @@ logger = logging.getLogger(__name__)
 import time, os, copy, threading, re, platform, sys
 from StringIO import StringIO
 from functools import partial
-from datetime import datetime, time
+from datetime import datetime, time, date
 from string import Template
 import urllib
 import email
 import traceback
 
 try:
-    from PyQt5.Qt import (QApplication, QMenu, QTimer)
+    from PyQt5.Qt import (QApplication, QMenu, QTimer, QCursor, Qt)
     from PyQt5.QtCore import QBuffer
 except ImportError as e:
-    from PyQt4.Qt import (QApplication, QMenu, QTimer)
+    from PyQt4.Qt import (QApplication, QMenu, QTimer, QCursor, Qt)
     from PyQt4.QtCore import QBuffer
 
 from calibre.constants import numeric_version as calibre_version
 
 from calibre.ptempfile import PersistentTemporaryFile, PersistentTemporaryDirectory, remove_dir
 from calibre.ebooks.metadata import MetaInformation
-from calibre.ebooks.metadata.meta import get_metadata
+from calibre.ebooks.metadata.meta import get_metadata as calibre_get_metadata
 from calibre.gui2 import error_dialog, warning_dialog, question_dialog, info_dialog
 from calibre.gui2.dialogs.message_box import ViewLog
 from calibre.gui2.dialogs.confirm_delete import confirm
@@ -48,20 +48,45 @@ try:
 except NameError:
     pass # load_translations() added in calibre 1.9
 
-from calibre_plugins.fanficfare_plugin.common_utils import (set_plugin_icon_resources, get_icon,
-                                                            create_menu_action_unique, get_library_uuid)
+try:
+    # should be present from cal2.3.0 on.
+    from calibre.ebooks.covers import generate_cover as cal_generate_cover
+    HAS_CALGC=True
+except:
+    HAS_CALGC=False
 
-from calibre_plugins.fanficfare_plugin.fanficfare import adapters, exceptions
-from calibre_plugins.fanficfare_plugin.fanficfare.epubutils import get_dcsource, get_dcsource_chaptercount, get_story_url_from_html
-from calibre_plugins.fanficfare_plugin.fanficfare.geturls import get_urls_from_page, get_urls_from_html, get_urls_from_text, get_urls_from_imap
+from calibre.library.field_metadata import FieldMetadata
+field_metadata = FieldMetadata()
+    
+from calibre_plugins.fanficfare_plugin.common_utils import (
+    set_plugin_icon_resources, get_icon, create_menu_action_unique,
+    get_library_uuid)
+
+from calibre_plugins.fanficfare_plugin.fanficfare import (
+    adapters, exceptions)
+
+from calibre_plugins.fanficfare_plugin.fanficfare.epubutils import (
+    get_dcsource, get_dcsource_chaptercount, get_story_url_from_html)
+
+from calibre_plugins.fanficfare_plugin.fanficfare.geturls import (
+    get_urls_from_page, get_urls_from_html,get_urls_from_text,
+    get_urls_from_imap)
+
+from calibre_plugins.fanficfare_plugin.fff_util import (
+    get_fff_adapter, get_fff_config, get_fff_personalini)
+
+from calibre_plugins.fanficfare_plugin.config import (
+    permitted_values, rejecturllist, STD_COLS_SKIP)
+
+from calibre_plugins.fanficfare_plugin.prefs import (
+    prefs, SAVE_YES, SAVE_NO, SAVE_YES_IF_IMG, SAVE_YES_UNLESS_IMG)
 
-from calibre_plugins.fanficfare_plugin.fff_util import (get_fff_adapter, get_fff_config, get_fff_personalini)
-from calibre_plugins.fanficfare_plugin.config import (permitted_values, rejecturllist)
-from calibre_plugins.fanficfare_plugin.prefs import prefs
 from calibre_plugins.fanficfare_plugin.dialogs import (
     AddNewDialog, UpdateExistingDialog,
-    LoopProgressDialog, UserPassDialog, AboutDialog, CollectURLDialog, RejectListDialog, EmailPassDialog,
+    LoopProgressDialog, UserPassDialog, AboutDialog, CollectURLDialog,
+    RejectListDialog, EmailPassDialog,
     OVERWRITE, OVERWRITEALWAYS, UPDATE, UPDATEALWAYS, ADDNEW, SKIP, CALIBREONLY,
+    CALIBREONLYSAVECOL,
     NotGoingToDownload, RejectUrlEntry )
 
 # because calibre immediately transforms html into zip and don't want
@@ -350,7 +375,7 @@ class FanFicFarePlugin(InterfaceAction):
         # get_resources will return a dictionary mapping names to bytes. Names that
         # are not found in the zip file will not be in the returned dictionary.
 
-        text = get_resources('about.txt')
+        text = get_resources('about.html')
         AboutDialog(self.gui,self.qaction.icon(),self.version + text).exec_()
 
     def create_menu_item_ex(self, parent_menu, menu_text, image=None, tooltip=None,
@@ -413,17 +438,31 @@ class FanFicFarePlugin(InterfaceAction):
             if prefs['imapsessionpass']:
                 self.imap_pass = imap_pass
 
+        self.busy_cursor()
+        self.gui.status_bar.show_message(_('Fetching Story URLs from Email...'))
         url_list = get_urls_from_imap(prefs['imapserver'],
                                       prefs['imapuser'],
                                       imap_pass,
                                       prefs['imapfolder'],
                                       prefs['imapmarkread'],)
+        reject_list=set()
+        if prefs['auto_reject_from_email']:
+            # need to normalize for reject list.
+            reject_list = set([x for x in url_list if rejecturllist.check(adapters.getNormalStoryURLSite(x)[0])])
+        url_list = url_list - reject_list
 
+        self.gui.status_bar.show_message(_('Finished Fetching Story URLs from Email.'),3000)
+        self.restore_cursor()
+        
         if url_list:
             self.add_dialog("\n".join(url_list),merge=False)
         else:
+            
+            msg = _('No Valid Story URLs Found in Unread Emails.')
+            if reject_list:
+                msg = msg + '<p>'+(_('(%d Story URLs Skipped, on Rejected URL List)')%len(reject_list))+'</p>'
             info_dialog(self.gui, _('Get Story URLs from Email'),
-                        _('No Valid Story URLs Found in Unread Emails.'),
+                        msg,
                         show=True,
                         show_copy_button=False)
 
@@ -444,8 +483,14 @@ class FanFicFarePlugin(InterfaceAction):
             return
         url = u"%s"%d.url.text()
 
+        self.busy_cursor()
+        self.gui.status_bar.show_message(_('Fetching Story URLs from Page...'))
+
         url_list = self.get_urls_from_page(url)
 
+        self.gui.status_bar.show_message(_('Finished Fetching Story URLs from Page.'),3000)
+        self.restore_cursor()
+        
         if url_list:
             self.add_dialog("\n".join(url_list),merge=d.anthology,anthology_url=url)
         else:
@@ -630,6 +675,9 @@ class FanFicFarePlugin(InterfaceAction):
             remove_dir(tdir)
             return
 
+        self.busy_cursor()
+        self.gui.status_bar.show_message(_('Fetching Story URLs for Series...'))
+        
         # get list from identifiers:url/uri if present, but only if
         # it's *not* a valid story URL.
         mergeurl = self.get_story_url(db,book_id)
@@ -638,6 +686,9 @@ class FanFicFarePlugin(InterfaceAction):
 
         url_list_text = "\n".join(url_list)
 
+        self.gui.status_bar.show_message(_('Finished Fetching Story URLs for Series.'),3000)
+        self.restore_cursor()
+        
         #print("urlmapfile:%s"%urlmapfile)
 
         # AddNewDialog collects URLs, format and presents buttons.
@@ -759,10 +810,10 @@ class FanFicFarePlugin(InterfaceAction):
         # No need to do anything with perfs here, but we could.
         prefs
 
-    def make_id_searchstr(self,url):
+    def do_id_search(self,url):
         # older idents can be uri vs url and have | instead of : after
         # http, plus many sites are now switching to https.
-        return 'identifiers:"~ur(i|l):~^%s$"'%re.sub(r'https?\\\:','https?(\:|\|)',re.escape(url))
+        return self.gui.current_db.search_getting_ids('identifiers:"~ur(i|l):~^%s$"'%re.sub(r'https?\\\:','https?(\:|\|)',re.escape(url)),None)
 
     def prep_downloads(self, options, books, merge=False, extrapayload=None):
         '''Fetch metadata for stories from servers, launch BG job when done.'''
@@ -868,8 +919,6 @@ class FanFicFarePlugin(InterfaceAction):
             # book has already been flagged bad for whatever reason.
             return
 
-        skip_date_update = False
-
         adapter = get_fff_adapter(url,fileform)
         ## save and share cookiejar and pagecache between all
         ## downloads.
@@ -880,67 +929,91 @@ class FanFicFarePlugin(InterfaceAction):
             options['cookiejar'] = adapter.get_empty_cookiejar()
         adapter.set_cookiejar(options['cookiejar'])
 
-        # reduce foreground sleep time for ffnet when few books.
-        if 'ffnetcount' in options and \
-                adapter.getConfig('tweak_fg_sleep') and \
-                adapter.getSiteDomain() == 'www.fanfiction.net':
-            minslp = float(adapter.getConfig('min_fg_sleep'))
-            maxslp = float(adapter.getConfig('max_fg_sleep'))
-            dwnlds = float(adapter.getConfig('max_fg_sleep_at_downloads'))
-            m = (maxslp-minslp) / (dwnlds-1)
-            b = minslp - m
-            slp = min(maxslp,m*float(options['ffnetcount'])+b)
-            #print("m:%s b:%s = %s"%(m,b,slp))
-            adapter.set_sleep(slp)
-
-        ## three tries, that's enough if both user/pass & is_adult needed,
-        ## or a couple tries of one or the other
-        for x in range(0,2):
-            try:
-                adapter.getStoryMetadataOnly(get_cover=False)
-            except exceptions.FailedToLogin, f:
-                logger.warn("Login Failed, Need Username/Password.")
-                userpass = UserPassDialog(self.gui,url,f)
-                userpass.exec_() # exec_ will make it act modal
-                if userpass.status:
-                    adapter.username = userpass.user.text()
-                    adapter.password = userpass.passwd.text()
-
-            except exceptions.AdultCheckRequired:
-                if question_dialog(self.gui, _('Are You an Adult?'), '<p>'+
-                                   _("%s requires that you be an adult.  Please confirm you are an adult in your locale:")%url,
-                                   show_copy_button=False):
-                    adapter.is_adult=True
-
-        # let other exceptions percolate up.
-        story = adapter.getStoryMetadataOnly(get_cover=False)
-
-        series = story.getMetadata('series')
-        if not merge and series and prefs['checkforseriesurlid']:
-            # try to find *series anthology* by *seriesUrl* identifier url or uri first.
-            searchstr = self.make_id_searchstr(story.getMetadata('seriesUrl'))
-            identicalbooks = db.search_getting_ids(searchstr, None)
-            # print("searchstr:%s"%searchstr)
-            # print("identicalbooks:%s"%identicalbooks)
-            if len(identicalbooks) > 0 and question_dialog(self.gui, _('Skip Story?'),'''
-                                                              <h3>%s</h3>
-                                                              <p>%s</p>
-                                                              <p>%s</p>
-                                                              <p>%s</p>
-                                                           '''%(
-                                                           _('Skip Anthology Story?'),
-                                                           _('"<b>%s</b>" is in series "<b><a href="%s">%s</a></b>" that you have an anthology book for.')%(story.getMetadata('title'),story.getMetadata('seriesUrl'),series[:series.index(' [')]),
-                                                           _("Click '<b>Yes</b>' to Skip."),
-                                                           _("Click '<b>No</b>' to download anyway.")),
-                                                           show_copy_button=False):
-                book['comment'] = _("Story in Series Anthology(%s).")%series
-                book['title'] = story.getMetadata('title')
-                book['author'] = [story.getMetadata('author')]
-                book['good']=False
-                book['icon']='rotate-right.png'
-                book['status'] = _('Skipped')
-                return
+        if collision in (CALIBREONLY, CALIBREONLYSAVECOL):
+            ## Getting metadata from configured column.
+            custom_columns = self.gui.library_view.model().custom_columns
+            if ( collision in (CALIBREONLYSAVECOL) and
+                 prefs['savemetacol'] != '' and 
+                 prefs['savemetacol'] in custom_columns ):
 
+                savedmeta_book_id = book['calibre_id']
+                # won't have calibre_id if update by URL vs book.
+                if not savedmeta_book_id:
+                    identicalbooks = self.do_id_search(url)
+                    if len(identicalbooks) == 1:
+                        savedmeta_book_id = identicalbooks.pop()
+                        
+                if savedmeta_book_id:
+                    label = custom_columns[prefs['savemetacol']]['label']
+                    savedmetadata = db.get_custom(savedmeta_book_id, label=label, index_is_id=True)
+                else:
+                    savedmetadata = None
+                    
+                if savedmetadata:
+                    # sets flag inside story so getStoryMetadataOnly won't hit server.
+                    adapter.setStoryMetadata(savedmetadata)
+                
+            # let other exceptions percolate up.
+            story = adapter.getStoryMetadataOnly(get_cover=False)
+        else:
+            # reduce foreground sleep time for ffnet when few books.
+            if 'ffnetcount' in options and \
+                    adapter.getConfig('tweak_fg_sleep') and \
+                    adapter.getSiteDomain() == 'www.fanfiction.net':
+                minslp = float(adapter.getConfig('min_fg_sleep'))
+                maxslp = float(adapter.getConfig('max_fg_sleep'))
+                dwnlds = float(adapter.getConfig('max_fg_sleep_at_downloads'))
+                m = (maxslp-minslp) / (dwnlds-1)
+                b = minslp - m
+                slp = min(maxslp,m*float(options['ffnetcount'])+b)
+                #print("m:%s b:%s = %s"%(m,b,slp))
+                adapter.set_sleep(slp)
+    
+            ## three tries, that's enough if both user/pass & is_adult needed,
+            ## or a couple tries of one or the other
+            for x in range(0,2):
+                try:
+                    adapter.getStoryMetadataOnly(get_cover=False)
+                except exceptions.FailedToLogin, f:
+                    logger.warn("Login Failed, Need Username/Password.")
+                    userpass = UserPassDialog(self.gui,url,f)
+                    userpass.exec_() # exec_ will make it act modal
+                    if userpass.status:
+                        adapter.username = userpass.user.text()
+                        adapter.password = userpass.passwd.text()
+    
+                except exceptions.AdultCheckRequired:
+                    if question_dialog(self.gui, _('Are You an Adult?'), '<p>'+
+                                       _("%s requires that you be an adult.  Please confirm you are an adult in your locale:")%url,
+                                       show_copy_button=False):
+                        adapter.is_adult=True
+    
+            # let other exceptions percolate up.
+            story = adapter.getStoryMetadataOnly(get_cover=False)
+    
+            series = story.getMetadata('series')
+            if not merge and series and prefs['checkforseriesurlid']:
+                # try to find *series anthology* by *seriesUrl* identifier url or uri first.
+                identicalbooks = self.do_id_search(story.getMetadata('seriesUrl'))
+                # print("identicalbooks:%s"%identicalbooks)
+                if len(identicalbooks) > 0 and question_dialog(self.gui, _('Skip Story?'),'''
+                                                                  <h3>%s</h3>
+                                                                  <p>%s</p>
+                                                                  <p>%s</p>
+                                                                  <p>%s</p>
+                                                               '''%(
+                                                               _('Skip Anthology Story?'),
+                                                               _('"<b>%s</b>" is in series "<b><a href="%s">%s</a></b>" that you have an anthology book for.')%(story.getMetadata('title'),story.getMetadata('seriesUrl'),series[:series.index(' [')]),
+                                                               _("Click '<b>Yes</b>' to Skip."),
+                                                               _("Click '<b>No</b>' to download anyway.")),
+                                                               show_copy_button=False):
+                    book['comment'] = _("Story in Series Anthology(%s).")%series
+                    book['title'] = story.getMetadata('title')
+                    book['author'] = [story.getMetadata('author')]
+                    book['good']=False
+                    book['icon']='rotate-right.png'
+                    book['status'] = _('Skipped')
+                    return
 
         ################################################################################################################################################33
 
@@ -950,7 +1023,9 @@ class FanFicFarePlugin(InterfaceAction):
 
         # all_metadata duplicates some data, but also includes extra_entries, etc.
         book['all_metadata'] = story.getAllMetadata(removeallentities=True)
-
+        # get metadata to save in configured column.
+        book['savemetacol'] = story.dump_html_metadata()
+        
         book['title'] = story.getMetadata("title", removeallentities=True)
         book['author_sort'] = book['author'] = story.getList("author", removeallentities=True)
         book['publisher'] = story.getMetadata("site")
@@ -977,7 +1052,7 @@ class FanFicFarePlugin(InterfaceAction):
             book['timestamp'] = None # need *something* there for calibre.
 
         if not merge:# skip all the collision code when d/ling for merging.
-            if collision in (CALIBREONLY):
+            if collision in (CALIBREONLY, CALIBREONLYSAVECOL):
                 book['icon'] = 'metadata.png'
                 book['status'] = _('Meta')
 
@@ -995,9 +1070,7 @@ class FanFicFarePlugin(InterfaceAction):
                 logger.debug("from URL(%s)"%url)
 
                 # try to find by identifier url or uri first.
-                searchstr = self.make_id_searchstr(url)
-                identicalbooks = db.search_getting_ids(searchstr, None)
-                # print("searchstr:%s"%searchstr)
+                identicalbooks = self.do_id_search(url)
                 # print("identicalbooks:%s"%identicalbooks)
                 if len(identicalbooks) < 1:
                     # find dups
@@ -1018,7 +1091,7 @@ class FanFicFarePlugin(InterfaceAction):
                     raise NotGoingToDownload(_("More than one identical book by Identifer URL or title/author(s)--can't tell which book to update/overwrite."),"minusminus.png")
 
                 ## changed: add new book when CALIBREONLY if none found.
-                if collision == CALIBREONLY and not identicalbooks:
+                if collision in (CALIBREONLY, CALIBREONLYSAVECOL) and not identicalbooks:
                     collision = ADDNEW
                     options['collision'] = ADDNEW
 
@@ -1073,7 +1146,7 @@ class FanFicFarePlugin(InterfaceAction):
                                 return
 
             if book_id != None and collision != ADDNEW:
-                if collision in (CALIBREONLY):
+                if collision in (CALIBREONLY, CALIBREONLYSAVECOL):
                     book['comment'] = _('Metadata collected.')
                     # don't need temp file created below.
                     return
@@ -1091,9 +1164,6 @@ class FanFicFarePlugin(InterfaceAction):
                         if chaptercount == urlchaptercount:
                             if collision == UPDATE:
                                 raise NotGoingToDownload(_("Already contains %d chapters.")%chaptercount,'edit-undo.png')
-                            else:
-                                # UPDATEALWAYS
-                                skip_date_update = True
                         elif chaptercount > urlchaptercount:
                             raise NotGoingToDownload(_("Existing epub contains %d chapters, web site only has %d. Use Overwrite to force update.") % (chaptercount,urlchaptercount),'dialog_error.png')
                         elif chaptercount == 0:
@@ -1132,6 +1202,51 @@ class FanFicFarePlugin(InterfaceAction):
                     #print("calibre_series:%s [%s]"%book['calibre_series'])
 
         if book['good']: # there shouldn't be any !'good' books at this point.
+            
+            ## Filling calibre_std_* and calibre_cust_* metadata
+            book['calibre_columns']={}
+            # std columns
+            mi = db.get_metadata(book['calibre_id'],index_is_id=True)
+            # book['calibre_columns']['calibre_std_identifiers']=\
+            #     {'val':', '.join(["%s:%s"%(k,v) for (k,v) in mi.get_identifiers().iteritems()]),
+            #                      'label':_('Ids')}
+            for k in mi.standard_field_keys():
+            # for k in mi:
+                if k in STD_COLS_SKIP:
+                    continue
+                (label,value,v,fmd) = mi.format_field_extended(k)
+                if not label and k in field_metadata:
+                    label=field_metadata[k]['name']
+                key='calibre_std_'+k
+                
+                # if k == 'user_categories':
+                #     value=u', '.join(mi.get(k))
+                #     label=_('User Categories')
+    
+                if label: # only if it has a human readable name.
+                    if value is None or not book['calibre_id']:
+                        ## if existing book, populate existing calibre column
+                        ## values in metadata, else '' to hide.
+                        value=''                    
+                    book['calibre_columns'][key]={'val':value,'label':label}
+                    #logger.debug("%s(%s): %s"%(label,key,value))
+    
+            # custom columns
+            for k, column in self.gui.library_view.model().custom_columns.iteritems():
+                if k != prefs['savemetacol']:
+                    key='calibre_cust_'+k[1:]
+                    label=column['name']
+                    value=db.get_custom(book['calibre_id'],
+                                        label=column['label'],
+                                        index_is_id=True)
+                    # custom always have name.
+                    if value is None or not book['calibre_id']:
+                        ## if existing book, populate existing calibre column
+                        ## values in metadata, else '' to hide.
+                        value=''
+                    book['calibre_columns'][key]={'val':value,'label':label}
+                    # logger.debug("%s(%s): %s"%(label,key,value))
+        
             # if still 'good', make a temp file to write the output to.
             # For HTML format users, make the filename inside the zip something reasonable.
             # For crazy long titles/authors, limit it to 200chars.
@@ -1143,17 +1258,6 @@ class FanFicFarePlugin(InterfaceAction):
             logger.debug("outfile:"+tmp.name)
             book['outfile'] = tmp.name
 
-            # cookiejar = PersistentTemporaryFile(prefix=story.formatFileName("${title}-${author}-",allowunsafefilename=False)[:100],
-            #                                     suffix='.cookiejar',
-            #                                     dir=options['tdir'])
-            # adapter.save_cookiejar(cookiejar.name)
-            # book['cookiejar'] = cookiejar.name
-            # pagecache = PersistentTemporaryFile(prefix=story.formatFileName("${title}-${author}-",allowunsafefilename=False)[:100],
-            #                                     suffix='.pagecache',
-            #                                     dir=options['tdir'])
-            # adapter.save_pagecache(pagecache.name)
-            # book['pagecache'] = pagecache.name
-
         return
 
     def start_download_job(self,book_list,
@@ -1170,7 +1274,7 @@ class FanFicFarePlugin(InterfaceAction):
 
         ## No need to BG process when CALIBREONLY!  Fake it.
         #print("options:%s"%options)
-        if options['collision'] == CALIBREONLY:
+        if options['collision'] in (CALIBREONLY, CALIBREONLYSAVECOL):
             class NotJob(object):
                 def __init__(self,result):
                     self.failed=False
@@ -1255,10 +1359,10 @@ class FanFicFarePlugin(InterfaceAction):
         logger.debug("add/update %s %s"%(book['title'],book['url']))
         mi = self.make_mi_from_book(book)
 
-        if options['collision'] != CALIBREONLY:
+        if options['collision'] not in (CALIBREONLY, CALIBREONLYSAVECOL):
             self.add_book_or_update_format(book,options,prefs,mi)
 
-        if options['collision'] == CALIBREONLY or \
+        if options['collision'] in (CALIBREONLY, CALIBREONLYSAVECOL) or \
                 ( (options['updatemeta'] or book['added']) and book['good'] ):
             try:
                 self.update_metadata(db, book['calibre_id'], book, mi, options)
@@ -1285,7 +1389,7 @@ class FanFicFarePlugin(InterfaceAction):
         failed_list = filter(lambda x : not x['good'] , book_list)
         failed_ids = [ x['calibre_id'] for x in failed_list ]
 
-        if options['collision'] != CALIBREONLY and \
+        if options['collision'] not in (CALIBREONLY, CALIBREONLYSAVECOL) and \
                 (prefs['addtolists'] or prefs['addtoreadlists']):
             self.update_reading_lists(all_ids,add=True)
 
@@ -1349,7 +1453,7 @@ class FanFicFarePlugin(InterfaceAction):
             if countpagesstats:
                 cp_plugin.count_statistics(all_ids,countpagesstats)
 
-        if prefs['autoconvert'] and options['collision'] != CALIBREONLY:
+        if prefs['autoconvert'] and options['collision'] not in (CALIBREONLY, CALIBREONLYSAVECOL):
             self.gui.status_bar.show_message(_('Starting auto conversion of %d books.')%(len(all_ids)), 3000)
             self.gui.iactions['Convert Books'].auto_convert_auto_add(all_ids)
 
@@ -1609,15 +1713,6 @@ class FanFicFarePlugin(InterfaceAction):
             if not oldmi.languages:
                 mi.languages=['en']
 
-        if options['fileform'] == 'epub' and prefs['updatecover']:
-            existingepub = db.format(book_id,'EPUB',index_is_id=True, as_file=True)
-            epubmi = get_metadata(existingepub,'EPUB')
-            if epubmi.cover_data[1] is not None:
-                try:
-                    db.set_cover(book_id, epubmi.cover_data[1])
-                except:
-                    logger.info("Failed to set_cover, skipping")
-
         # implement 'newonly' flags here by setting to the current
         # value again.
         if not book['added']:
@@ -1640,6 +1735,11 @@ class FanFicFarePlugin(InterfaceAction):
         #print("all_metadata: %s"%book['all_metadata'])
         custom_columns = self.gui.library_view.model().custom_columns
 
+        # save metadata to configured column
+        if 'savemetacol' in book and prefs['savemetacol'] != '' and prefs['savemetacol'] in custom_columns:
+            label = custom_columns[prefs['savemetacol']]['label']
+            self.set_custom(db, book_id, 'comment', book['savemetacol'], label=label, commit=True) # book['comment'] book['savemetacol'] = story.dump_html_metadata()
+
         #print("prefs['custom_cols'] %s"%prefs['custom_cols'])
         for col, meta in prefs['custom_cols'].iteritems():
             #print("setting %s to %s"%(col,meta))
@@ -1757,73 +1857,105 @@ class FanFicFarePlugin(InterfaceAction):
 
         db.commit()
 
-        if 'Generate Cover' in self.gui.iactions and (book['added'] or not prefs['gcnewonly']):
-
-            #logger.debug("Do Generate Cover added:%s gcnewonly:%s"%(book['added'],prefs['gcnewonly']))
-
-            # force a refresh if generating cover so complex composite
-            # custom columns are current and correct
-            db.refresh_ids([book_id])
-
-            gc_plugin = self.gui.iactions['Generate Cover']
-            setting_name = None
-            if prefs['allow_gc_from_ini']:
-                if not configuration: # might already have it from allow_custcol_from_ini
-                    configuration = get_fff_config(book['url'],options['fileform'])
-
-                # template => regexp to match => GC Setting to use.
-                # generate_cover_settings:
-                # ${category} => Buffy:? the Vampire Slayer => Buffy
-                # for line in configuration.getConfig('generate_cover_settings').splitlines():
-                #     if "=>" in line:
-                #         (template,regexp,setting) = map( lambda x: x.strip(), line.split("=>") )
-                for (template,regexp,setting) in configuration.get_generate_cover_settings():
-                    value = Template(template).safe_substitute(book['all_metadata']).encode('utf8')
-                    # print("%s(%s) => %s => %s"%(template,value,regexp,setting))
-                    if re.search(regexp,value):
-                        setting_name = setting
-                        break
+        # updating calibre cover from book.
+        if options['fileform'] == 'epub' and (
+            (prefs['updatecover'] and not prefs['updatecalcover']) ## backward compat
+            or prefs['updatecalcover'] == SAVE_YES ## yes, always
+            or (prefs['updatecalcover'] == SAVE_YES_IF_IMG ## yes, if image.
+                and book['all_metadata']['cover_image'] )): # in ('specific','first','default','old')
+            existingepub = db.format(book_id,'EPUB',index_is_id=True, as_file=True)
+            epubmi = calibre_get_metadata(existingepub,'EPUB')
+            if epubmi.cover_data[1] is not None:
+                try:
+                    db.set_cover(book_id, epubmi.cover_data[1])
+                except:
+                    logger.info("Failed to set_cover, skipping")
 
+        # First, should cover generation happen at all?
+        if (book['added'] or not prefs['gcnewonly']) and ( # skip if not new book and gcnewonly is True
+            prefs['gencalcover'] == SAVE_YES ## yes, always
+            or (prefs['gencalcover'] == SAVE_YES_UNLESS_IMG ## yes, unless image.
+                and book['all_metadata']['cover_image'] not in ('specific','first','default')) ):
+        
+            cover_generated = False # flag for polish below.
+            # Yes, should do gencov.  Which?
+            if prefs['calibre_gen_cover'] and HAS_CALGC:
+                # calibre's builtin, if available.
+                cdata = cal_generate_cover(mi)
+                db.set_cover(book_id, cdata)
+                cover_generated = True
+            elif prefs['plugin_gen_cover'] and 'Generate Cover' in self.gui.iactions:
+                # plugin, if available.
+    
+                #logger.debug("Do Generate Cover added:%s gcnewonly:%s"%(book['added'],prefs['gcnewonly']))
+    
+                # force a refresh if generating cover so complex composite
+                # custom columns are current and correct
+                db.refresh_ids([book_id])
+    
+                gc_plugin = self.gui.iactions['Generate Cover']
+                setting_name = None
+                if prefs['allow_gc_from_ini']:
+                    if not configuration: # might already have it from allow_custcol_from_ini
+                        configuration = get_fff_config(book['url'],options['fileform'])
+    
+                    # template => regexp to match => GC Setting to use.
+                    # generate_cover_settings:
+                    # ${category} => Buffy:? the Vampire Slayer => Buffy
+                    # for line in configuration.getConfig('generate_cover_settings').splitlines():
+                    #     if "=>" in line:
+                    #         (template,regexp,setting) = map( lambda x: x.strip(), line.split("=>") )
+                    for (template,regexp,setting) in configuration.get_generate_cover_settings():
+                        value = Template(template).safe_substitute(book['all_metadata']).encode('utf8')
+                        print("%s(%s) => %s => %s"%(template,value,regexp,setting))
+                        if re.search(regexp,value):
+                            setting_name = setting
+                            break
+    
+                    if setting_name:
+                        logger.debug("Generate Cover Setting from generate_cover_settings(%s)"%setting_name)
+                        if setting_name not in gc_plugin.get_saved_setting_names():
+                            logger.info("GC Name %s not found, discarding! (check personal.ini for typos)"%setting_name)
+                            setting_name = None
+    
+                if not setting_name and book['all_metadata']['site'] in prefs['gc_site_settings']:
+                    setting_name =  prefs['gc_site_settings'][book['all_metadata']['site']]
+                    logger.debug("Generate Cover Setting from site(%s)"%setting_name)
+    
+                if not setting_name and 'Default' in prefs['gc_site_settings']:
+                    setting_name =  prefs['gc_site_settings']['Default']
+                    logger.debug("Generate Cover Setting from Default(%s)"%setting_name)
+    
                 if setting_name:
-                    logger.debug("Generate Cover Setting from generate_cover_settings(%s)"%setting_name)
-                    if setting_name not in gc_plugin.get_saved_setting_names():
-                        logger.info("GC Name %s not found, discarding! (check personal.ini for typos)"%setting_name)
-                        setting_name = None
-
-            if not setting_name and book['all_metadata']['site'] in prefs['gc_site_settings']:
-                setting_name =  prefs['gc_site_settings'][book['all_metadata']['site']]
-                logger.debug("Generate Cover Setting from site(%s)"%setting_name)
-
-            if not setting_name and 'Default' in prefs['gc_site_settings']:
-                setting_name =  prefs['gc_site_settings']['Default']
-                logger.debug("Generate Cover Setting from Default(%s)"%setting_name)
-
-            if setting_name:
-                logger.debug("Running Generate Cover with settings %s."%setting_name)
-                realmi = db.get_metadata(book_id, index_is_id=True)
-                gc_plugin.generate_cover_for_book(realmi,saved_setting_name=setting_name)
-
-                if prefs['gc_polish_cover'] and \
-                        options['fileform'] == "epub":
-                    # set cover inside epub from calibre's polish feature
-                    from calibre.ebooks.oeb.polish.main import polish, ALL_OPTS
-                    from calibre.utils.logging import Log
-                    from collections import namedtuple
-
-                    # Couldn't find a better way to get the cover path.
-                    cover_path = os.path.join(db.library_path, db.path(book_id, index_is_id=True), 'cover.jpg')
-                    data = {'cover':cover_path}
-                    #print("cover_path:%s"%cover_path)
-                    opts = ALL_OPTS.copy()
-                    opts.update(data)
-                    O = namedtuple('Options', ' '.join(ALL_OPTS.iterkeys()))
-                    opts = O(**opts)
-
-                    log = Log(level=Log.DEBUG)
-                    outfile = db.format_abspath(book_id, formmapping[options['fileform']], index_is_id=True)
-                    #print("polish cover outfile:%s"%outfile)
-                    polish({outfile:outfile}, opts, log, logger.info)
-
+                    logger.debug("Running Generate Cover with settings %s."%setting_name)
+                    realmi = db.get_metadata(book_id, index_is_id=True)
+                    gc_plugin.generate_cover_for_book(realmi,saved_setting_name=setting_name)
+                    cover_generated = True
+    
+            if cover_generated and prefs['gc_polish_cover'] and \
+                    options['fileform'] == "epub":
+                # set cover inside epub from calibre's polish feature
+                from calibre.ebooks.oeb.polish.main import polish, ALL_OPTS
+                from calibre.utils.logging import Log
+                from collections import namedtuple
+    
+                # Couldn't find a better way to get the cover path.
+                cover_path = os.path.join(db.library_path,
+                                          db.path(book_id, index_is_id=True),
+                                          'cover.jpg')
+                data = {'cover':cover_path}
+                #print("cover_path:%s"%cover_path)
+                opts = ALL_OPTS.copy()
+                opts.update(data)
+                O = namedtuple('Options', ' '.join(ALL_OPTS.iterkeys()))
+                opts = O(**opts)
+    
+                log = Log(level=Log.DEBUG)
+                outfile = db.format_abspath(book_id,
+                                            formmapping[options['fileform']],
+                                            index_is_id=True)
+                #print("polish cover outfile:%s"%outfile)
+                polish({outfile:outfile}, opts, log, logger.info)
 
     def get_clean_reading_lists(self,lists):
         if lists == None or lists.strip() == "" :
@@ -2018,7 +2150,7 @@ class FanFicFarePlugin(InterfaceAction):
             existingepub = None
             if path == None and db.has_format(book_id,'EPUB',index_is_id=True):
                 existingepub = db.format(book_id,'EPUB',index_is_id=True, as_file=True)
-                mi = get_metadata(existingepub,'EPUB')
+                mi = calibre_get_metadata(existingepub,'EPUB')
                 identifiers = mi.get_identifiers()
                 if 'url' in identifiers:
                     # print("url from get_metadata:%s"%identifiers['url'].replace('|',':'))
@@ -2172,6 +2304,12 @@ class FanFicFarePlugin(InterfaceAction):
 
         return book
 
+    def busy_cursor(self):
+        QApplication.setOverrideCursor(QCursor(Qt.WaitCursor))
+
+    def restore_cursor(self):
+        QApplication.restoreOverrideCursor()
+    
 def split_text_to_urls(urls):
     # remove dups while preserving order.
     dups=set()

calibre-plugin/fff_util.py

@@ -18,15 +18,7 @@ from calibre_plugins.fanficfare_plugin.fanficfare.configurable import Configurat
 from calibre_plugins.fanficfare_plugin.prefs import prefs
 
 def get_fff_personalini():
-    if prefs['includeimages']:
-        # this is a cheat to make it easier for users.
-        return '''[epub]
-include_images:true
-keep_summary_html:true
-make_firstimage_cover:true
-''' + prefs['personal.ini']
-    else:
-        return prefs['personal.ini']
+    return prefs['personal.ini']
 
 def get_fff_config(url,fileform="epub",personalini=None):
     if not personalini:

calibre-plugin/jobs.py

@@ -11,6 +11,7 @@ import logging
 logger = logging.getLogger(__name__)
 
 import time, traceback
+from StringIO import StringIO
     
 from calibre.utils.ipc.server import Server
 from calibre.utils.ipc.job import ParallelJob
@@ -99,7 +100,7 @@ def do_download_for_worker(book,options,notification=lambda x,y:x):
     with fffbase:
         
         from calibre_plugins.fanficfare_plugin.dialogs import (NotGoingToDownload,
-                OVERWRITE, OVERWRITEALWAYS, UPDATE, UPDATEALWAYS, ADDNEW, SKIP, CALIBREONLY)
+                OVERWRITE, OVERWRITEALWAYS, UPDATE, UPDATEALWAYS, ADDNEW, SKIP, CALIBREONLY, CALIBREONLYSAVECOL)
         from calibre_plugins.fanficfare_plugin.fanficfare import adapters, writers, exceptions
         from calibre_plugins.fanficfare_plugin.fanficfare.epubutils import get_update_data
         
@@ -151,9 +152,11 @@ def do_download_for_worker(book,options,notification=lambda x,y:x):
             outfile = book['outfile']
     
             ## No need to download at all.  Shouldn't ever get down here.
-            if options['collision'] in (CALIBREONLY):
+            if options['collision'] in (CALIBREONLY, CALIBREONLYSAVECOL):
                 logger.info("Skipping CALIBREONLY 'update' down inside worker--this shouldn't be happening...")
                 book['comment'] = 'Metadata collected.'
+                book['all_metadata'] = story.getAllMetadata(removeallentities=True)
+                book['savemetacol'] = story.dump_html_metadata()
                 
             ## checks were done earlier, it's new or not dup or newer--just write it.
             elif options['collision'] in (ADDNEW, SKIP, OVERWRITE, OVERWRITEALWAYS) or \
@@ -161,23 +164,19 @@ def do_download_for_worker(book,options,notification=lambda x,y:x):
     
                 # preserve logfile even on overwrite.
                 if 'epub_for_update' in book:
-                    (urlignore,
-                     chaptercountignore,
-                     oldchaptersignore,
-                     oldimgsignore,
-                     oldcoverignore,
-                     calibrebookmarkignore,
-                     # only logfile set in adapter, so others aren't used.
-                     adapter.logfile) = get_update_data(book['epub_for_update'])
-    
+                    
+                    adapter.logfile = get_update_data(book['epub_for_update'])[6]
                     # change the existing entries id to notid so
                     # write_epub writes a whole new set to indicate overwrite.
                     if adapter.logfile:
                         adapter.logfile = adapter.logfile.replace("span id","span notid")
                 
                 logger.info("write to %s"%outfile)
+                inject_cal_cols(book,story,configuration)
                 writer.writeStory(outfilename=outfile, forceOverwrite=True)
                 book['comment'] = 'Download %s completed, %s chapters.'%(options['fileform'],story.getMetadata("numChapters"))
+                book['all_metadata'] = story.getAllMetadata(removeallentities=True)
+                book['savemetacol'] = story.dump_html_metadata()
                 
             ## checks were done earlier, just update it.
             elif 'epub_for_update' in book and options['collision'] in (UPDATE, UPDATEALWAYS):
@@ -191,12 +190,14 @@ def do_download_for_worker(book,options,notification=lambda x,y:x):
                  adapter.oldimgs,
                  adapter.oldcover,
                  adapter.calibrebookmark,
-                 adapter.logfile) = get_update_data(book['epub_for_update'])
+                 adapter.logfile) = get_update_data(book['epub_for_update'])[0:7]
     
                 # dup handling from fff_plugin needed for anthology updates.
                 if options['collision'] == UPDATE:
                     if chaptercount == urlchaptercount:
                         book['comment']=_("Already contains %d chapters.  Reuse as is.")%chaptercount
+                        book['all_metadata'] = story.getAllMetadata(removeallentities=True)
+                        book['savemetacol'] = story.dump_html_metadata()
                         book['outfile'] = book['epub_for_update'] # for anthology merge ops.
                         return book
     
@@ -211,10 +212,13 @@ def do_download_for_worker(book,options,notification=lambda x,y:x):
                 logger.info("Do update - epub(%d) vs url(%d)" % (chaptercount, urlchaptercount))
                 logger.info("write to %s"%outfile)
     
+                inject_cal_cols(book,story,configuration)
                 writer.writeStory(outfilename=outfile, forceOverwrite=True)
                 
                 book['comment'] = _('Update %s completed, added %s chapters for %s total.')%\
                     (options['fileform'],(urlchaptercount-chaptercount),urlchaptercount)
+                book['all_metadata'] = story.getAllMetadata(removeallentities=True)
+                book['savemetacol'] = story.dump_html_metadata()
             
             if options['smarten_punctuation'] and options['fileform'] == "epub" \
                     and calibre_version >= (0, 9, 39):
@@ -249,3 +253,22 @@ def do_download_for_worker(book,options,notification=lambda x,y:x):
             
         #time.sleep(10)
     return book
+
+## calibre's columns for an existing book are pased in and injected
+## into the story's metadata.  For convenience, we also add labels and
+## valid_entries for them in a special [injected] section that has
+## even less precedence than [defaults]
+def inject_cal_cols(book,story,configuration):
+    configuration.remove_section('injected')
+    if 'calibre_columns' in book:
+        injectini = ['[injected]']
+        extra_valid = []
+        for k, v in book['calibre_columns'].iteritems():
+            story.setMetadata(k,v['val'])
+            injectini.append('%s_label:%s'%(k,v['label']))
+            extra_valid.append(k)
+        if extra_valid: # if empty, there's nothing to add.
+            injectini.append("add_to_extra_valid_entries:,"+','.join(extra_valid))
+            configuration.readfp(StringIO('\n'.join(injectini)))
+            #print("added:\n%s\n"%('\n'.join(injectini)))
+    

calibre-plugin/plugin-defaults.ini

@@ -235,9 +235,18 @@ connect_timeout:60.0
 ## To use this, make sure you go to the Generate Cover tab in FanFicFare
 ## config and check 'Allow generate_cover_settings from personal.ini
 ## to override'
+## You can check for an existing cover image in the metadata
+## cover_image.  cover_image can be:
+## specific -- The site has an image that is specifically this story's cover
+## first -- The first image in the story or story desc is used as cover
+## default -- A default_cover_image was used
+## old -- There was already a cover image in the epub.
+## This is useful because GC plugin can use the existing image.
 #generate_cover_settings:
 # ${category} => Buffy:? [tT]he Vampire Slayer => BuffyCover
 # ${category} => Star Trek => StarTrekCover
+# ${cover_image} => => CompleteCover
+# ${cover_image} => (specific|first|default) => CoverWithOrigImage
 
 ## If set false, the summary will have all html stripped.
 ## Both this and include_images must be true to get images in the
@@ -442,12 +451,12 @@ output_css:
 ## include images from img tags in the body and summary of
 ## stories.  Images will be converted to jpg for size if possible.
 ## include_images is *only* available in epub and html output format.
-#include_images:false
+include_images:true
 
 ## If set, the first image found will be made the cover image.  If
 ## keep_summary_html is true, any images in summary will be before any
 ## in chapters.
-#make_firstimage_cover: false
+make_firstimage_cover: true
 
 ## If set, the epub will never have a cover, even include_images is on
 ## and the site has specific cover images.
@@ -584,6 +593,18 @@ hits_label:Hits
 collections_label:Collections
 bookmarks_label:Bookmarks
 
+## AO3 doesn't have anything it calls 'genre'.  The adapter used to be
+## hardcoded to include the site specific metadata freeformtags &
+## ao3categories in the standard metadata field genre.  By making it
+## configurable, users can change it.
+include_in_genre: freeformtags, ao3categories
+
+## AO3 uses the word 'category' differently than most sites.  The
+## adapter used to be hardcoded to include the site specific metadata
+## fandom in the standard metadata field category.  By making it
+## configurable, users can change it.
+include_in_category:fandoms
+
 ## freeformtags was previously typo'ed as freefromtags.  This way,
 ## freefromtags will still work for people who've used it.
 include_in_freefromtags:freeformtags
@@ -595,7 +616,7 @@ include_in_freefromtags:freeformtags
 #extra_subject_tags:fandoms,freeformtags,ao3categories
 
 ## AO3 chapters can include several different types of notes.  We've
-## traditional included them all in the chapter text, but this allows
+## traditionally included them all in the chapter text, but this allows
 ## you to customize which you include.  Copy this parameter to your
 ## personal.ini and list the ones you don't want.
 #exclude_notes:authorheadnotes,chaptersummary,chapterheadnotes,chapterfootnotes,authorfootnotes
@@ -745,7 +766,7 @@ extraships:Spike/Buffy
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
-## Virtually all eFiction-based sites allow downloading the whole story in
+## Virtually all eFiction Base adapters allow downloading the whole story in
 ## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
 ## metadata and chapters can be loaded in one step
 bulk_load:true
@@ -930,7 +951,7 @@ extraships:Harry Potter/Hermione Granger
 #username:YourName
 #password:yourpassword
 
-## Virtually all eFiction-based sites allow downloading the whole story in
+## Virtually all eFiction Base adapters allow downloading the whole story in
 ## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
 ## metadata and chapters can be loaded in one step
 bulk_load:true
@@ -1070,6 +1091,23 @@ extracategories:In Death
 ## cover image.  This lets you exclude them.
 cover_exclusion_regexp:/public/style_emoticons/.*
 
+[it-could-happen.net]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+## RPF == http://en.wikipedia.org/wiki/Real_person_fiction
+## Site dedicated to these categories/characters/ships
+extracategories:Glee RPF
+extracharacters:Darren Criss, Chris Colfer
+extraships:Darren Criss/Chris Colfer
+
+## Virtually all eFiction Base adapters allow downloading the whole story in
+## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
+## metadata and chapters can be loaded in one step
+bulk_load:true
+
 [ksarchive.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:Star Trek
@@ -1089,7 +1127,7 @@ eroticatags_label:Erotica Tags
 extra_titlepage_entries: eroticatags
 
 [lotrfanfiction.com]
-## Virtually all eFiction-based sites allow downloading the whole story in
+## Virtually all eFiction Base adapters allow downloading the whole story in
 ## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
 ## metadata and chapters can be loaded in one step
 bulk_load:true
@@ -1343,7 +1381,7 @@ extracategories:Transgender
 ## confirm they are adult for adult content.
 #is_adult:true
 
-## Virtually all eFiction-based sites allow downloading the whole story in
+## Virtually all eFiction Base adapters allow downloading the whole story in
 ## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
 ## metadata and chapters can be loaded in one step
 bulk_load:true
@@ -1367,7 +1405,7 @@ extraships:Harry Potter/Draco Malfoy
 ## personal.ini, not defaults.ini.
 #is_adult:true
 
-## Virtually all eFiction-based sites allow downloading the whole story in
+## Virtually all eFiction Base adapters allow downloading the whole story in
 ## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
 ## metadata and chapters can be loaded in one step
 bulk_load:true
@@ -1381,7 +1419,7 @@ extracategories:Criminal Minds
 ## personal.ini, not defaults.ini.
 #is_adult:true
 
-## Virtually all eFiction-based sites allow downloading the whole story in
+## Virtually all eFiction Base adapters allow downloading the whole story in
 ## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
 ## metadata and chapters can be loaded in one step
 bulk_load:true
@@ -1454,6 +1492,8 @@ extra_titlepage_entries:readings,awards
 awards_label:Awards
 readings_label:Readings
 
+## some sites include images that we don't ever want becoming the
+## cover image.  This lets you exclude them.
 cover_exclusion_regexp:art/.*Awards.jpg
 
 [voracity2.e-fic.com]
@@ -1650,6 +1690,10 @@ dateUpdated_format:%%Y-%%m-%%d %%H:%%M:%%S
 ## to on, but can be switched off if it is found to cause problems.
 fix_fimf_blockquotes:true
 
+## some sites include images that we don't ever want becoming the
+## cover image.  This lets you exclude them.
+cover_exclusion_regexp:/images/emoticons/
+
 ## Site dedicated to these categories/characters/ships
 extracategories:My Little Pony: Friendship is Magic
 
@@ -1733,7 +1777,7 @@ extraships:InuYasha/Kagome
 ## Site dedicated to these categories/characters/ships
 extracategories:Lord of the Rings
 
-## Virtually all eFiction-based sites allow downloading the whole story in
+## Virtually all eFiction Base adapters allow downloading the whole story in
 ## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
 ## metadata and chapters can be loaded in one step
 bulk_load:true

calibre-plugin/prefs.py

@@ -15,9 +15,32 @@ import copy
 from calibre.utils.config import JSONConfig
 from calibre.gui2.ui import get_gui
 
-from calibre_plugins.fanficfare_plugin.dialogs import UPDATE
+from calibre_plugins.fanficfare_plugin.dialogs import SAVE_UPDATE
 from calibre_plugins.fanficfare_plugin.common_utils import get_library_uuid
 
+# Show translated strings, but save the same string in prefs so your
+# prefs are the same in different languages.
+YES=_('Yes, Always')
+SAVE_YES='Yes'
+YES_IF_IMG=_('Yes, if EPUB has a cover image')
+SAVE_YES_IF_IMG='Yes, if img'
+YES_UNLESS_IMG=_('Yes, unless FanFicFare found a cover image')
+SAVE_YES_UNLESS_IMG='Yes, unless img'
+NO=_('No')
+SAVE_NO='No'
+calcover_save_options = {
+    YES:SAVE_YES,
+    SAVE_YES:YES,
+    YES_IF_IMG:SAVE_YES_IF_IMG,
+    SAVE_YES_IF_IMG:YES_IF_IMG,
+    YES_UNLESS_IMG:SAVE_YES_UNLESS_IMG,
+    SAVE_YES_UNLESS_IMG:YES_UNLESS_IMG,
+    NO:SAVE_NO,
+    SAVE_NO:NO,
+    }
+updatecalcover_order=[YES,YES_IF_IMG,NO]
+gencalcover_order=[YES,YES_UNLESS_IMG,NO]
+
 # if don't have any settings for FanFicFarePlugin, copy from
 # predecessor FanFictionDownLoaderPlugin.
 FFDL_PREFS_NAMESPACE = 'FanFictionDownLoaderPlugin'
@@ -35,7 +58,6 @@ Dup from another site'''
 default_prefs['reject_always'] = False
 
 default_prefs['updatemeta'] = True
-default_prefs['updatecover'] = False
 default_prefs['updateepubcover'] = False
 default_prefs['keeptags'] = False
 default_prefs['suppressauthorsort'] = False
@@ -46,10 +68,9 @@ default_prefs['autoconvert'] = False
 default_prefs['urlsfromclip'] = True
 default_prefs['updatedefault'] = True
 default_prefs['fileform'] = 'epub'
-default_prefs['collision'] = UPDATE
+default_prefs['collision'] = SAVE_UPDATE
 default_prefs['deleteotherforms'] = False
 default_prefs['adddialogstaysontop'] = False
-default_prefs['includeimages'] = False
 default_prefs['lookforurlinhtml'] = False
 default_prefs['checkforseriesurlid'] = True
 default_prefs['checkforurlchange'] = True
@@ -63,6 +84,11 @@ default_prefs['addtolists'] = False
 default_prefs['addtoreadlists'] = False
 default_prefs['addtolistsonread'] = False
 
+default_prefs['updatecalcover'] = None
+default_prefs['gencalcover'] = SAVE_YES
+default_prefs['updatecover'] = False
+default_prefs['calibre_gen_cover'] = False
+default_prefs['plugin_gen_cover'] = True
 default_prefs['gcnewonly'] = False
 default_prefs['gc_site_settings'] = {}
 default_prefs['allow_gc_from_ini'] = True
@@ -72,6 +98,7 @@ default_prefs['countpagesstats'] = []
 default_prefs['wordcountmissing'] = False
 
 default_prefs['errorcol'] = ''
+default_prefs['savemetacol'] = ''
 default_prefs['custom_cols'] = {}
 default_prefs['custom_cols_newonly'] = {}
 default_prefs['allow_custcol_from_ini'] = True
@@ -84,6 +111,7 @@ default_prefs['imappass'] = ''
 default_prefs['imapsessionpass'] = False
 default_prefs['imapfolder'] = 'INBOX'
 default_prefs['imapmarkread'] = True
+default_prefs['auto_reject_from_email'] = False
 
 def set_library_config(library_config,db):
     db.prefs.set_namespaced(PREFS_NAMESPACE,

calibre-plugin/translations/README.md

@@ -0,0 +1,6 @@
+# Translations
+
+If you're interested in helping provide translations for this project,
+please use the
+[Transifex](https://www.transifex.com/projects/p/calibre-plugins/resources/)
+website to add translations to this, or other calibre plugins that support it.

fanficfare/adapters/__init__.py

@@ -134,6 +134,7 @@ import adapter_fhsarchivecom
 import adapter_fanfictionjunkiesde
 import adapter_devianthearts
 import adapter_tgstorytimecom
+import adapter_itcouldhappennet
 
 ## This bit of complexity allows adapters to be added by just adding
 ## importing.  It eliminates the long if/else clauses we used to need
@@ -196,6 +197,12 @@ def getAdapter(config,url,anyurl=False):
 def getConfigSections():
     return [cls.getConfigSection() for cls in __class_list]
 
+def get_bulk_load_sites():
+    # for now, all eFiction Base adapters are assumed to allow bulk_load.
+    return [cls.getConfigSection().replace('www.','') for cls in
+            filter( lambda x : issubclass(x,base_efiction_adapter.BaseEfictionAdapter),
+                    __class_list)]
+
 def getSiteExamples():
     l=[]
     for cls in sorted(__class_list, key=lambda x : x.getConfigSection()):

fanficfare/adapters/adapter_archiveofourownorg.py

@@ -230,7 +230,6 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
         fandoms = a.findAll('a',{'class':"tag"})
         for fandom in fandoms:
             self.story.addToList('fandoms',fandom.string)
-            self.story.addToList('category',fandom.string)
 		
         a = metasoup.find('dd',{'class':"warning tags"})
         if a != None:
@@ -243,7 +242,6 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
             genres = a.findAll('a',{'class':"tag"})
             for genre in genres:
                 self.story.addToList('freeformtags',genre.string)
-                self.story.addToList('genre',genre.string)
                 
         a = metasoup.find('dd',{'class':"category tags"})
         if a != None:
@@ -251,7 +249,6 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
             for genre in genres:
                 if genre != "Gen":
                     self.story.addToList('ao3categories',genre.string)
-                    self.story.addToList('genre',genre.string)
 		
         a = metasoup.find('dd',{'class':"character tags"})
         if a != None:

fanficfare/adapters/adapter_ficwadcom.py

@@ -135,7 +135,7 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
         self.story.setMetadata('title', stripHTML(titleh4.a))
 
         # Find authorid and URL from... author url.
-        a = soup.find('span',{'class':'author'}).find('a', href=re.compile(r"^/author/\d+"))
+        a = soup.find('span',{'class':'author'}).find('a', href=re.compile(r"^/a/"))
         self.story.setMetadata('authorId',a['href'].split('/')[2])
         self.story.setMetadata('authorUrl','http://'+self.host+a['href'])
         self.story.setMetadata('author',a.string)

fanficfare/adapters/adapter_fimfictionnet.py

@@ -176,7 +176,10 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
             if coverurl.startswith('//'): # fix for img urls missing 'http:'
                 coverurl = "http:"+coverurl
             if get_cover:
-                self.setCoverImage(self.url,coverurl)
+                # try setting from href, if fails, try using the img src
+                if self.setCoverImage(self.url,coverurl)[0] == "failedtoload":
+                    coverurl = storyImage.find('img')['src']
+                    self.setCoverImage(self.url,coverurl)
 
             coverSource = storyImage.find('a', {'class':'source'})
             if coverSource:

fanficfare/adapters/adapter_grangerenchantedcom.py

@@ -176,7 +176,7 @@ class GrangerEnchantedCom(BaseSiteAdapter):
             raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
             
         # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
+        soup = self.make_soup(data)
         # print data
 
         # Now go hunting for all the meta data and the chapter list.
@@ -217,11 +217,10 @@ class GrangerEnchantedCom(BaseSiteAdapter):
             if 'Summary' in label:
                 ## Everything until the next span class='label'
                 svalue = ""
-                while not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
+                while value and not defaultGetattr(value,'class') == 'label' and '<span class="label">' not in unicode(value):
+                    svalue += unicode(value)
                     value = value.nextSibling
                 self.setDescription(url,svalue)
-                #self.story.setMetadata('description',stripHTML(svalue))
 
             if 'Rated' in label:
                 self.story.setMetadata('rating', value)
@@ -271,7 +270,7 @@ class GrangerEnchantedCom(BaseSiteAdapter):
             series_url = 'http://'+self.host+'/'+self.section+'/'+a['href']
 
             # use BeautifulSoup HTML parser to make everything easier to find.
-            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            seriessoup = self.make_soup(self._fetchUrl(series_url))
             # can't use ^viewstory...$ in case of higher rated stories with javascript href.
             storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
             i=1
@@ -300,8 +299,7 @@ class GrangerEnchantedCom(BaseSiteAdapter):
 
         logger.debug('Getting chapter text from: %s' % url)
 
-        soup = bs.BeautifulSoup(self._fetchUrl(url),
-                                selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        soup = self.make_soup(self._fetchUrl(url))
         
         div = soup.find('div', {'id' : 'story1'})
 

fanficfare/adapters/adapter_itcouldhappennet.py

@@ -0,0 +1,46 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2015 FanFicFare team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Software: eFiction
+import re
+from base_efiction_adapter import BaseEfictionAdapter
+
+class ItCouldHappenNetSiteAdapter(BaseEfictionAdapter):
+
+    @staticmethod
+    def getSiteDomain():
+        return 'it-could-happen.net'
+
+    @classmethod
+    def getSiteAbbrev(seluuf):
+        return 'ich'
+
+    @classmethod
+    def getDateFormat(self):
+        return "%B %d, %Y"
+
+    def handleMetadataPair(self, key, value):
+        # This site is all one 'category' as it's usually defined and
+        # uses Category for what is usually genre.
+        if key == 'Categories':
+            for val in re.split("\s*,\s*", value):
+                self.story.addToList('genre', val)
+        else:
+            super(ItCouldHappenNetSiteAdapter, self).handleMetadataPair(key, value)
+
+def getClass():
+    return ItCouldHappenNetSiteAdapter

fanficfare/adapters/adapter_ponyfictionarchivenet.py

@@ -171,7 +171,8 @@ class PonyFictionArchiveNetAdapter(BaseSiteAdapter):
             self.story.addToList('warnings',warning.string)
         
         status = soup.find('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
-        self.story.setMetadata('status',status.string)
+        if status: # apparently this site can have stories with neither In-Progress or Complete.
+            self.story.setMetadata('status',status.string)
         
         section = soup.findAll('span', {'class' : 'General'})[1]            
         

fanficfare/adapters/adapter_storiesonlinenet.py

@@ -21,7 +21,7 @@ logger = logging.getLogger(__name__)
 import re
 import urllib2
 
-from .. import BeautifulSoup as bs
+#from .. import BeautifulSoup as bs
 from ..htmlcleanup import stripHTML
 from .. import exceptions as exceptions
 
@@ -40,22 +40,22 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
         self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
         self.password = ""
         self.is_adult=False
-        
+
         # get storyId from url
         self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2].split(':')[0])
         if 'storyInfo' in self.story.getMetadata('storyId'):
             self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        
+
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/s/'+self.story.getMetadata('storyId'))
-        
+
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','strol')
 
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%Y-%m-%d"
-            
+
     @staticmethod # must be @staticmethod, don't remove it.
     def getSiteDomain():
         # The site domain.  Does have www here, if it uses it.
@@ -78,7 +78,7 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
                 or "Access to unlinked chapters requires" in data:
             self.needToLogin = True
         return self.needToLogin
-        
+
     def performLogin(self, url):
         params = {}
 
@@ -91,13 +91,13 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
         params['rememberMe'] = '1'
         params['page'] = 'http://'+self.getSiteDomain()+'/'
         params['submit'] = 'Login'
-    
+
         loginUrl = 'http://' + self.getSiteDomain() + '/login.php'
         logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['theusername']))
-    
+
         d = self._fetchUrl(loginUrl, params,usecache=False)
-    
+
         if "My Account" not in d : #Member Account
             logger.info("Failed to login to URL %s as %s" % (loginUrl,
                                                               params['theusername']))
@@ -114,7 +114,7 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
         return True
 
     ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
+    def doExtractChapterUrlsAndMetadata(self, get_cover=True):
 
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
@@ -137,22 +137,22 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
             # need to log in for this one.
             self.performLogin(url)
             data = self._fetchUrl(url+":i",usecache=False)
-        
+
         if "Access denied. This story has not been validated by the adminstrators of this site." in data:
             raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
         elif "Error! The story you're trying to access is being filtered by your choice of contents filtering." in data:
             raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Error! The story you're trying to access is being filtered by your choice of contents filtering.")
-            
+
         # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
+        soup = self.make_soup(data)
         #print data
 
         # Now go hunting for all the meta data and the chapter list.
-        
+
         ## Title
         a = soup.find('h1')
         self.story.setMetadata('title',stripHTML(a))
-        
+
         notice = soup.find('div', {'class' : 'notice'})
         if notice:
             self.story.setMetadata('notice',unicode(notice))
@@ -178,7 +178,7 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
         page=0
         i=0
         while i == 0:
-            asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getList('authorUrl')[0]+"/"+str(page)))
+            asoup = self.make_soup(self._fetchUrl(self.story.getList('authorUrl')[0]+"/"+str(page)))
 
             a = asoup.findAll('td', {'class' : 'lc2'})
             for lc2 in a:
@@ -196,8 +196,8 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
 
         score = lc2.findNext('th', {'class' : 'num'}).text
         if score != '-':
-            self.story.setMetadata('score', score)        
-        
+            self.story.setMetadata('score', score)
+
         lc4 = lc2.findNext('td', {'class' : 'lc4'})
         desc = lc4.contents[0]
 
@@ -212,7 +212,7 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
                 self.story.setMetadata('seriesUrl',seriesUrl)
                 series_name = stripHTML(a)
                 logger.debug("Series name= %s" % series_name)
-                series_soup = bs.BeautifulSoup(self._fetchUrl(seriesUrl))
+                series_soup = self.make_soup(self._fetchUrl(seriesUrl))
                 if series_soup:
                     logger.debug("Retrieving Series - looking for name")
                     series_name = series_soup.find('span', {'id' : 'ptitle'}).text.partition(' — ')[0]
@@ -221,7 +221,7 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
                 desc = lc4.contents[2]
                 # Check if series is in a universe
                 universe_url = self.story.getList('authorUrl')[0]  + "&type=uni"
-                universes_soup = bs.BeautifulSoup(self._fetchUrl(universe_url) )
+                universes_soup = self.make_soup(self._fetchUrl(universe_url) )
                 logger.debug("Universe url='{0}'".format(universe_url))
                 if universes_soup:
                     universes = universes_soup.findAll('div', {'class' : 'ser-box'})
@@ -255,11 +255,11 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
                 universe_name = stripHTML(a)
                 universeUrl = 'http://'+self.host+a['href']
                 logger.debug("Retrieving Universe - about to get page - universeUrl='{0}".format(universeUrl))
-                universe_soup = bs.BeautifulSoup(self._fetchUrl(universeUrl))
+                universe_soup = self.make_soup(self._fetchUrl(universeUrl))
                 logger.debug("Retrieving Universe - have page")
                 if universe_soup:
                     logger.debug("Retrieving Universe - looking for name")
-                    universe_name = universe_soup.find('h1', {'id' : 'ptitle'}).text.partition(' —')[0]
+                    universe_name = universe_soup.find('h1', {'id' : 'ptitle'}).text.partition('—')[0]
                     logger.debug("Universes name: '{0}'".format(universe_name))
 
                 self.story.setMetadata('universeUrl',universeUrl)
@@ -274,7 +274,7 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
             pass
 
         self.setDescription('http://'+self.host+'/s/'+self.story.getMetadata('storyId'),desc)
-            
+
         for b in lc4.findAll('b'):
             #logger.debug('Getting metadata: "%s"' % b)
             label = b.text
@@ -284,24 +284,38 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
             else:
                 value = b.nextSibling
             #logger.debug('label: "%s", value: "%s"' % (label, value))
-            
+
             if 'Sex' in label:
                 self.story.setMetadata('rating', value)
-                
+
             if 'Tags' in label or 'Codes' in label:
                 for code in re.split(r'\s*,\s*', value.strip()):
                      self.story.addToList('sitetags',code)
-                    
+
             if 'Posted' in label:
                 self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
                 self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-                
+
             if 'Concluded' in label:
                 self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-                
+
             if 'Updated' in label:
                 self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-#                
+
+        # Some books have a cover in the index page.
+        # Samples are:
+        #     http://storiesonline.net/s/11999
+        #     http://storiesonline.net/s/10823
+        if get_cover:
+            logger.debug("Looking for the cover image...")
+            cover_url = ""
+            img = soup.find('img')
+            if img:
+                cover_url=img['src']
+            logger.debug("cover_url: %s"%cover_url)
+            if cover_url:
+                self.setCoverImage(url,cover_url)
+
         status = lc4.find('span', {'class' : 'ab'})
         if  status != None:
             self.story.setMetadata('status', 'In-Progress')
@@ -312,111 +326,146 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
         else:
             self.story.setMetadata('status', 'Completed')
 
-            
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
         logger.debug('Getting chapter text from: %s' % url)
 
-        soup = bs.BeautifulSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.find('div', {'id' : 'story'})
-        
+        soup = self.make_soup(self._fetchUrl(url))
+
+        chaptertag = soup.find('div', {'id' : 'story'})
+        if not chaptertag:
+            logger.debug("div id=story not found, try article")
+            chaptertag = soup.find('article', {'id' : 'story'})
+
         # some big chapters are split over several pages
-        pager = div.find('span', {'class' : 'pager'})
+        pager = chaptertag.find('span', {'class' : 'pager'})
+
+        self.cleanPage(chaptertag)
+
         if pager != None:
-            a = pager.previousSibling
-            while a != None:
-                logger.debug("before pager: {0}".format(a))
-                b = a.previousSibling
-                a.extract()
-                a = b
 
             urls=pager.findAll('a')
             urls=urls[:len(urls)-1]
+            logger.debug("pager urls:%s"%urls)
             pager.extract()
-            div.contents = div.contents[2:]
-#            logger.debug(div)
-                        
+            chaptertag.contents = chaptertag.contents[2:]
+
             for ur in urls:
-                soup = bs.BeautifulSoup(self._fetchUrl("http://"+self.getSiteDomain()+ur['href']),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-                div1 = soup.find('div', {'id' : 'story'})
-                
-                # Find the "Continues" marker on the current page and remove everything after that. 
-                continues = div.find('span', {'class' : 'conTag'})
-                if continues != None:
-                    while continues != None:
-#                        logger.debug("removing end: {0}".format(continues))
-                        b = continues.nextSibling
-                        continues.extract()
-                        continues = b
+                soup = self.make_soup(self._fetchUrl("http://"+self.getSiteDomain()+ur['href']))
 
-                # Find the "Continued" marker and delete everything before that
-                continued = div1.find('span', {'class' : 'conTag'})
-                if continued != None:
-                    a = continued.previousSibling
-                    while a != None:
-#                        logger.debug("before conTag: {0}".format(a))
-                        b = a.previousSibling
-                        a.extract()
-                        a = b
-                # Remove the pager from the end if this is the last page
-                endPager = div1.find('span', {'class' : 'pager'})
-                if endPager != None:
-                    b = endPager.nextSibling
-                    while endPager != None:
-                        logger.debug("removing end: {0}".format(endPager))
-                        b = endPager.nextSibling
-                        endPager.extract()
-                        endPager = b
-                    div1.contents = div1.contents[:len(div1) - 2]
-#                logger.debug("after removing pager: {0}".format(div1))
-                for tag in div1.contents[2:]:
-                    div.append(tag)
+                pagetag = soup.find('div', {'id' : 'story'})
+                if not pagetag:
+                    logger.debug("div id=story not found, try article")
+                    pagetag = soup.find('article', {'id' : 'story'})
 
-        # If it is a chapter, there are dates at the start for when it was posted or modified. These plus 
-        # everything before them can be discarded. 
-        postedDates = div.findAll('div', {'class' : 'date'})
+                self.cleanPage(pagetag)
+
+                for tag in pagetag.contents[2:]:
+                    chaptertag.append(tag)
+
+
+        if None == chaptertag:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+
+        return self.utf8FromSoup(url,chaptertag)
+
+    def cleanPage(self,pagetag):
+        "Consolidate 'page' clean up code so it can be called."
+
+        # some big chapters are split over several pages
+        # remove FIRST pager and everything before it.
+        tag = pagetag.find('span', {'class' : 'pager'})
+        while tag != None:
+            # logger.debug("remove before pager: {0}".format(tag))
+            prev = tag.previousSibling
+            tag.extract()
+            tag = prev
+
+        # Find the "Continues" marker on the current page and
+        # remove everything after that.  This is actually
+        # effecting the *previous* 'page'.  EXCEPT!--they are
+        # putting a 'conTag' at the *top* now, too.  So this
+        # was nuking every page but the first and last.  Now
+        # only if 'Continues'
+        for contag in pagetag.findAll('span', {'class' : 'conTag'}):
+            # remove everything after continues...
+            if 'Continuation' in contag.text:
+                tag = contag
+                while tag != None:
+                    # logger.debug("remove before Continuation: {0}".format(tag))
+                    prev = tag.previousSibling
+                    tag.extract()
+                    tag = prev
+            elif 'Continues' in contag.text:
+                tag = contag
+                while tag != None:
+                    # logger.debug("remove after Continues: {0}".format(tag))
+                    nxt = tag.nextSibling
+                    tag.extract()
+                    tag = nxt
+
+        # some big chapters are split over several pages
+        # remove LAST pager and everything before it.
+        # Only needed on last page.
+        tag = pagetag.find('span', {'class' : 'pager'})
+        while tag != None:
+            # logger.debug("remove after pager: {0}".format(tag))
+            nxt = tag.nextSibling
+            tag.extract()
+            tag = nxt
+
+        # If it is a chapter, there are dates at the start for when it was posted or modified. These plus
+        # everything before them can be discarded.
+        postedDates = pagetag.findAll('div', {'class' : 'date'})
         if postedDates:
             a = postedDates[0].previousSibling
             while a != None:
-#                logger.debug("before dates: {0}".format(a))
+                # logger.debug("before dates: {0}".format(a))
                 b = a.previousSibling
                 a.extract()
                 a = b
-            for a in div.findAll('div', {'class' : 'date'}):
+            for a in pagetag.findAll('div', {'class' : 'date'}):
                 a.extract()
 
         # For single chapter stories, there is a copyright statement. Remove this and everything
         # before it.
-        copy = div.find('h4', {'class': 'copy'})
+        copy = pagetag.find('h4', {'class': 'copy'})
         while copy != None:
-#            logger.debug("before copyright: {0}".format(copy))
+            # logger.debug("before copyright: {0}".format(copy))
             b = copy.previousSibling
             copy.extract()
             copy = b
 
         # For a story or the last chapter, remove voting form and the in library box
-        a = div.find('div', {'id' : 'vote-form'})
+        a = pagetag.find('div', {'id' : 'vote-form'})
         if a != None:
             a.extract()
-        a = div.find('div', {'id' : 'b-man-div'})
+        a = pagetag.find('div', {'id' : 'top-header'})
+        if a != None:
+            a.extract()
+        a = pagetag.find('div', {'id' : 'b-man-div'})
         if a != None:
             a.extract()
 
-        # Kill the "The End" header and everything after it.
-        a = div.find(['h2', 'h3'], {'class' : 'end'})
-        logger.debug("Chapter end= '{0}'".format(a))
+        # Kill the vote form and everything after it.
+        a = pagetag.find('div', {'class' : 'vform'})
+        # logger.debug("Chapter end= '{0}'".format(a))
         while a != None:
             b = a.nextSibling
             a.extract()
             a=b
 
+        # Kill the vote form and everything after it.
+        a = pagetag.find('h3', {'class' : 'end'})
+        # logger.debug("Chapter end= '{0}'".format(a))
+        while a != None:
+            b = a.nextSibling
+            a.extract()
+            a=b
+
+        foot = pagetag.find('footer')
+        if foot != None:
+            foot.extract()
+
 
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)

fanficfare/adapters/base_adapter.py

@@ -379,10 +379,12 @@ class BaseSiteAdapter(Configurable):
                                                                self.getConfig('allow_unsafe_filename')),
                                      self._fetchUrlRaw,
                                      cover=True)
+                self.story.setMetadata('cover_image','default')
 
             # no new cover, set old cover, if there is one.
             if not self.story.cover and self.oldcover:
                 self.story.oldcover = self.oldcover
+                self.story.setMetadata('cover_image','old')
                 
             # cheesy way to carry calibre bookmark file forward across update.
             if self.calibrebookmark:
@@ -402,6 +404,13 @@ class BaseSiteAdapter(Configurable):
             self.metadataDone = True
         return self.story
 
+    def setStoryMetadata(self,metahtml):
+        if metahtml:
+            self.story.load_html_metadata(metahtml)
+            self.metadataDone = True
+            if not self.story.getMetadataRaw('dateUpdated'):
+                self.story.setMetadata('dateUpdated',self.story.getMetadataRaw('datePublished'))
+    
     def hookForUpdates(self,chaptercount):
         "Usually not needed."
         return chaptercount
@@ -479,8 +488,10 @@ class BaseSiteAdapter(Configurable):
 
     def setCoverImage(self,storyurl,imgurl):
         if self.getConfig('include_images'):
-            self.story.addImgUrl(storyurl,imgurl,self._fetchUrlRaw,cover=True,
-                                 coverexclusion=self.getConfig('cover_exclusion_regexp'))
+            return self.story.addImgUrl(storyurl,imgurl,self._fetchUrlRaw,cover=True,
+                                        coverexclusion=self.getConfig('cover_exclusion_regexp'))
+        else:
+            return (None,None)
 
     # bs3 & bs4 are different here.
     # will move to a bs3 vs bs4 block if there's lots of changes.

fanficfare/adapters/base_efiction_adapter.py

@@ -89,7 +89,6 @@ class BaseEfictionAdapter(BaseSiteAdapter):
         """
         return ["Windows-1252", "utf8"]
 
-
     @classmethod
     def getPathToArchive(cls):
         """

fanficfare/cli.py

@@ -144,14 +144,20 @@ def main(argv=None, parser=None, passed_defaultsini=None, passed_personalini=Non
     else:
         urls = args
 
-    for url in urls:
-        try:
-            do_download(url,
-                        options,
-                        passed_defaultsini,
-                        passed_personalini)
-        except Exception, e:
-            print "URL(%s) Failed: Exception (%s). Run URL individually for more detail."%(url,e)
+    if len(urls) > 1:
+        for url in urls:
+            try:
+                do_download(url,
+                            options,
+                            passed_defaultsini,
+                            passed_personalini)
+            except Exception, e:
+                print "URL(%s) Failed: Exception (%s). Run URL individually for more detail."%(url,e)
+    else:
+        do_download(urls[0],
+                    options,
+                    passed_defaultsini,
+                    passed_personalini)
 
 # make rest a function and loop on it.
 def do_download(arg,
@@ -312,7 +318,13 @@ def do_download(arg,
                 # update now handled by pre-populating the old
                 # images and chapters in the adapter rather than
                 # merging epubs.
-                url, chaptercount, adapter.oldchapters, adapter.oldimgs, adapter.oldcover, adapter.calibrebookmark, adapter.logfile = get_update_data(output_filename)
+                (url,
+                 chaptercount,
+                 adapter.oldchapters,
+                 adapter.oldimgs,
+                 adapter.oldcover,
+                 adapter.calibrebookmark,
+                 adapter.logfile) = (get_update_data(output_filename))[0:7]
 
                 print 'Do update - epub(%d) vs url(%d)' % (chaptercount, urlchaptercount)
 

fanficfare/configurable.py

@@ -41,7 +41,38 @@ def re_compile(regex,line):
         return re.compile(regex)
     except Exception, e: 
         raise exceptions.RegularExpresssionFailed(e,regex,line)
-    
+
+# fall back labels.
+titleLabels = {
+    'category':'Category',
+    'genre':'Genre',
+    'language':'Language',
+    'status':'Status',
+    'series':'Series',
+    'characters':'Characters',
+    'ships':'Relationships',
+    'datePublished':'Published',
+    'dateUpdated':'Updated',
+    'dateCreated':'Packaged',
+    'rating':'Rating',
+    'warnings':'Warnings',
+    'numChapters':'Chapters',
+    'numWords':'Words',
+    'site':'Site',
+    'storyId':'Story ID',
+    'authorId':'Author ID',
+    'extratags':'Extra Tags',
+    'title':'Title',
+    'storyUrl':'Story URL',
+    'description':'Summary',
+    'author':'Author',
+    'authorUrl':'Author URL',
+    'formatname':'File Format',
+    'formatext':'File Extension',
+    'siteabbrev':'Site Abbrev',
+    'version':'Downloader Version'
+    }
+
 formatsections = ['html','txt','epub','mobi']
 othersections = ['defaults','overrides']
 
@@ -111,22 +142,18 @@ def get_valid_set_options():
                
                'fix_fimf_blockquotes':(['fimfiction.net'],None,boollist),
                'fail_on_password':(['fimfiction.net'],None,boollist),
-               'do_update_hook':(['fimfiction.net'],None,boollist),
+               'do_update_hook':(['fimfiction.net',
+                                  'archiveofourown.org'],None,boollist),
 
                'force_login':(['phoenixsong.net'],None,boollist),
                'non_breaking_spaces':(['fictionmania.tv'],None,boollist),
                'universe_as_series':(['storiesonline.net'],None,boollist),
                'strip_text_links':(['bloodshedverse.com'],None,boollist),
 
-               # eFiction Base
-               'bulk_load':(['fannation.shades-of-moonlight.com',
-                             'fhsarchive.com',
-                             'lotrfanfiction.com',
-                             'themaplebookshelf.com',
-                             'devianthearts.com',
-                             'thehookupzone.net',
-                             'libraryofmoria.com',
-                             ],None,boollist),
+               # eFiction Base adapters allow bulk_load
+               # kept forgetting to add them, so now it's automatic.
+               'bulk_load':(adapters.get_bulk_load_sites(),
+                            None,boollist),
                
                'include_logpage':(None,['epub'],boollist+['smart']),
                
@@ -136,6 +163,7 @@ def get_valid_set_options():
                'grayscale_images':(None,['epub','html'],boollist),
                'no_image_processing':(None,['epub','html'],boollist),
                }
+
     return dict(valdict)
 
 def get_valid_scalar_entries():
@@ -163,6 +191,7 @@ def get_valid_scalar_entries():
                  'seriesHTML',
                  'langcode',
                  'output_css',
+                 'cover_image',
                  ])
 
 def get_valid_entries():
@@ -307,7 +336,8 @@ class Configuration(ConfigParser.SafeConfigParser):
 
         self.linenos=dict() # key by section or section,key -> lineno
         
-        self.sectionslist = ['defaults']
+        ## [injected] section has even less priority than [defaults]
+        self.sectionslist = ['defaults','injected']
 
         if site.startswith("www."):
             sitewith = site
@@ -609,3 +639,13 @@ class Configurable(object):
 
     def get_config_list(self, sections, key):
         return self.configuration.get_config_list(sections,key)
+
+    def get_label(self, entry):
+        if self.hasConfig(entry+"_label"):
+            label=self.getConfig(entry+"_label")
+        elif entry in titleLabels:
+            label=titleLabels[entry]
+        else:
+            label=entry.title()
+        return label
+

fanficfare/defaults.ini

@@ -597,6 +597,18 @@ hits_label:Hits
 collections_label:Collections
 bookmarks_label:Bookmarks
 
+## AO3 doesn't have anything it calls 'genre'.  The adapter used to be
+## hardcoded to include the site specific metadata freeformtags &
+## ao3categories in the standard metadata field genre.  By making it
+## configurable, users can change it.
+include_in_genre: freeformtags, ao3categories
+
+## AO3 uses the word 'category' differently than most sites.  The
+## adapter used to be hardcoded to include the site specific metadata
+## fandom in the standard metadata field category.  By making it
+## configurable, users can change it.
+include_in_category:fandoms
+
 ## freeformtags was previously typo'ed as freefromtags.  This way,
 ## freefromtags will still work for people who've used it.
 include_in_freefromtags:freeformtags
@@ -608,7 +620,7 @@ include_in_freefromtags:freeformtags
 #extra_subject_tags:fandoms,freeformtags,ao3categories
 
 ## AO3 chapters can include several different types of notes.  We've
-## traditional included them all in the chapter text, but this allows
+## traditionally included them all in the chapter text, but this allows
 ## you to customize which you include.  Copy this parameter to your
 ## personal.ini and list the ones you don't want.
 #exclude_notes:authorheadnotes,chaptersummary,chapterheadnotes,chapterfootnotes,authorfootnotes
@@ -758,7 +770,7 @@ extraships:Spike/Buffy
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
-## Virtually all eFiction-based sites allow downloading the whole story in
+## Virtually all eFiction Base adapters allow downloading the whole story in
 ## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
 ## metadata and chapters can be loaded in one step
 bulk_load:true
@@ -925,7 +937,7 @@ extraships:Harry Potter/Hermione Granger
 #username:YourName
 #password:yourpassword
 
-## Virtually all eFiction-based sites allow downloading the whole story in
+## Virtually all eFiction Base adapters allow downloading the whole story in
 ## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
 ## metadata and chapters can be loaded in one step
 bulk_load:true
@@ -1065,6 +1077,23 @@ extracategories:In Death
 ## cover image.  This lets you exclude them.
 cover_exclusion_regexp:/public/style_emoticons/.*
 
+[it-could-happen.net]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+## RPF == http://en.wikipedia.org/wiki/Real_person_fiction
+## Site dedicated to these categories/characters/ships
+extracategories:Glee RPF
+extracharacters:Darren Criss, Chris Colfer
+extraships:Darren Criss/Chris Colfer
+
+## Virtually all eFiction Base adapters allow downloading the whole story in
+## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
+## metadata and chapters can be loaded in one step
+bulk_load:true
+
 [ksarchive.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:Star Trek
@@ -1084,7 +1113,7 @@ eroticatags_label:Erotica Tags
 extra_titlepage_entries: eroticatags
 
 [lotrfanfiction.com]
-## Virtually all eFiction-based sites allow downloading the whole story in
+## Virtually all eFiction Base adapters allow downloading the whole story in
 ## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
 ## metadata and chapters can be loaded in one step
 bulk_load:true
@@ -1338,7 +1367,7 @@ extracategories:Transgender
 ## confirm they are adult for adult content.
 #is_adult:true
 
-## Virtually all eFiction-based sites allow downloading the whole story in
+## Virtually all eFiction Base adapters allow downloading the whole story in
 ## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
 ## metadata and chapters can be loaded in one step
 bulk_load:true
@@ -1362,7 +1391,7 @@ extraships:Harry Potter/Draco Malfoy
 ## personal.ini, not defaults.ini.
 #is_adult:true
 
-## Virtually all eFiction-based sites allow downloading the whole story in
+## Virtually all eFiction Base adapters allow downloading the whole story in
 ## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
 ## metadata and chapters can be loaded in one step
 bulk_load:true
@@ -1376,7 +1405,7 @@ extracategories:Criminal Minds
 ## personal.ini, not defaults.ini.
 #is_adult:true
 
-## Virtually all eFiction-based sites allow downloading the whole story in
+## Virtually all eFiction Base adapters allow downloading the whole story in
 ## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
 ## metadata and chapters can be loaded in one step
 bulk_load:true
@@ -1449,6 +1478,8 @@ extra_titlepage_entries:readings,awards
 awards_label:Awards
 readings_label:Readings
 
+## some sites include images that we don't ever want becoming the
+## cover image.  This lets you exclude them.
 cover_exclusion_regexp:art/.*Awards.jpg
 
 [voracity2.e-fic.com]
@@ -1639,6 +1670,10 @@ dateUpdated_format:%%Y-%%m-%%d %%H:%%M:%%S
 ## to on, but can be switched off if it is found to cause problems.
 fix_fimf_blockquotes:true
 
+## some sites include images that we don't ever want becoming the
+## cover image.  This lets you exclude them.
+cover_exclusion_regexp:/images/emoticons/
+
 ## Site dedicated to these categories/characters/ships
 extracategories:My Little Pony: Friendship is Magic
 
@@ -1722,7 +1757,7 @@ extraships:InuYasha/Kagome
 ## Site dedicated to these categories/characters/ships
 extracategories:Lord of the Rings
 
-## Virtually all eFiction-based sites allow downloading the whole story in
+## Virtually all eFiction Base adapters allow downloading the whole story in
 ## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
 ## metadata and chapters can be loaded in one step
 bulk_load:true

fanficfare/exceptions.py

@@ -24,6 +24,13 @@ class FailedToDownload(Exception):
     def __str__(self):
         return self.error
 
+class RejectImage(Exception):
+    def __init__(self,error):
+        self.error=error
+
+    def __str__(self):
+        return self.error
+
 class InvalidStoryURL(Exception):
     def __init__(self,url,domain,example):
         self.url=url

fanficfare/geturls.py

@@ -15,12 +15,12 @@
 # limitations under the License.
 #
 
-import re
-import urlparse
-import urllib2 as u2
-
-import imaplib
+import collections
 import email
+import imaplib
+import re
+import urllib2 as u2
+import urlparse
 
 from BeautifulSoup import BeautifulSoup 
 from gziphttp import GZipProcessor
@@ -74,10 +74,8 @@ def get_urls_from_page(url,configuration=None,normalize=False):
     return get_urls_from_html(data,url,configuration,normalize,restrictsearch)
 
 def get_urls_from_html(data,url=None,configuration=None,normalize=False,restrictsearch=None):
+    urls = collections.OrderedDict()
 
-    normalized = [] # normalized url
-    retlist = [] # orig urls.
-    
     if not configuration:
         configuration = Configuration("test1.com","EPUB")
 
@@ -93,7 +91,6 @@ def get_urls_from_html(data,url=None,configuration=None,normalize=False,restrict
             #print("1 urlhref:%s"%href)
             # this (should) catch normal story links, some javascript
             # 'are you old enough' links, and 'Report This' links.
-            # The 'normalized' set prevents duplicates.
             if 'story.php' in a['href']:
                 #print("trying:%s"%a['href'])
                 m = re.search(r"(?P<sid>(view)?story\.php\?(sid|psid|no|story|stid)=\d+)",a['href'])
@@ -106,31 +103,29 @@ def get_urls_from_html(data,url=None,configuration=None,normalize=False,restrict
                 #print("2 urlhref:%s"%href)
                 adapter = adapters.getAdapter(configuration,href)
                 #print("found adapter")
-                if adapter.story.getMetadata('storyUrl') not in normalized:
-                    normalized.append(adapter.story.getMetadata('storyUrl'))
-                    retlist.append(href)
+                if adapter.story.getMetadata('storyUrl') not in urls:
+                    urls[adapter.story.getMetadata('storyUrl')] = [href]
+                else:
+                    urls[adapter.story.getMetadata('storyUrl')].append(href)
             except Exception, e:
                 #print e
                 pass
 
-    if normalize:
-        return normalized
-    else:
-        return retlist
+    # Simply return the longest URL with the assumption that it contains the
+    # most user readable metadata, if not normalized
+    return urls.keys() if normalize else [max(value, key=len) for key, value in urls.items()]
+
 
 def get_urls_from_text(data,configuration=None,normalize=False):
-
-    normalized = [] # normalized url
-    retlist = [] # orig urls.
+    urls = collections.OrderedDict()
     data=unicode(data)
-    
+
     if not configuration:
         configuration = Configuration("test1.com","EPUB")
     
     for href in re.findall('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+', data):
         # this (should) catch normal story links, some javascript
         # 'are you old enough' links, and 'Report This' links.
-        # The 'normalized' set prevents duplicates.
         if 'story.php' in href:
             m = re.search(r"(?P<sid>(view)?story\.php\?(sid|psid|no|story|stid)=\d+)",href)
             if m != None:
@@ -138,16 +133,17 @@ def get_urls_from_text(data,configuration=None,normalize=False):
         try:
             href = href.replace('&index=1','')
             adapter = adapters.getAdapter(configuration,href)
-            if adapter.story.getMetadata('storyUrl') not in normalized:
-                normalized.append(adapter.story.getMetadata('storyUrl'))
-                retlist.append(href)
+            if adapter.story.getMetadata('storyUrl') not in urls:
+                urls[adapter.story.getMetadata('storyUrl')] = [href]
+            else:
+                urls[adapter.story.getMetadata('storyUrl')].append(href)
         except:
             pass
 
-    if normalize:
-        return normalized
-    else:
-        return retlist
+    # Simply return the longest URL with the assumption that it contains the
+    # most user readable metadata, if not normalized
+    return urls.keys() if normalize else [max(value, key=len) for key, value in urls.items()]
+
 
 def form_url(parenturl,url):
      url = url.strip() # ran across an image with a space in the

fanficfare/htmlcleanup.py

@@ -81,10 +81,10 @@ def removeEntities(text):
     
     try:
         t = text.decode('utf-8')
-    except UnicodeEncodeError, e:
+    except (UnicodeEncodeError,UnicodeDecodeError), e:
         try:
             t = text.encode ('ascii', 'xmlcharrefreplace') 
-        except UnicodeEncodeError, e:
+        except (UnicodeEncodeError,UnicodeDecodeError), e:
             t = text
     text = t 
     # replace numeric versions of [&<>] with named versions,

fanficfare/story.py

@@ -18,12 +18,16 @@
 import os, re
 import urlparse
 import string
+import json
+import datetime
 from math import floor
 from functools import partial
 import logging
 logger = logging.getLogger(__name__)
 import urlparse as up
 
+import bs4
+
 import exceptions
 from htmlcleanup import conditionalRemoveEntities, removeAllEntities
 from configurable import Configurable, re_compile
@@ -44,6 +48,8 @@ imagetypes = {
 
 try:
     from calibre.utils.magick import Image
+    from StringIO import StringIO
+    from gif import GifInfo, CHECK_IS_ANIMATED
     convtype = {'jpg':'JPG', 'png':'PNG'}
 
     def convert_image(url,data,sizes,grayscale,
@@ -55,6 +61,10 @@ try:
         owidth, oheight = img.size
         nwidth, nheight = sizes
         scaled, nwidth, nheight = fit_image(owidth, oheight, nwidth, nheight)
+
+        if normalize_format_name(img.format)=="gif" and GifInfo(StringIO(data),CHECK_IS_ANIMATED).frameCount > 1:
+            raise exceptions.RejectImage("Animated gifs come out purely--not going to use it.")
+
         if scaled:
             img.size = (nwidth, nheight)
             export = True
@@ -224,7 +234,7 @@ langs = {
     "Devanagari":"hi",
 
     ## These are from/for AO3:
-    
+
     u'العربية':'ar',
     u'беларуская':'be',
     u'Български език':'bg',
@@ -325,7 +335,7 @@ class InExMatch:
         else:
             retval = self.match == value
             #print(">>>>>>>>>>>>>%s==%s r: %s,%s=%s"%(self.match,value,self.negate,retval, self.negate != retval))
-            
+
         return self.negate != retval
 
     def __str__(self):
@@ -338,7 +348,7 @@ class InExMatch:
         else:
             s='='
         return u'InExMatch(%s %s%s %s)'%(self.keys,f,s,self.match)
-    
+
 ## metakey[,metakey]=~pattern
 ## metakey[,metakey]==string
 ## *for* part lines.  Effect only when trailing conditional key=~regexp matches
@@ -358,7 +368,7 @@ def set_in_ex_clude(setting):
             match = InExMatch(line)
             dest.append([match,condmatch])
     return dest
-              
+
 ## Two or three part lines.  Two part effect everything.
 ## Three part effect only those key(s) lists.
 ## pattern=>replacement
@@ -433,7 +443,7 @@ class Story(Configurable):
 
     def join_list(self, key, vallist):
         return self.getConfig("join_string_"+key,u", ").replace(SPACE_REPLACE,' ').join(map(unicode, vallist))
-                
+
     def setMetadata(self, key, value, condremoveentities=True):
 
         # keep as list type, but set as only value.
@@ -445,20 +455,20 @@ class Story(Configurable):
                 self.metadata[key]=conditionalRemoveEntities(value)
             else:
                 self.metadata[key]=value
-                
+
         if key == "language":
             try:
                 # getMetadata not just self.metadata[] to do replace_metadata.
                 self.setMetadata('langcode',langs[self.getMetadata(key)])
             except:
                 self.setMetadata('langcode','en')
-                
+
         if key == 'dateUpdated' and value:
             # Last Update tags for Bill.
             self.addToList('lastupdate',value.strftime("Last Update Year/Month: %Y/%m"))
             self.addToList('lastupdate',value.strftime("Last Update: %Y/%m/%d"))
 
-        
+
     def do_in_ex_clude(self,which,value,key):
         if value and which in self.in_ex_cludes:
             include = 'include' in which
@@ -487,7 +497,7 @@ class Story(Configurable):
             if include and keyfound and not found:
                 value = None
         return value
-        
+
 
     def doReplacements(self,value,key,return_list=False,seen_list=[]):
         value = self.do_in_ex_clude('include_metadata_pre',value,key)
@@ -526,7 +536,7 @@ class Story(Configurable):
                         # print("replacement,value:%s,%s->%s"%(replacement,value,regexp.sub(replacement,value)))
                         value = regexp.sub(replacement,value)
                         retlist = [value]
-                    
+
         for val in retlist:
             retlist = map(partial(self.do_in_ex_clude,'include_metadata_post',key=key),retlist)
             retlist = map(partial(self.do_in_ex_clude,'exclude_metadata_post',key=key),retlist)
@@ -538,6 +548,69 @@ class Story(Configurable):
         else:
             return self.join_list(key,retlist)
 
+    # for saving an html-ified copy of metadata.
+    def dump_html_metadata(self):
+        lines=[]
+        for k,v in sorted(self.metadata.iteritems()):
+            classes=['metadata']
+            if isinstance(v, (datetime.date, datetime.datetime, datetime.time)):
+                classes.append("datetime")
+                val = v.isoformat()
+            elif isinstance(v,list):
+                classes.append("list")
+                if '' in v:
+                    v.remove('')
+                if None in v:
+                    v.remove(None)
+                #logger.debug("k:%s v:%s"%(k,v))
+                # force ints/floats to strings.
+                val = "<ul>\n<li>%s</li>\n</ul>" % "</li>\n<li>".join([ "%s"%x for x in v ])
+            elif isinstance(v, (int)):
+                classes.append("int")
+                val = v
+            else:
+                val = v
+
+            # don't include items passed in for calibre cols, etc.
+            if not k.startswith('calibre_') and k not in ['output_css']:
+                lines.append("<p><span class='label'>%s</span>: <div class='%s' id='%s'>%s</div><p>\n"%(
+                        self.get_label(k),
+                        " ".join(classes),
+                        k,val))
+        return "\n".join(lines)
+
+    # for loading an html-ified copy of metadata.
+    def load_html_metadata(self,data):
+        soup = bs4.BeautifulSoup(data,'html5lib')
+        for tag in soup.find_all('div','metadata'):
+            val = None
+            if 'datetime' in tag['class']:
+                v = tag.string
+                try:
+                    val = datetime.datetime.strptime(v, '%Y-%m-%dT%H:%M:%S.%f')
+                except ValueError:
+                    try:
+                        val = datetime.datetime.strptime(v, '%Y-%m-%dT%H:%M:%S')
+                    except ValueError:
+                        try:
+                            val = datetime.datetime.strptime(v, '%Y-%m-%d')
+                        except ValueError:
+                            pass
+            elif 'list' in tag['class']:
+                val = []
+                for i in tag.find_all('li'):
+                    val.append(i.string)
+            elif 'int' in tag['class']:
+                val = int(tag.string)
+            else:
+                val = unicode("\n".join([ unicode(c) for c in tag.contents ]))
+                
+            #logger.debug("key(%s)=val(%s)"%(tag['id'],val))
+            if val:
+                self.metadata[tag['id']]=val
+                
+        # self.metadata = json.loads(s, object_hook=datetime_decoder)
+        
     def getMetadataRaw(self,key):
         if self.isValidMetaEntry(key) and self.metadata.has_key(key):
             return self.metadata[key]
@@ -610,7 +683,7 @@ class Story(Configurable):
                                                     self.getMetadata('author', removeallentities, doreplacements)))
 
         self.extendList("extratags",self.getConfigList("extratags"))
-            
+
         if self.getMetadataRaw('seriesUrl'):
             self.setMetadata('seriesHTML',linkhtml%('series',self.getMetadata('seriesUrl', removeallentities, doreplacements),
                                                     self.getMetadata('series', removeallentities, doreplacements)))
@@ -665,9 +738,6 @@ class Story(Configurable):
         if not value in self.metadata[listname]:
             self.metadata[listname].append(value)
 
-        if listname == 'category' and self.getConfig('add_genre_when_multi_category') and len(self.metadata[listname]) > 1:
-            self.addToList('genre',self.getConfig('add_genre_when_multi_category'))
-
     def isList(self,listname):
         'Everything set with an include_in_* is considered a list.'
         return self.isListType(listname) or \
@@ -696,6 +766,8 @@ class Story(Configurable):
                                             doreplacements=doreplacements)]
             else:
                 retlist = self.getMetadataRaw(listname)
+                if retlist is None:
+                    retlist = []
 
         if retlist:
             if doreplacements:
@@ -703,12 +775,15 @@ class Story(Configurable):
                 for val in retlist:
                     newretlist.extend(self.doReplacements(val,listname,return_list=True))
                 retlist = newretlist
-                
+
             if removeallentities:
                 retlist = map(removeAllEntities,retlist)
-                
+
             retlist = filter( lambda x : x!=None and x!='' ,retlist)
 
+        if listname == 'genre' and self.getConfig('add_genre_when_multi_category') and len(self.getList('category')) > 1:
+            retlist.append(self.getConfig('add_genre_when_multi_category'))
+
         # reorder ships so b/a and c/b/a become a/b and a/b/c.  Only on '/',
         # use replace_metadata to change separator first if needed.
         # ships=>[ ]*(/|&amp;|&)[ ]*=>/
@@ -874,6 +949,7 @@ class Story(Configurable):
                 self.imgurls.insert(0,imgurl)
                 newsrc = "images/cover.%s"%ext
                 self.cover=newsrc
+                self.setMetadata('cover_image','specific')
                 self.imgtuples.insert(0,{'newsrc':newsrc,'mime':mime,'data':data})
             else:
                 self.imgurls.append(imgurl)
@@ -888,6 +964,7 @@ class Story(Configurable):
                         not (coverexclusion and re.search(coverexclusion,imgurl)):
                     newsrc = "images/cover.%s"%ext
                     self.cover=newsrc
+                    self.setMetadata('cover_image','first')
                     self.imgtuples.append({'newsrc':newsrc,'mime':mime,'data':data})
                     self.imgurls.append(imgurl)
 
@@ -921,4 +998,3 @@ def commaGroups(s):
         groups.append(s[-3:])
         s = s[:-3]
     return s + ','.join(reversed(groups))
-

fanficfare/writers/base_writer.py

@@ -45,36 +45,6 @@ class BaseStoryWriter(Configurable):
         self.adapter = adapter
         self.story = adapter.getStoryMetadataOnly() # only cache the metadata initially.
         
-        # fall back labels.
-        self.titleLabels = {
-            'category':'Category',
-            'genre':'Genre',
-            'language':'Language',
-            'status':'Status',
-            'series':'Series',
-            'characters':'Characters',
-            'ships':'Relationships',
-            'datePublished':'Published',
-            'dateUpdated':'Updated',
-            'dateCreated':'Packaged',
-            'rating':'Rating',
-            'warnings':'Warnings',
-            'numChapters':'Chapters',
-            'numWords':'Words',
-            'site':'Site',
-            'storyId':'Story ID',
-            'authorId':'Author ID',
-            'extratags':'Extra Tags',
-            'title':'Title',
-            'storyUrl':'Story URL',
-            'description':'Summary',
-            'author':'Author',
-            'authorUrl':'Author URL',
-            'formatname':'File Format',
-            'formatext':'File Extension',
-            'siteabbrev':'Site Abbrev',
-            'version':'Downloader Version'
-            }
         self.story.setMetadata('formatname',self.getFormatName())
         self.story.setMetadata('formatext',self.getFormatExt())
 
@@ -135,15 +105,16 @@ class BaseStoryWriter(Configurable):
                             TEMPLATE=WIDE_ENTRY
                         else:
                             TEMPLATE=ENTRY
-                            
-                        if self.hasConfig(entry+"_label"):
-                            label=self.getConfig(entry+"_label")
-                        elif entry in self.titleLabels:
-                            logger.debug("Using fallback label for %s_label"%entry)
-                            label=self.titleLabels[entry]
-                        else:
-                            label="%s"%entry.title()
-                            logger.debug("No known label for %s, fallback to '%s'"%(entry,label))
+
+                        label=self.get_label(entry)
+                        # if self.hasConfig(entry+"_label"):
+                        #     label=self.getConfig(entry+"_label")
+                        # elif entry in self.titleLabels:
+                        #     logger.debug("Using fallback label for %s_label"%entry)
+                        #     label=self.titleLabels[entry]
+                        # else:
+                        #     label="%s"%entry.title()
+                        #     logger.debug("No known label for %s, fallback to '%s'"%(entry,label))
 
                         # If the label for the title entry is empty, use the
                         # 'no title' option if there is one.

fanficfare/writers/writer_epub.py

@@ -258,14 +258,15 @@ div { margin: 0pt; padding: 0pt; }
             if self.isValidMetaEntry(entry):
                 val = self.story.getMetadata(entry)
                 if val and ( entry not in oldvalues or val != oldvalues[entry] ):
-                    if self.hasConfig(entry+"_label"):
-                        label=self.getConfig(entry+"_label")
-                    elif entry in self.titleLabels:
-                        logger.debug("Using fallback label for %s_label"%entry)
-                        label=self.titleLabels[entry]
-                    else:
-                        label="%s"%entry.title()
-                        logger.debug("No known label for %s, fallback to '%s'"%(entry,label))
+                    label=self.get_label(entry)
+                    # if self.hasConfig(entry+"_label"):
+                    #     label=self.getConfig(entry+"_label")
+                    # elif entry in self.titleLabels:
+                    #     logger.debug("Using fallback label for %s_label"%entry)
+                    #     label=self.titleLabels[entry]
+                    # else:
+                    #     label="%s"%entry.title()
+                    #     logger.debug("No known label for %s, fallback to '%s'"%(entry,label))
 
                     retval = retval + ENTRY.substitute({'id':entry,
                                                         'label':label,

included_dependencies/gif.py

@@ -0,0 +1,283 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+A pure Python GIF metadata extractor.
+Supports adjustable detail to fine-tune performance.
+
+Example code and full epydoc docstrings included.
+
+Uses:
+ - Identifying whether a GIF is static or animated.
+ - Extracting the dimensions, pixel aspect ratio, number of frames, loop count,
+    global palette or palette size, and background color.
+ - Extracting comments and other plaintext.
+ - Testing for various structural errors.
+
+TODO:
+ - Provide basic support for XMP Metadata extraction
+   - http://en.wikipedia.org/wiki/Extensible_Metadata_Platform#Location_in_file_types
+   - http://www.matthewflickinger.com/lab/whatsinagif/bits_and_bytes.asp
+   - Generate test GIF with http://code.google.com/p/python-xmp-toolkit/
+
+Changelog:
+ - 0.2.2: Audited the code and made some corrections.
+ - 0.2.1: 40% speed improvement (went from 15 to 9 seconds for 1000 images)
+ - 0.2.0: Feature-complete
+ - 0.1.0: Initial release
+"""
+
+__appname__ = "gif.py"
+__author__  = "Stephan Sokolow (deitarion/SSokolow)"
+__version__ = "0.2.2"
+__license__ = "PSF License 2.4 or higher (The Python License)"
+
+#{ Check Types (enum, numerical ordering is significant)
+CHECK_IS_GIF_FILE   = 0  #: Just check for a valid GIF header.
+CHECK_IS_ANIMATED   = 1  #: Check whether the file has more than one frame.
+CHECK_COUNT_FRAMES  = 2  #: Count the number of frames in the file.
+CHECK_PARSE_PALETTE = 3  #: Parse the palette and resolve the background color.
+CHECK_READ_COMMENTS = 4  #: Load comments (can sometimes be large) into the L{GifInfo} object.
+CHECK_READ_ALL_TEXT = 5  #: Also load the contents of Plain Text extension blocks.
+
+CHECK_ALL = CHECK_READ_ALL_TEXT #: alias to allow for future modifications
+
+#{  Warning Codes (bitfield)
+WARN_NONE = 0        #: No warnings
+WARN_BAD_IMG = 1     #: Corruption (of the [sub]block size field(s)) or truncation detected in an image block
+WARN_BAD_EXT = 2     #: Corruption (of the [sub]block size field(s)) or truncation detected in an extension block
+WARN_BAD_SIZE = 4    #: An image block specifies dimensions exceeding the global canvas size
+WARN_BAD_BGCOLOR = 8 #: The background color index specified is greater than the palette size
+WARN_EOF  = 16       #: File is missing it's trailer. (Corrupt elsewhere, truncated, or breaking spec by using EOF as the terminator.)
+WARN_TRUNC  = 32     #: File is definitively either truncated or corrupt. (An EOF was encountered part-way through a structure.)
+WARN_LOOP_POS = 64   #: Netscape Application Extension block (animation-control) was present but not first in the file.
+#}
+
+import struct
+
+#{ Structures used by GifInfo
+gifHeaderStruct = struct.Struct('<xxxxxxHHBBB')  #: File header
+gifImageStruct = struct.Struct('<HHHHB')         #: Image block header
+gifExtenStruct = struct.Struct('<BB')            #: Top-level extension block header
+gifNetscapeStruct = struct.Struct('<BBH')        #: NETSCAPE Loop-control sub-block
+gifColorTripleStruct = struct.Struct('<BBB')     #: RGB palette element
+gifPlaintextStruct = struct.Struct('<BHHHHBBBB') #: Plain Text block header
+#}
+
+class BadHeaderException(Exception):
+    """Raised when no valid GIF header is found"""
+
+class GifInfo(object):
+    """A class for loading and storing metadata from GIF files.
+
+    Accepts paths and file-like objects.
+
+    When using L{CHECK_ALL}, this can also be used to walk past a valid
+    GIF file in an un-delimited byte stream in order to identify the point at
+    which the following file starts. (It doesn't C{fh.seek(0)} or C{fh.close()} after
+    use)
+    """
+    warnFlags = WARN_NONE  #: A bit field of C{WARN_*} flags set by L{__init__}
+    checkLevel = CHECK_ALL #: Default C{CHECK_*} level used by L{__init__}
+
+    #{ Pre-defined "unset" values for GIF Metadata
+    path = None        #: The path to the file, if one was passed to L{__init__}
+    version = None     #: C{87a} or C{89a}
+    width = None
+    height = None
+    loopCount = None
+    pixelAspect = None
+    paletteSize = None #: Always calculated if a global palette is present
+    palette = None     #: The palette as a list of integer RGB tuples. Requires L{CHECK_PARSE_PALETTE}.
+    bgColor = None     #: Global background color as an RGB tuple. Requires L{CHECK_PARSE_PALETTE}.
+    comments = None    #: Text in Comment (0xFE) extension blocks as a list of strings. Requires L{CHECK_READ_COMMENTS}
+    otherText = None   #: Text in "Plain Text" (0x01) extension blocks as a list of strings. Requires L{CHECK_READ_ALL_TEXT}
+    frameCount = 0
+    #}
+
+    def __init__(self, fh, checkLevel=checkLevel):
+        """
+        @param fh: A path or file-like object for a GIF file.
+        @param checkLevel: A C{CHECK_*} constant.
+
+        @raises BadHeaderException: The given file lacks a valid GIF header.
+        @raises IOError: The underlying C{open()} system call failed.
+        """
+        self.checkLevel = checkLevel
+        if isinstance(fh, basestring):
+            self.path = fh
+            fh = open(fh, 'rb')
+
+        header = fh.read(gifHeaderStruct.size)
+        if len(header) < gifHeaderStruct.size:
+            raise BadHeaderException("File is too small to be a GIF")
+
+        self.version = header[3:6]
+        if header[0:3] != 'GIF' or self.version not in ['87a', '89a']:
+            raise BadHeaderException("File does not have a recognizable GIF header")
+        elif self.checkLevel <= CHECK_IS_GIF_FILE:
+            return
+
+        self.width, self.height, GCTF_Byte, bgColor, self.pixelAspect = gifHeaderStruct.unpack(header)
+
+        if self.pixelAspect:
+            self.pixelAspect = (self.pixelAspect + 15) / 64.0
+
+        rawPalette = self._getPalette(fh, GCTF_Byte)
+        self.paletteSize = int(len(rawPalette) / 3)
+        if self.checkLevel >= CHECK_PARSE_PALETTE:
+            self.palette = []
+            for pos in range(0, self.paletteSize):
+                self.palette.append(gifColorTripleStruct.unpack_from(rawPalette, pos * 3))
+
+        if self.paletteSize and bgColor > self.paletteSize:
+            self.warnFlags = self.warnFlags | WARN_BAD_BGCOLOR
+        elif self.palette:
+            self.bgColor = self.palette[bgColor]
+
+        # Iterate blocks
+        self.firstBlock = True
+        blocktype = self._read(fh, 1)
+        while not blocktype == chr(0x3B) and not self.warnFlags & WARN_EOF:
+            self._blockHandlers.get(blocktype, lambda x, y:'')(self, fh)
+            if self.checkLevel <= CHECK_IS_ANIMATED and self.frameCount > 1:
+                return
+
+            self.firstBlock = False
+            blocktype = self._read(fh, 1)
+
+        del self.firstBlock
+
+    def _handleImageBlock(self, fh):
+        """"""
+        self.frameCount += 1
+        try:
+            x, y, w, h, LCTF_Byte = gifImageStruct.unpack(self._read(fh, gifImageStruct.size))
+        except:
+            self.warnFlags = self.warnFlags | WARN_EOF | WARN_TRUNC
+            return
+
+        if x + w > self.width or y + h > self.height:
+            self.warnFlags = self.warnFlags | WARN_BAD_SIZE
+
+        self._getPalette(fh, LCTF_Byte) # Skip the local color table if present
+        fh.read(1)                      # Skip the LZW minimum code size.
+
+        # Skip content and test for the block terminator
+        if not self._skipSubBlocks(fh): # For example, if it's a zero-length string like EOF would return.
+            self.warnFlags = self.warnFlags | WARN_BAD_IMG
+
+    def _handleGenericExtensionBlock(self, fh):
+        """@todo: Rewrite this so extension block types have method handlers."""
+        try:
+            extType, blkSize = gifExtenStruct.unpack(self._read(fh, gifExtenStruct.size))
+        except:
+            self.warnFlags = self.warnFlags | WARN_EOF | WARN_TRUNC
+            return
+        startOffset = fh.tell()
+
+        if extType == 0x01 and self.checkLevel >= CHECK_READ_ALL_TEXT: # Plain Text Block
+            self._read(fh, gifPlaintextStruct.size)
+            self.otherText = self.otherText or []
+            blkSize = self._read(fh, 1)
+            while blkSize and blkSize != '\x00':
+                self.otherText.append(self._read(fh, ord(blkSize)))
+                blkSize = self._read(fh, 1)
+        elif extType == 0xFE and self.checkLevel >= CHECK_READ_COMMENTS: # Comment Block
+            self.comments = self.comments or []
+            blkSize = self._read(fh, 1)
+            while blkSize and blkSize != '\x00':
+                self.comments.append(self._read(fh, ord(blkSize)))
+                blkSize = self._read(fh, 1)
+        elif extType == 0xFF: # Application Block
+            if blkSize == 0x0B and self._read(fh, blkSize) == "NETSCAPE2.0":
+                try:
+                    a, b, self.loopCount = gifNetscapeStruct.unpack(self._read(fh, gifNetscapeStruct.size))
+                except:
+                    self.warnFlags = self.warnFlags | WARN_EOF | WARN_TRUNC
+                    return
+
+                if a != 3 and b != 1:
+                    self.warnFlags = self.warnFlags | WARN_BAD_EXT
+
+                if not self.firstBlock:
+                    self.warnFlags = self.warnFlags | WARN_LOOP_POS
+        else:
+            fh.seek( startOffset + blkSize ) # Skip the contents
+
+        # Test for the block terminator
+        if not self._skipSubBlocks(fh):
+            self.warnFlags = self.warnFlags | WARN_BAD_EXT
+
+    def _getPalette(self, handle, bitfield):
+        """Using the size value from C{bitfield},
+        load the palette at C{handle}'s current file pointer position."""
+        if bitfield & int("10000000", 2):
+            nBits = bitfield & int("00000111", 2)
+            tableSize = 3 * 2**( nBits + 1 )
+            return handle.read(tableSize)
+        else:
+            return ''
+
+    def _read(self, handle, size):
+        """Attempt to read the specified number of bytes. Set L{WARN_EOF} if
+        fewer are received."""
+        content = handle.read(size)
+        if len(content) < size:
+            self.warnFlags = self.warnFlags | WARN_EOF
+        return content
+
+    def _skipSubBlocks(self, handle):
+        """Skip sub-blocks beginning at the current file pointer position
+        using fseek."""
+        offset = handle.tell()
+        blkSize = handle.read(1)
+        while blkSize and blkSize != '\x00':
+            offset += ord(blkSize) + 1
+            handle.seek(offset)
+            blkSize = handle.read(1)
+        return blkSize
+
+    _blockHandlers = {
+            chr(0x2C) : _handleImageBlock,
+            chr(0x21) : _handleGenericExtensionBlock,
+        }
+
+def gif_is_animated(path):
+    """A simple convenience function for testing whether a GIF is animated.
+    @rtype: C{bool}
+    """
+    return GifInfo(file(path,'rb'), CHECK_IS_ANIMATED).frameCount > 1
+
+if __name__ == '__main__':
+    from optparse import OptionParser
+    parser = OptionParser(description=__doc__.split('\n\n')[0],
+            version="%%prog v%s" % __version__, usage="%prog <path> ...")
+
+    opts, args = parser.parse_args()
+
+    if args:
+        for fpath in args:
+            try:
+                info = GifInfo(fpath, CHECK_COUNT_FRAMES)
+                warnFlags = (
+                        (info.warnFlags & WARN_BAD_IMG     and 'I' or ' ') +
+                        (info.warnFlags & WARN_BAD_EXT     and 'X' or ' ') +
+                        (info.warnFlags & WARN_BAD_SIZE    and 'C' or ' ') +
+                        (info.warnFlags & WARN_BAD_BGCOLOR and 'B' or ' ') +
+                        (info.warnFlags & WARN_EOF         and 'E' or ' ') +
+                        (info.warnFlags & WARN_TRUNC       and 'T' or ' ') +
+                        (info.warnFlags & WARN_LOOP_POS    and 'L' or ' ')
+                    )
+                print "[%s](%3s Frames): %s" % (warnFlags, info.frameCount, info.path)
+            except BadHeaderException, err:
+                print "%s: %s" % (str(err), fpath)
+        print "\nWarning Flags:"
+        print " I = Image Chunk Corruption/Truncation"
+        print " X = Extension Chunk Corruption/Truncation"
+        print " C = Image Chunk Dimensions Exceed Global Canvas"
+        print " B = Bad Background Color (Index Exceeds Palette Size)"
+        print " E = Unexpected EOF Encountered (Missing Image Terminator)"
+        print " T = EOF Encountered Within A Block Header (Corrupt or Truncated File)"
+        print " L = Loop-control block misplaced within the file"
+        print
+        print "Note: A nearly-threefold speed-up can be had by using CHECK_IS_ANIMATED rather than CHECK_COUNT_FRAMES"

included_dependencies/html5lib/__init__.py

@@ -20,4 +20,4 @@ from .serializer import serialize
 
 __all__ = ["HTMLParser", "parse", "parseFragment", "getTreeBuilder",
            "getTreeWalker", "serialize"]
-__version__ = "0.999"
+__version__ = "0.99999"

included_dependencies/html5lib/constants.py

@@ -1,292 +1,290 @@
 from __future__ import absolute_import, division, unicode_literals
 
 import string
-import gettext
-_ = gettext.gettext
 
 EOF = None
 
 E = {
     "null-character":
-        _("Null character in input stream, replaced with U+FFFD."),
+        "Null character in input stream, replaced with U+FFFD.",
     "invalid-codepoint":
-        _("Invalid codepoint in stream."),
+        "Invalid codepoint in stream.",
     "incorrectly-placed-solidus":
-        _("Solidus (/) incorrectly placed in tag."),
+        "Solidus (/) incorrectly placed in tag.",
     "incorrect-cr-newline-entity":
-        _("Incorrect CR newline entity, replaced with LF."),
+        "Incorrect CR newline entity, replaced with LF.",
     "illegal-windows-1252-entity":
-        _("Entity used with illegal number (windows-1252 reference)."),
+        "Entity used with illegal number (windows-1252 reference).",
     "cant-convert-numeric-entity":
-        _("Numeric entity couldn't be converted to character "
-          "(codepoint U+%(charAsInt)08x)."),
+        "Numeric entity couldn't be converted to character "
+        "(codepoint U+%(charAsInt)08x).",
     "illegal-codepoint-for-numeric-entity":
-        _("Numeric entity represents an illegal codepoint: "
-          "U+%(charAsInt)08x."),
+        "Numeric entity represents an illegal codepoint: "
+        "U+%(charAsInt)08x.",
     "numeric-entity-without-semicolon":
-        _("Numeric entity didn't end with ';'."),
+        "Numeric entity didn't end with ';'.",
     "expected-numeric-entity-but-got-eof":
-        _("Numeric entity expected. Got end of file instead."),
+        "Numeric entity expected. Got end of file instead.",
     "expected-numeric-entity":
-        _("Numeric entity expected but none found."),
+        "Numeric entity expected but none found.",
     "named-entity-without-semicolon":
-        _("Named entity didn't end with ';'."),
+        "Named entity didn't end with ';'.",
     "expected-named-entity":
-        _("Named entity expected. Got none."),
+        "Named entity expected. Got none.",
     "attributes-in-end-tag":
-        _("End tag contains unexpected attributes."),
+        "End tag contains unexpected attributes.",
     'self-closing-flag-on-end-tag':
-        _("End tag contains unexpected self-closing flag."),
+        "End tag contains unexpected self-closing flag.",
     "expected-tag-name-but-got-right-bracket":
-        _("Expected tag name. Got '>' instead."),
+        "Expected tag name. Got '>' instead.",
     "expected-tag-name-but-got-question-mark":
-        _("Expected tag name. Got '?' instead. (HTML doesn't "
-          "support processing instructions.)"),
+        "Expected tag name. Got '?' instead. (HTML doesn't "
+        "support processing instructions.)",
     "expected-tag-name":
-        _("Expected tag name. Got something else instead"),
+        "Expected tag name. Got something else instead",
     "expected-closing-tag-but-got-right-bracket":
-        _("Expected closing tag. Got '>' instead. Ignoring '</>'."),
+        "Expected closing tag. Got '>' instead. Ignoring '</>'.",
     "expected-closing-tag-but-got-eof":
-        _("Expected closing tag. Unexpected end of file."),
+        "Expected closing tag. Unexpected end of file.",
     "expected-closing-tag-but-got-char":
-        _("Expected closing tag. Unexpected character '%(data)s' found."),
+        "Expected closing tag. Unexpected character '%(data)s' found.",
     "eof-in-tag-name":
-        _("Unexpected end of file in the tag name."),
+        "Unexpected end of file in the tag name.",
     "expected-attribute-name-but-got-eof":
-        _("Unexpected end of file. Expected attribute name instead."),
+        "Unexpected end of file. Expected attribute name instead.",
     "eof-in-attribute-name":
-        _("Unexpected end of file in attribute name."),
+        "Unexpected end of file in attribute name.",
     "invalid-character-in-attribute-name":
-        _("Invalid character in attribute name"),
+        "Invalid character in attribute name",
     "duplicate-attribute":
-        _("Dropped duplicate attribute on tag."),
+        "Dropped duplicate attribute on tag.",
     "expected-end-of-tag-name-but-got-eof":
-        _("Unexpected end of file. Expected = or end of tag."),
+        "Unexpected end of file. Expected = or end of tag.",
     "expected-attribute-value-but-got-eof":
-        _("Unexpected end of file. Expected attribute value."),
+        "Unexpected end of file. Expected attribute value.",
     "expected-attribute-value-but-got-right-bracket":
-        _("Expected attribute value. Got '>' instead."),
+        "Expected attribute value. Got '>' instead.",
     'equals-in-unquoted-attribute-value':
-        _("Unexpected = in unquoted attribute"),
+        "Unexpected = in unquoted attribute",
     'unexpected-character-in-unquoted-attribute-value':
-        _("Unexpected character in unquoted attribute"),
+        "Unexpected character in unquoted attribute",
     "invalid-character-after-attribute-name":
-        _("Unexpected character after attribute name."),
+        "Unexpected character after attribute name.",
     "unexpected-character-after-attribute-value":
-        _("Unexpected character after attribute value."),
+        "Unexpected character after attribute value.",
     "eof-in-attribute-value-double-quote":
-        _("Unexpected end of file in attribute value (\")."),
+        "Unexpected end of file in attribute value (\").",
     "eof-in-attribute-value-single-quote":
-        _("Unexpected end of file in attribute value (')."),
+        "Unexpected end of file in attribute value (').",
     "eof-in-attribute-value-no-quotes":
-        _("Unexpected end of file in attribute value."),
+        "Unexpected end of file in attribute value.",
     "unexpected-EOF-after-solidus-in-tag":
-        _("Unexpected end of file in tag. Expected >"),
+        "Unexpected end of file in tag. Expected >",
     "unexpected-character-after-solidus-in-tag":
-        _("Unexpected character after / in tag. Expected >"),
+        "Unexpected character after / in tag. Expected >",
     "expected-dashes-or-doctype":
-        _("Expected '--' or 'DOCTYPE'. Not found."),
+        "Expected '--' or 'DOCTYPE'. Not found.",
     "unexpected-bang-after-double-dash-in-comment":
-        _("Unexpected ! after -- in comment"),
+        "Unexpected ! after -- in comment",
     "unexpected-space-after-double-dash-in-comment":
-        _("Unexpected space after -- in comment"),
+        "Unexpected space after -- in comment",
     "incorrect-comment":
-        _("Incorrect comment."),
+        "Incorrect comment.",
     "eof-in-comment":
-        _("Unexpected end of file in comment."),
+        "Unexpected end of file in comment.",
     "eof-in-comment-end-dash":
-        _("Unexpected end of file in comment (-)"),
+        "Unexpected end of file in comment (-)",
     "unexpected-dash-after-double-dash-in-comment":
-        _("Unexpected '-' after '--' found in comment."),
+        "Unexpected '-' after '--' found in comment.",
     "eof-in-comment-double-dash":
-        _("Unexpected end of file in comment (--)."),
+        "Unexpected end of file in comment (--).",
     "eof-in-comment-end-space-state":
-        _("Unexpected end of file in comment."),
+        "Unexpected end of file in comment.",
     "eof-in-comment-end-bang-state":
-        _("Unexpected end of file in comment."),
+        "Unexpected end of file in comment.",
     "unexpected-char-in-comment":
-        _("Unexpected character in comment found."),
+        "Unexpected character in comment found.",
     "need-space-after-doctype":
-        _("No space after literal string 'DOCTYPE'."),
+        "No space after literal string 'DOCTYPE'.",
     "expected-doctype-name-but-got-right-bracket":
-        _("Unexpected > character. Expected DOCTYPE name."),
+        "Unexpected > character. Expected DOCTYPE name.",
     "expected-doctype-name-but-got-eof":
-        _("Unexpected end of file. Expected DOCTYPE name."),
+        "Unexpected end of file. Expected DOCTYPE name.",
     "eof-in-doctype-name":
-        _("Unexpected end of file in DOCTYPE name."),
+        "Unexpected end of file in DOCTYPE name.",
     "eof-in-doctype":
-        _("Unexpected end of file in DOCTYPE."),
+        "Unexpected end of file in DOCTYPE.",
     "expected-space-or-right-bracket-in-doctype":
-        _("Expected space or '>'. Got '%(data)s'"),
+        "Expected space or '>'. Got '%(data)s'",
     "unexpected-end-of-doctype":
-        _("Unexpected end of DOCTYPE."),
+        "Unexpected end of DOCTYPE.",
     "unexpected-char-in-doctype":
-        _("Unexpected character in DOCTYPE."),
+        "Unexpected character in DOCTYPE.",
     "eof-in-innerhtml":
-        _("XXX innerHTML EOF"),
+        "XXX innerHTML EOF",
     "unexpected-doctype":
-        _("Unexpected DOCTYPE. Ignored."),
+        "Unexpected DOCTYPE. Ignored.",
     "non-html-root":
-        _("html needs to be the first start tag."),
+        "html needs to be the first start tag.",
     "expected-doctype-but-got-eof":
-        _("Unexpected End of file. Expected DOCTYPE."),
+        "Unexpected End of file. Expected DOCTYPE.",
     "unknown-doctype":
-        _("Erroneous DOCTYPE."),
+        "Erroneous DOCTYPE.",
     "expected-doctype-but-got-chars":
-        _("Unexpected non-space characters. Expected DOCTYPE."),
+        "Unexpected non-space characters. Expected DOCTYPE.",
     "expected-doctype-but-got-start-tag":
-        _("Unexpected start tag (%(name)s). Expected DOCTYPE."),
+        "Unexpected start tag (%(name)s). Expected DOCTYPE.",
     "expected-doctype-but-got-end-tag":
-        _("Unexpected end tag (%(name)s). Expected DOCTYPE."),
+        "Unexpected end tag (%(name)s). Expected DOCTYPE.",
     "end-tag-after-implied-root":
-        _("Unexpected end tag (%(name)s) after the (implied) root element."),
+        "Unexpected end tag (%(name)s) after the (implied) root element.",
     "expected-named-closing-tag-but-got-eof":
-        _("Unexpected end of file. Expected end tag (%(name)s)."),
+        "Unexpected end of file. Expected end tag (%(name)s).",
     "two-heads-are-not-better-than-one":
-        _("Unexpected start tag head in existing head. Ignored."),
+        "Unexpected start tag head in existing head. Ignored.",
     "unexpected-end-tag":
-        _("Unexpected end tag (%(name)s). Ignored."),
+        "Unexpected end tag (%(name)s). Ignored.",
     "unexpected-start-tag-out-of-my-head":
-        _("Unexpected start tag (%(name)s) that can be in head. Moved."),
+        "Unexpected start tag (%(name)s) that can be in head. Moved.",
     "unexpected-start-tag":
-        _("Unexpected start tag (%(name)s)."),
+        "Unexpected start tag (%(name)s).",
     "missing-end-tag":
-        _("Missing end tag (%(name)s)."),
+        "Missing end tag (%(name)s).",
     "missing-end-tags":
-        _("Missing end tags (%(name)s)."),
+        "Missing end tags (%(name)s).",
     "unexpected-start-tag-implies-end-tag":
-        _("Unexpected start tag (%(startName)s) "
-          "implies end tag (%(endName)s)."),
+        "Unexpected start tag (%(startName)s) "
+        "implies end tag (%(endName)s).",
     "unexpected-start-tag-treated-as":
-        _("Unexpected start tag (%(originalName)s). Treated as %(newName)s."),
+        "Unexpected start tag (%(originalName)s). Treated as %(newName)s.",
     "deprecated-tag":
-        _("Unexpected start tag %(name)s. Don't use it!"),
+        "Unexpected start tag %(name)s. Don't use it!",
     "unexpected-start-tag-ignored":
-        _("Unexpected start tag %(name)s. Ignored."),
+        "Unexpected start tag %(name)s. Ignored.",
     "expected-one-end-tag-but-got-another":
-        _("Unexpected end tag (%(gotName)s). "
-          "Missing end tag (%(expectedName)s)."),
+        "Unexpected end tag (%(gotName)s). "
+        "Missing end tag (%(expectedName)s).",
     "end-tag-too-early":
-        _("End tag (%(name)s) seen too early. Expected other end tag."),
+        "End tag (%(name)s) seen too early. Expected other end tag.",
     "end-tag-too-early-named":
-        _("Unexpected end tag (%(gotName)s). Expected end tag (%(expectedName)s)."),
+        "Unexpected end tag (%(gotName)s). Expected end tag (%(expectedName)s).",
     "end-tag-too-early-ignored":
-        _("End tag (%(name)s) seen too early. Ignored."),
+        "End tag (%(name)s) seen too early. Ignored.",
     "adoption-agency-1.1":
-        _("End tag (%(name)s) violates step 1, "
-          "paragraph 1 of the adoption agency algorithm."),
+        "End tag (%(name)s) violates step 1, "
+        "paragraph 1 of the adoption agency algorithm.",
     "adoption-agency-1.2":
-        _("End tag (%(name)s) violates step 1, "
-          "paragraph 2 of the adoption agency algorithm."),
+        "End tag (%(name)s) violates step 1, "
+        "paragraph 2 of the adoption agency algorithm.",
     "adoption-agency-1.3":
-        _("End tag (%(name)s) violates step 1, "
-          "paragraph 3 of the adoption agency algorithm."),
+        "End tag (%(name)s) violates step 1, "
+        "paragraph 3 of the adoption agency algorithm.",
     "adoption-agency-4.4":
-        _("End tag (%(name)s) violates step 4, "
-          "paragraph 4 of the adoption agency algorithm."),
+        "End tag (%(name)s) violates step 4, "
+        "paragraph 4 of the adoption agency algorithm.",
     "unexpected-end-tag-treated-as":
-        _("Unexpected end tag (%(originalName)s). Treated as %(newName)s."),
+        "Unexpected end tag (%(originalName)s). Treated as %(newName)s.",
     "no-end-tag":
-        _("This element (%(name)s) has no end tag."),
+        "This element (%(name)s) has no end tag.",
     "unexpected-implied-end-tag-in-table":
-        _("Unexpected implied end tag (%(name)s) in the table phase."),
+        "Unexpected implied end tag (%(name)s) in the table phase.",
     "unexpected-implied-end-tag-in-table-body":
-        _("Unexpected implied end tag (%(name)s) in the table body phase."),
+        "Unexpected implied end tag (%(name)s) in the table body phase.",
     "unexpected-char-implies-table-voodoo":
-        _("Unexpected non-space characters in "
-          "table context caused voodoo mode."),
+        "Unexpected non-space characters in "
+        "table context caused voodoo mode.",
     "unexpected-hidden-input-in-table":
-        _("Unexpected input with type hidden in table context."),
+        "Unexpected input with type hidden in table context.",
     "unexpected-form-in-table":
-        _("Unexpected form in table context."),
+        "Unexpected form in table context.",
     "unexpected-start-tag-implies-table-voodoo":
-        _("Unexpected start tag (%(name)s) in "
-          "table context caused voodoo mode."),
+        "Unexpected start tag (%(name)s) in "
+        "table context caused voodoo mode.",
     "unexpected-end-tag-implies-table-voodoo":
-        _("Unexpected end tag (%(name)s) in "
-          "table context caused voodoo mode."),
+        "Unexpected end tag (%(name)s) in "
+        "table context caused voodoo mode.",
     "unexpected-cell-in-table-body":
-        _("Unexpected table cell start tag (%(name)s) "
-          "in the table body phase."),
+        "Unexpected table cell start tag (%(name)s) "
+        "in the table body phase.",
     "unexpected-cell-end-tag":
-        _("Got table cell end tag (%(name)s) "
-          "while required end tags are missing."),
+        "Got table cell end tag (%(name)s) "
+        "while required end tags are missing.",
     "unexpected-end-tag-in-table-body":
-        _("Unexpected end tag (%(name)s) in the table body phase. Ignored."),
+        "Unexpected end tag (%(name)s) in the table body phase. Ignored.",
     "unexpected-implied-end-tag-in-table-row":
-        _("Unexpected implied end tag (%(name)s) in the table row phase."),
+        "Unexpected implied end tag (%(name)s) in the table row phase.",
     "unexpected-end-tag-in-table-row":
-        _("Unexpected end tag (%(name)s) in the table row phase. Ignored."),
+        "Unexpected end tag (%(name)s) in the table row phase. Ignored.",
     "unexpected-select-in-select":
-        _("Unexpected select start tag in the select phase "
-          "treated as select end tag."),
+        "Unexpected select start tag in the select phase "
+        "treated as select end tag.",
     "unexpected-input-in-select":
-        _("Unexpected input start tag in the select phase."),
+        "Unexpected input start tag in the select phase.",
     "unexpected-start-tag-in-select":
-        _("Unexpected start tag token (%(name)s in the select phase. "
-          "Ignored."),
+        "Unexpected start tag token (%(name)s in the select phase. "
+        "Ignored.",
     "unexpected-end-tag-in-select":
-        _("Unexpected end tag (%(name)s) in the select phase. Ignored."),
+        "Unexpected end tag (%(name)s) in the select phase. Ignored.",
     "unexpected-table-element-start-tag-in-select-in-table":
-        _("Unexpected table element start tag (%(name)s) in the select in table phase."),
+        "Unexpected table element start tag (%(name)s) in the select in table phase.",
     "unexpected-table-element-end-tag-in-select-in-table":
-        _("Unexpected table element end tag (%(name)s) in the select in table phase."),
+        "Unexpected table element end tag (%(name)s) in the select in table phase.",
     "unexpected-char-after-body":
-        _("Unexpected non-space characters in the after body phase."),
+        "Unexpected non-space characters in the after body phase.",
     "unexpected-start-tag-after-body":
-        _("Unexpected start tag token (%(name)s)"
-          " in the after body phase."),
+        "Unexpected start tag token (%(name)s)"
+        " in the after body phase.",
     "unexpected-end-tag-after-body":
-        _("Unexpected end tag token (%(name)s)"
-          " in the after body phase."),
+        "Unexpected end tag token (%(name)s)"
+        " in the after body phase.",
     "unexpected-char-in-frameset":
-        _("Unexpected characters in the frameset phase. Characters ignored."),
+        "Unexpected characters in the frameset phase. Characters ignored.",
     "unexpected-start-tag-in-frameset":
-        _("Unexpected start tag token (%(name)s)"
-          " in the frameset phase. Ignored."),
+        "Unexpected start tag token (%(name)s)"
+        " in the frameset phase. Ignored.",
     "unexpected-frameset-in-frameset-innerhtml":
-        _("Unexpected end tag token (frameset) "
-          "in the frameset phase (innerHTML)."),
+        "Unexpected end tag token (frameset) "
+        "in the frameset phase (innerHTML).",
     "unexpected-end-tag-in-frameset":
-        _("Unexpected end tag token (%(name)s)"
-          " in the frameset phase. Ignored."),
+        "Unexpected end tag token (%(name)s)"
+        " in the frameset phase. Ignored.",
     "unexpected-char-after-frameset":
-        _("Unexpected non-space characters in the "
-          "after frameset phase. Ignored."),
+        "Unexpected non-space characters in the "
+        "after frameset phase. Ignored.",
     "unexpected-start-tag-after-frameset":
-        _("Unexpected start tag (%(name)s)"
-          " in the after frameset phase. Ignored."),
+        "Unexpected start tag (%(name)s)"
+        " in the after frameset phase. Ignored.",
     "unexpected-end-tag-after-frameset":
-        _("Unexpected end tag (%(name)s)"
-          " in the after frameset phase. Ignored."),
+        "Unexpected end tag (%(name)s)"
+        " in the after frameset phase. Ignored.",
     "unexpected-end-tag-after-body-innerhtml":
-        _("Unexpected end tag after body(innerHtml)"),
+        "Unexpected end tag after body(innerHtml)",
     "expected-eof-but-got-char":
-        _("Unexpected non-space characters. Expected end of file."),
+        "Unexpected non-space characters. Expected end of file.",
     "expected-eof-but-got-start-tag":
-        _("Unexpected start tag (%(name)s)"
-          ". Expected end of file."),
+        "Unexpected start tag (%(name)s)"
+        ". Expected end of file.",
     "expected-eof-but-got-end-tag":
-        _("Unexpected end tag (%(name)s)"
-          ". Expected end of file."),
+        "Unexpected end tag (%(name)s)"
+        ". Expected end of file.",
     "eof-in-table":
-        _("Unexpected end of file. Expected table content."),
+        "Unexpected end of file. Expected table content.",
     "eof-in-select":
-        _("Unexpected end of file. Expected select content."),
+        "Unexpected end of file. Expected select content.",
     "eof-in-frameset":
-        _("Unexpected end of file. Expected frameset content."),
+        "Unexpected end of file. Expected frameset content.",
     "eof-in-script-in-script":
-        _("Unexpected end of file. Expected script content."),
+        "Unexpected end of file. Expected script content.",
     "eof-in-foreign-lands":
-        _("Unexpected end of file. Expected foreign content"),
+        "Unexpected end of file. Expected foreign content",
     "non-void-element-with-trailing-solidus":
-        _("Trailing solidus not allowed on element %(name)s"),
+        "Trailing solidus not allowed on element %(name)s",
     "unexpected-html-element-in-foreign-content":
-        _("Element %(name)s not allowed in a non-html context"),
+        "Element %(name)s not allowed in a non-html context",
     "unexpected-end-tag-before-html":
-        _("Unexpected end tag (%(name)s) before html."),
+        "Unexpected end tag (%(name)s) before html.",
     "XXX-undefined-error":
-        _("Undefined error (this sucks and should be fixed)"),
+        "Undefined error (this sucks and should be fixed)",
 }
 
 namespaces = {
@@ -298,7 +296,7 @@ namespaces = {
     "xmlns": "http://www.w3.org/2000/xmlns/"
 }
 
-scopingElements = frozenset((
+scopingElements = frozenset([
     (namespaces["html"], "applet"),
     (namespaces["html"], "caption"),
     (namespaces["html"], "html"),
@@ -316,9 +314,9 @@ scopingElements = frozenset((
     (namespaces["svg"], "foreignObject"),
     (namespaces["svg"], "desc"),
     (namespaces["svg"], "title"),
-))
+])
 
-formattingElements = frozenset((
+formattingElements = frozenset([
     (namespaces["html"], "a"),
     (namespaces["html"], "b"),
     (namespaces["html"], "big"),
@@ -333,9 +331,9 @@ formattingElements = frozenset((
     (namespaces["html"], "strong"),
     (namespaces["html"], "tt"),
     (namespaces["html"], "u")
-))
+])
 
-specialElements = frozenset((
+specialElements = frozenset([
     (namespaces["html"], "address"),
     (namespaces["html"], "applet"),
     (namespaces["html"], "area"),
@@ -416,22 +414,22 @@ specialElements = frozenset((
     (namespaces["html"], "wbr"),
     (namespaces["html"], "xmp"),
     (namespaces["svg"], "foreignObject")
-))
+])
 
-htmlIntegrationPointElements = frozenset((
+htmlIntegrationPointElements = frozenset([
     (namespaces["mathml"], "annotaion-xml"),
     (namespaces["svg"], "foreignObject"),
     (namespaces["svg"], "desc"),
     (namespaces["svg"], "title")
-))
+])
 
-mathmlTextIntegrationPointElements = frozenset((
+mathmlTextIntegrationPointElements = frozenset([
     (namespaces["mathml"], "mi"),
     (namespaces["mathml"], "mo"),
     (namespaces["mathml"], "mn"),
     (namespaces["mathml"], "ms"),
     (namespaces["mathml"], "mtext")
-))
+])
 
 adjustForeignAttributes = {
     "xlink:actuate": ("xlink", "actuate", namespaces["xlink"]),
@@ -451,21 +449,21 @@ adjustForeignAttributes = {
 unadjustForeignAttributes = dict([((ns, local), qname) for qname, (prefix, local, ns) in
                                   adjustForeignAttributes.items()])
 
-spaceCharacters = frozenset((
+spaceCharacters = frozenset([
     "\t",
     "\n",
     "\u000C",
     " ",
     "\r"
-))
+])
 
-tableInsertModeElements = frozenset((
+tableInsertModeElements = frozenset([
     "table",
     "tbody",
     "tfoot",
     "thead",
     "tr"
-))
+])
 
 asciiLowercase = frozenset(string.ascii_lowercase)
 asciiUppercase = frozenset(string.ascii_uppercase)
@@ -486,7 +484,7 @@ headingElements = (
     "h6"
 )
 
-voidElements = frozenset((
+voidElements = frozenset([
     "base",
     "command",
     "event-source",
@@ -502,11 +500,11 @@ voidElements = frozenset((
     "input",
     "source",
     "track"
-))
+])
 
-cdataElements = frozenset(('title', 'textarea'))
+cdataElements = frozenset(['title', 'textarea'])
 
-rcdataElements = frozenset((
+rcdataElements = frozenset([
     'style',
     'script',
     'xmp',
@@ -514,27 +512,27 @@ rcdataElements = frozenset((
     'noembed',
     'noframes',
     'noscript'
-))
+])
 
 booleanAttributes = {
-    "": frozenset(("irrelevant",)),
-    "style": frozenset(("scoped",)),
-    "img": frozenset(("ismap",)),
-    "audio": frozenset(("autoplay", "controls")),
-    "video": frozenset(("autoplay", "controls")),
-    "script": frozenset(("defer", "async")),
-    "details": frozenset(("open",)),
-    "datagrid": frozenset(("multiple", "disabled")),
-    "command": frozenset(("hidden", "disabled", "checked", "default")),
-    "hr": frozenset(("noshade")),
-    "menu": frozenset(("autosubmit",)),
-    "fieldset": frozenset(("disabled", "readonly")),
-    "option": frozenset(("disabled", "readonly", "selected")),
-    "optgroup": frozenset(("disabled", "readonly")),
-    "button": frozenset(("disabled", "autofocus")),
-    "input": frozenset(("disabled", "readonly", "required", "autofocus", "checked", "ismap")),
-    "select": frozenset(("disabled", "readonly", "autofocus", "multiple")),
-    "output": frozenset(("disabled", "readonly")),
+    "": frozenset(["irrelevant"]),
+    "style": frozenset(["scoped"]),
+    "img": frozenset(["ismap"]),
+    "audio": frozenset(["autoplay", "controls"]),
+    "video": frozenset(["autoplay", "controls"]),
+    "script": frozenset(["defer", "async"]),
+    "details": frozenset(["open"]),
+    "datagrid": frozenset(["multiple", "disabled"]),
+    "command": frozenset(["hidden", "disabled", "checked", "default"]),
+    "hr": frozenset(["noshade"]),
+    "menu": frozenset(["autosubmit"]),
+    "fieldset": frozenset(["disabled", "readonly"]),
+    "option": frozenset(["disabled", "readonly", "selected"]),
+    "optgroup": frozenset(["disabled", "readonly"]),
+    "button": frozenset(["disabled", "autofocus"]),
+    "input": frozenset(["disabled", "readonly", "required", "autofocus", "checked", "ismap"]),
+    "select": frozenset(["disabled", "readonly", "autofocus", "multiple"]),
+    "output": frozenset(["disabled", "readonly"]),
 }
 
 # entitiesWindows1252 has to be _ordered_ and needs to have an index. It
@@ -574,7 +572,7 @@ entitiesWindows1252 = (
     376     # 0x9F  0x0178  LATIN CAPITAL LETTER Y WITH DIAERESIS
 )
 
-xmlEntities = frozenset(('lt;', 'gt;', 'amp;', 'apos;', 'quot;'))
+xmlEntities = frozenset(['lt;', 'gt;', 'amp;', 'apos;', 'quot;'])
 
 entities = {
     "AElig": "\xc6",
@@ -3088,8 +3086,8 @@ tokenTypes = {
     "ParseError": 7
 }
 
-tagTokenTypes = frozenset((tokenTypes["StartTag"], tokenTypes["EndTag"],
-                           tokenTypes["EmptyTag"]))
+tagTokenTypes = frozenset([tokenTypes["StartTag"], tokenTypes["EndTag"],
+                           tokenTypes["EmptyTag"]])
 
 
 prefixes = dict([(v, k) for k, v in namespaces.items()])

included_dependencies/html5lib/filters/lint.py

@@ -1,8 +1,5 @@
 from __future__ import absolute_import, division, unicode_literals
 
-from gettext import gettext
-_ = gettext
-
 from . import _base
 from ..constants import cdataElements, rcdataElements, voidElements
 
@@ -23,24 +20,24 @@ class Filter(_base.Filter):
             if type in ("StartTag", "EmptyTag"):
                 name = token["name"]
                 if contentModelFlag != "PCDATA":
-                    raise LintError(_("StartTag not in PCDATA content model flag: %(tag)s") % {"tag": name})
+                    raise LintError("StartTag not in PCDATA content model flag: %(tag)s" % {"tag": name})
                 if not isinstance(name, str):
-                    raise LintError(_("Tag name is not a string: %(tag)r") % {"tag": name})
+                    raise LintError("Tag name is not a string: %(tag)r" % {"tag": name})
                 if not name:
-                    raise LintError(_("Empty tag name"))
+                    raise LintError("Empty tag name")
                 if type == "StartTag" and name in voidElements:
-                    raise LintError(_("Void element reported as StartTag token: %(tag)s") % {"tag": name})
+                    raise LintError("Void element reported as StartTag token: %(tag)s" % {"tag": name})
                 elif type == "EmptyTag" and name not in voidElements:
-                    raise LintError(_("Non-void element reported as EmptyTag token: %(tag)s") % {"tag": token["name"]})
+                    raise LintError("Non-void element reported as EmptyTag token: %(tag)s" % {"tag": token["name"]})
                 if type == "StartTag":
                     open_elements.append(name)
                 for name, value in token["data"]:
                     if not isinstance(name, str):
-                        raise LintError(_("Attribute name is not a string: %(name)r") % {"name": name})
+                        raise LintError("Attribute name is not a string: %(name)r" % {"name": name})
                     if not name:
-                        raise LintError(_("Empty attribute name"))
+                        raise LintError("Empty attribute name")
                     if not isinstance(value, str):
-                        raise LintError(_("Attribute value is not a string: %(value)r") % {"value": value})
+                        raise LintError("Attribute value is not a string: %(value)r" % {"value": value})
                 if name in cdataElements:
                     contentModelFlag = "CDATA"
                 elif name in rcdataElements:
@@ -51,43 +48,43 @@ class Filter(_base.Filter):
             elif type == "EndTag":
                 name = token["name"]
                 if not isinstance(name, str):
-                    raise LintError(_("Tag name is not a string: %(tag)r") % {"tag": name})
+                    raise LintError("Tag name is not a string: %(tag)r" % {"tag": name})
                 if not name:
-                    raise LintError(_("Empty tag name"))
+                    raise LintError("Empty tag name")
                 if name in voidElements:
-                    raise LintError(_("Void element reported as EndTag token: %(tag)s") % {"tag": name})
+                    raise LintError("Void element reported as EndTag token: %(tag)s" % {"tag": name})
                 start_name = open_elements.pop()
                 if start_name != name:
-                    raise LintError(_("EndTag (%(end)s) does not match StartTag (%(start)s)") % {"end": name, "start": start_name})
+                    raise LintError("EndTag (%(end)s) does not match StartTag (%(start)s)" % {"end": name, "start": start_name})
                 contentModelFlag = "PCDATA"
 
             elif type == "Comment":
                 if contentModelFlag != "PCDATA":
-                    raise LintError(_("Comment not in PCDATA content model flag"))
+                    raise LintError("Comment not in PCDATA content model flag")
 
             elif type in ("Characters", "SpaceCharacters"):
                 data = token["data"]
                 if not isinstance(data, str):
-                    raise LintError(_("Attribute name is not a string: %(name)r") % {"name": data})
+                    raise LintError("Attribute name is not a string: %(name)r" % {"name": data})
                 if not data:
-                    raise LintError(_("%(type)s token with empty data") % {"type": type})
+                    raise LintError("%(type)s token with empty data" % {"type": type})
                 if type == "SpaceCharacters":
                     data = data.strip(spaceCharacters)
                     if data:
-                        raise LintError(_("Non-space character(s) found in SpaceCharacters token: %(token)r") % {"token": data})
+                        raise LintError("Non-space character(s) found in SpaceCharacters token: %(token)r" % {"token": data})
 
             elif type == "Doctype":
                 name = token["name"]
                 if contentModelFlag != "PCDATA":
-                    raise LintError(_("Doctype not in PCDATA content model flag: %(name)s") % {"name": name})
+                    raise LintError("Doctype not in PCDATA content model flag: %(name)s" % {"name": name})
                 if not isinstance(name, str):
-                    raise LintError(_("Tag name is not a string: %(tag)r") % {"tag": name})
+                    raise LintError("Tag name is not a string: %(tag)r" % {"tag": name})
                 # XXX: what to do with token["data"] ?
 
             elif type in ("ParseError", "SerializeError"):
                 pass
 
             else:
-                raise LintError(_("Unknown token type: %(type)s") % {"type": type})
+                raise LintError("Unknown token type: %(type)s" % {"type": type})
 
             yield token

included_dependencies/html5lib/html5parser.py

@@ -18,6 +18,7 @@ from .constants import cdataElements, rcdataElements
 from .constants import tokenTypes, ReparseException, namespaces
 from .constants import htmlIntegrationPointElements, mathmlTextIntegrationPointElements
 from .constants import adjustForeignAttributes as adjustForeignAttributesMap
+from .constants import E
 
 
 def parse(doc, treebuilder="etree", encoding=None,
@@ -129,6 +130,17 @@ class HTMLParser(object):
 
         self.framesetOK = True
 
+    @property
+    def documentEncoding(self):
+        """The name of the character encoding
+        that was used to decode the input stream,
+        or :obj:`None` if that is not determined yet.
+
+        """
+        if not hasattr(self, 'tokenizer'):
+            return None
+        return self.tokenizer.stream.charEncoding[0]
+
     def isHTMLIntegrationPoint(self, element):
         if (element.name == "annotation-xml" and
                 element.namespace == namespaces["mathml"]):
@@ -245,7 +257,7 @@ class HTMLParser(object):
         # XXX The idea is to make errorcode mandatory.
         self.errors.append((self.tokenizer.stream.position(), errorcode, datavars))
         if self.strict:
-            raise ParseError
+            raise ParseError(E[errorcode] % datavars)
 
     def normalizeToken(self, token):
         """ HTML5 specific normalizations to the token stream """
@@ -868,7 +880,7 @@ def getPhases(debug):
             self.startTagHandler = utils.MethodDispatcher([
                 ("html", self.startTagHtml),
                 (("base", "basefont", "bgsound", "command", "link", "meta",
-                  "noframes", "script", "style", "title"),
+                  "script", "style", "title"),
                  self.startTagProcessInHead),
                 ("body", self.startTagBody),
                 ("frameset", self.startTagFrameset),
@@ -1205,8 +1217,7 @@ def getPhases(debug):
             attributes["name"] = "isindex"
             self.processStartTag(impliedTagToken("input", "StartTag",
                                                  attributes=attributes,
-                                                 selfClosing=
-                                                 token["selfClosing"]))
+                                                 selfClosing=token["selfClosing"]))
             self.processEndTag(impliedTagToken("label"))
             self.processStartTag(impliedTagToken("hr", "StartTag"))
             self.processEndTag(impliedTagToken("form"))

included_dependencies/html5lib/inputstream.py

@@ -28,7 +28,18 @@ asciiLettersBytes = frozenset([item.encode("ascii") for item in asciiLetters])
 asciiUppercaseBytes = frozenset([item.encode("ascii") for item in asciiUppercase])
 spacesAngleBrackets = spaceCharactersBytes | frozenset([b">", b"<"])
 
-invalid_unicode_re = re.compile("[\u0001-\u0008\u000B\u000E-\u001F\u007F-\u009F\uD800-\uDFFF\uFDD0-\uFDEF\uFFFE\uFFFF\U0001FFFE\U0001FFFF\U0002FFFE\U0002FFFF\U0003FFFE\U0003FFFF\U0004FFFE\U0004FFFF\U0005FFFE\U0005FFFF\U0006FFFE\U0006FFFF\U0007FFFE\U0007FFFF\U0008FFFE\U0008FFFF\U0009FFFE\U0009FFFF\U000AFFFE\U000AFFFF\U000BFFFE\U000BFFFF\U000CFFFE\U000CFFFF\U000DFFFE\U000DFFFF\U000EFFFE\U000EFFFF\U000FFFFE\U000FFFFF\U0010FFFE\U0010FFFF]")
+
+invalid_unicode_no_surrogate = "[\u0001-\u0008\u000B\u000E-\u001F\u007F-\u009F\uFDD0-\uFDEF\uFFFE\uFFFF\U0001FFFE\U0001FFFF\U0002FFFE\U0002FFFF\U0003FFFE\U0003FFFF\U0004FFFE\U0004FFFF\U0005FFFE\U0005FFFF\U0006FFFE\U0006FFFF\U0007FFFE\U0007FFFF\U0008FFFE\U0008FFFF\U0009FFFE\U0009FFFF\U000AFFFE\U000AFFFF\U000BFFFE\U000BFFFF\U000CFFFE\U000CFFFF\U000DFFFE\U000DFFFF\U000EFFFE\U000EFFFF\U000FFFFE\U000FFFFF\U0010FFFE\U0010FFFF]"
+
+if utils.supports_lone_surrogates:
+    # Use one extra step of indirection and create surrogates with
+    # unichr. Not using this indirection would introduce an illegal
+    # unicode literal on platforms not supporting such lone
+    # surrogates.
+    invalid_unicode_re = re.compile(invalid_unicode_no_surrogate +
+                                    eval('"\\uD800-\\uDFFF"'))
+else:
+    invalid_unicode_re = re.compile(invalid_unicode_no_surrogate)
 
 non_bmp_invalid_codepoints = set([0x1FFFE, 0x1FFFF, 0x2FFFE, 0x2FFFF, 0x3FFFE,
                                   0x3FFFF, 0x4FFFE, 0x4FFFF, 0x5FFFE, 0x5FFFF,
@@ -164,13 +175,18 @@ class HTMLUnicodeInputStream(object):
 
         """
 
-        # Craziness
-        if len("\U0010FFFF") == 1:
+        if not utils.supports_lone_surrogates:
+            # Such platforms will have already checked for such
+            # surrogate errors, so no need to do this checking.
+            self.reportCharacterErrors = None
+            self.replaceCharactersRegexp = None
+        elif len("\U0010FFFF") == 1:
             self.reportCharacterErrors = self.characterErrorsUCS4
-            self.replaceCharactersRegexp = re.compile("[\uD800-\uDFFF]")
+            self.replaceCharactersRegexp = re.compile(eval('"[\\uD800-\\uDFFF]"'))
         else:
             self.reportCharacterErrors = self.characterErrorsUCS2
-            self.replaceCharactersRegexp = re.compile("([\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?<![\uD800-\uDBFF])[\uDC00-\uDFFF])")
+            self.replaceCharactersRegexp = re.compile(
+                eval('"([\\uD800-\\uDBFF](?![\\uDC00-\\uDFFF])|(?<![\\uD800-\\uDBFF])[\\uDC00-\\uDFFF])"'))
 
         # List of where new lines occur
         self.newLines = [0]
@@ -265,11 +281,12 @@ class HTMLUnicodeInputStream(object):
                 self._bufferedCharacter = data[-1]
                 data = data[:-1]
 
-        self.reportCharacterErrors(data)
+        if self.reportCharacterErrors:
+            self.reportCharacterErrors(data)
 
-        # Replace invalid characters
-        # Note U+0000 is dealt with in the tokenizer
-        data = self.replaceCharactersRegexp.sub("\ufffd", data)
+            # Replace invalid characters
+            # Note U+0000 is dealt with in the tokenizer
+            data = self.replaceCharactersRegexp.sub("\ufffd", data)
 
         data = data.replace("\r\n", "\n")
         data = data.replace("\r", "\n")

included_dependencies/html5lib/sanitizer.py

@@ -2,11 +2,26 @@ from __future__ import absolute_import, division, unicode_literals
 
 import re
 from xml.sax.saxutils import escape, unescape
+from six.moves import urllib_parse as urlparse
 
 from .tokenizer import HTMLTokenizer
 from .constants import tokenTypes
 
 
+content_type_rgx = re.compile(r'''
+                               ^
+                               # Match a content type <application>/<type>
+                               (?P<content_type>[-a-zA-Z0-9.]+/[-a-zA-Z0-9.]+)
+                               # Match any character set and encoding
+                               (?:(?:;charset=(?:[-a-zA-Z0-9]+)(?:;(?:base64))?)
+                                 |(?:;(?:base64))?(?:;charset=(?:[-a-zA-Z0-9]+))?)
+                               # Assume the rest is data
+                               ,.*
+                               $
+                               ''',
+                              re.VERBOSE)
+
+
 class HTMLSanitizerMixin(object):
     """ sanitization of XHTML+MathML+SVG and of inline style attributes."""
 
@@ -100,8 +115,8 @@ class HTMLSanitizerMixin(object):
                       'xml:base', 'xml:lang', 'xml:space', 'xmlns', 'xmlns:xlink', 'y',
                       'y1', 'y2', 'zoomAndPan']
 
-    attr_val_is_uri = ['href', 'src', 'cite', 'action', 'longdesc', 'poster',
-                       'xlink:href', 'xml:base']
+    attr_val_is_uri = ['href', 'src', 'cite', 'action', 'longdesc', 'poster', 'background', 'datasrc',
+                       'dynsrc', 'lowsrc', 'ping', 'poster', 'xlink:href', 'xml:base']
 
     svg_attr_val_allows_ref = ['clip-path', 'color-profile', 'cursor', 'fill',
                                'filter', 'marker', 'marker-start', 'marker-mid', 'marker-end',
@@ -138,7 +153,9 @@ class HTMLSanitizerMixin(object):
     acceptable_protocols = ['ed2k', 'ftp', 'http', 'https', 'irc',
                             'mailto', 'news', 'gopher', 'nntp', 'telnet', 'webcal',
                             'xmpp', 'callto', 'feed', 'urn', 'aim', 'rsync', 'tag',
-                            'ssh', 'sftp', 'rtsp', 'afs']
+                            'ssh', 'sftp', 'rtsp', 'afs', 'data']
+
+    acceptable_content_types = ['image/png', 'image/jpeg', 'image/gif', 'image/webp', 'image/bmp', 'text/plain']
 
     # subclasses may define their own versions of these constants
     allowed_elements = acceptable_elements + mathml_elements + svg_elements
@@ -147,6 +164,7 @@ class HTMLSanitizerMixin(object):
     allowed_css_keywords = acceptable_css_keywords
     allowed_svg_properties = acceptable_svg_properties
     allowed_protocols = acceptable_protocols
+    allowed_content_types = acceptable_content_types
 
     # Sanitize the +html+, escaping all elements not in ALLOWED_ELEMENTS, and
     # stripping out all # attributes not in ALLOWED_ATTRIBUTES. Style
@@ -189,10 +207,17 @@ class HTMLSanitizerMixin(object):
                                        unescape(attrs[attr])).lower()
                 # remove replacement characters from unescaped characters
                 val_unescaped = val_unescaped.replace("\ufffd", "")
-                if (re.match("^[a-z0-9][-+.a-z0-9]*:", val_unescaped) and
-                    (val_unescaped.split(':')[0] not in
-                     self.allowed_protocols)):
-                    del attrs[attr]
+                uri = urlparse.urlparse(val_unescaped)
+                if uri:
+                    if uri.scheme not in self.allowed_protocols:
+                        del attrs[attr]
+                    if uri.scheme == 'data':
+                        m = content_type_rgx.match(uri.path)
+                        if not m:
+                            del attrs[attr]
+                        elif m.group('content_type') not in self.allowed_content_types:
+                            del attrs[attr]
+
             for attr in self.svg_attr_val_allows_ref:
                 if attr in attrs:
                     attrs[attr] = re.sub(r'url\s*\(\s*[^#\s][^)]+?\)',
@@ -245,7 +270,7 @@ class HTMLSanitizerMixin(object):
             elif prop.split('-')[0].lower() in ['background', 'border', 'margin',
                                                 'padding']:
                 for keyword in value.split():
-                    if not keyword in self.acceptable_css_keywords and \
+                    if keyword not in self.acceptable_css_keywords and \
                             not re.match("^(#[0-9a-f]+|rgb\(\d+%?,\d*%?,?\d*%?\)?|\d{0,2}\.?\d{0,2}(cm|em|ex|in|mm|pc|pt|px|%|,|\))?)$", keyword):
                         break
                 else:

included_dependencies/html5lib/serializer/htmlserializer.py

@@ -1,9 +1,6 @@
 from __future__ import absolute_import, division, unicode_literals
 from six import text_type
 
-import gettext
-_ = gettext.gettext
-
 try:
     from functools import reduce
 except ImportError:
@@ -35,7 +32,7 @@ else:
                 v = utils.surrogatePairToCodepoint(v)
             else:
                 v = ord(v)
-            if not v in encode_entity_map or k.islower():
+            if v not in encode_entity_map or k.islower():
                 # prefer < over < and similarly for &, >, etc.
                 encode_entity_map[v] = k
 
@@ -208,7 +205,7 @@ class HTMLSerializer(object):
                 if token["systemId"]:
                     if token["systemId"].find('"') >= 0:
                         if token["systemId"].find("'") >= 0:
-                            self.serializeError(_("System identifer contains both single and double quote characters"))
+                            self.serializeError("System identifer contains both single and double quote characters")
                         quote_char = "'"
                     else:
                         quote_char = '"'
@@ -220,7 +217,7 @@ class HTMLSerializer(object):
             elif type in ("Characters", "SpaceCharacters"):
                 if type == "SpaceCharacters" or in_cdata:
                     if in_cdata and token["data"].find("</") >= 0:
-                        self.serializeError(_("Unexpected </ in CDATA"))
+                        self.serializeError("Unexpected </ in CDATA")
                     yield self.encode(token["data"])
                 else:
                     yield self.encode(escape(token["data"]))
@@ -231,7 +228,7 @@ class HTMLSerializer(object):
                 if name in rcdataElements and not self.escape_rcdata:
                     in_cdata = True
                 elif in_cdata:
-                    self.serializeError(_("Unexpected child element of a CDATA element"))
+                    self.serializeError("Unexpected child element of a CDATA element")
                 for (attr_namespace, attr_name), attr_value in token["data"].items():
                     # TODO: Add namespace support here
                     k = attr_name
@@ -279,20 +276,20 @@ class HTMLSerializer(object):
                 if name in rcdataElements:
                     in_cdata = False
                 elif in_cdata:
-                    self.serializeError(_("Unexpected child element of a CDATA element"))
+                    self.serializeError("Unexpected child element of a CDATA element")
                 yield self.encodeStrict("</%s>" % name)
 
             elif type == "Comment":
                 data = token["data"]
                 if data.find("--") >= 0:
-                    self.serializeError(_("Comment contains --"))
+                    self.serializeError("Comment contains --")
                 yield self.encodeStrict("<!--%s-->" % token["data"])
 
             elif type == "Entity":
                 name = token["name"]
                 key = name + ";"
-                if not key in entities:
-                    self.serializeError(_("Entity %s not recognized" % name))
+                if key not in entities:
+                    self.serializeError("Entity %s not recognized" % name)
                 if self.resolve_entities and key not in xmlEntities:
                     data = entities[key]
                 else:

included_dependencies/html5lib/treebuilders/dom.py

@@ -158,7 +158,7 @@ def getDomBuilder(DomImplementation):
             else:
                 # HACK: allow text nodes as children of the document node
                 if hasattr(self.dom, '_child_node_types'):
-                    if not Node.TEXT_NODE in self.dom._child_node_types:
+                    if Node.TEXT_NODE not in self.dom._child_node_types:
                         self.dom._child_node_types = list(self.dom._child_node_types)
                         self.dom._child_node_types.append(Node.TEXT_NODE)
                 self.dom.appendChild(self.dom.createTextNode(data))

included_dependencies/html5lib/treewalkers/__init__.py

@@ -10,8 +10,12 @@ returning an iterator generating tokens.
 
 from __future__ import absolute_import, division, unicode_literals
 
+__all__ = ["getTreeWalker", "pprint", "dom", "etree", "genshistream", "lxmletree",
+           "pulldom"]
+
 import sys
 
+from .. import constants
 from ..utils import default_etree
 
 treeWalkerCache = {}
@@ -55,3 +59,89 @@ def getTreeWalker(treeType, implementation=None, **kwargs):
             # XXX: NEVER cache here, caching is done in the etree submodule
             return etree.getETreeModule(implementation, **kwargs).TreeWalker
     return treeWalkerCache.get(treeType)
+
+
+def concatenateCharacterTokens(tokens):
+    pendingCharacters = []
+    for token in tokens:
+        type = token["type"]
+        if type in ("Characters", "SpaceCharacters"):
+            pendingCharacters.append(token["data"])
+        else:
+            if pendingCharacters:
+                yield {"type": "Characters", "data": "".join(pendingCharacters)}
+                pendingCharacters = []
+            yield token
+    if pendingCharacters:
+        yield {"type": "Characters", "data": "".join(pendingCharacters)}
+
+
+def pprint(walker):
+    """Pretty printer for tree walkers"""
+    output = []
+    indent = 0
+    for token in concatenateCharacterTokens(walker):
+        type = token["type"]
+        if type in ("StartTag", "EmptyTag"):
+            # tag name
+            if token["namespace"] and token["namespace"] != constants.namespaces["html"]:
+                if token["namespace"] in constants.prefixes:
+                    ns = constants.prefixes[token["namespace"]]
+                else:
+                    ns = token["namespace"]
+                name = "%s %s" % (ns, token["name"])
+            else:
+                name = token["name"]
+            output.append("%s<%s>" % (" " * indent, name))
+            indent += 2
+            # attributes (sorted for consistent ordering)
+            attrs = token["data"]
+            for (namespace, localname), value in sorted(attrs.items()):
+                if namespace:
+                    if namespace in constants.prefixes:
+                        ns = constants.prefixes[namespace]
+                    else:
+                        ns = namespace
+                    name = "%s %s" % (ns, localname)
+                else:
+                    name = localname
+                output.append("%s%s=\"%s\"" % (" " * indent, name, value))
+            # self-closing
+            if type == "EmptyTag":
+                indent -= 2
+
+        elif type == "EndTag":
+            indent -= 2
+
+        elif type == "Comment":
+            output.append("%s<!-- %s -->" % (" " * indent, token["data"]))
+
+        elif type == "Doctype":
+            if token["name"]:
+                if token["publicId"]:
+                    output.append("""%s<!DOCTYPE %s "%s" "%s">""" %
+                                  (" " * indent,
+                                   token["name"],
+                                   token["publicId"],
+                                   token["systemId"] if token["systemId"] else ""))
+                elif token["systemId"]:
+                    output.append("""%s<!DOCTYPE %s "" "%s">""" %
+                                  (" " * indent,
+                                   token["name"],
+                                   token["systemId"]))
+                else:
+                    output.append("%s<!DOCTYPE %s>" % (" " * indent,
+                                                       token["name"]))
+            else:
+                output.append("%s<!DOCTYPE >" % (" " * indent,))
+
+        elif type == "Characters":
+            output.append("%s\"%s\"" % (" " * indent, token["data"]))
+
+        elif type == "SpaceCharacters":
+            assert False, "concatenateCharacterTokens should have got rid of all Space tokens"
+
+        else:
+            raise ValueError("Unknown token type, %s" % type)
+
+    return "\n".join(output)

included_dependencies/html5lib/treewalkers/_base.py

@@ -1,8 +1,8 @@
 from __future__ import absolute_import, division, unicode_literals
 from six import text_type, string_types
 
-import gettext
-_ = gettext.gettext
+__all__ = ["DOCUMENT", "DOCTYPE", "TEXT", "ELEMENT", "COMMENT", "ENTITY", "UNKNOWN",
+           "TreeWalker", "NonRecursiveTreeWalker"]
 
 from xml.dom import Node
 
@@ -58,7 +58,7 @@ class TreeWalker(object):
                "namespace": to_text(namespace),
                "data": attrs}
         if hasChildren:
-            yield self.error(_("Void element has children"))
+            yield self.error("Void element has children")
 
     def startTag(self, namespace, name, attrs):
         assert namespace is None or isinstance(namespace, string_types), type(namespace)
@@ -122,7 +122,7 @@ class TreeWalker(object):
         return {"type": "Entity", "name": text_type(name)}
 
     def unknown(self, nodeType):
-        return self.error(_("Unknown node type: ") + nodeType)
+        return self.error("Unknown node type: " + nodeType)
 
 
 class NonRecursiveTreeWalker(TreeWalker):

included_dependencies/html5lib/treewalkers/dom.py

@@ -2,9 +2,6 @@ from __future__ import absolute_import, division, unicode_literals
 
 from xml.dom import Node
 
-import gettext
-_ = gettext.gettext
-
 from . import _base
 
 

included_dependencies/html5lib/treewalkers/etree.py

@@ -7,8 +7,6 @@ except ImportError:
         from ordereddict import OrderedDict
     except ImportError:
         OrderedDict = dict
-import gettext
-_ = gettext.gettext
 
 import re
 

included_dependencies/html5lib/treewalkers/lxmletree.py

@@ -4,9 +4,6 @@ from six import text_type
 from lxml import etree
 from ..treebuilders.etree import tag_regexp
 
-from gettext import gettext
-_ = gettext
-
 from . import _base
 
 from .. import ihatexml
@@ -130,7 +127,7 @@ class TreeWalker(_base.NonRecursiveTreeWalker):
     def getNodeDetails(self, node):
         if isinstance(node, tuple):  # Text node
             node, key = node
-            assert key in ("text", "tail"), _("Text nodes are text or tail, found %s") % key
+            assert key in ("text", "tail"), "Text nodes are text or tail, found %s" % key
             return _base.TEXT, ensure_str(getattr(node, key))
 
         elif isinstance(node, Root):
@@ -169,7 +166,7 @@ class TreeWalker(_base.NonRecursiveTreeWalker):
                     attrs, len(node) > 0 or node.text)
 
     def getFirstChild(self, node):
-        assert not isinstance(node, tuple), _("Text nodes have no children")
+        assert not isinstance(node, tuple), "Text nodes have no children"
 
         assert len(node) or node.text, "Node has no children"
         if node.text:
@@ -180,7 +177,7 @@ class TreeWalker(_base.NonRecursiveTreeWalker):
     def getNextSibling(self, node):
         if isinstance(node, tuple):  # Text node
             node, key = node
-            assert key in ("text", "tail"), _("Text nodes are text or tail, found %s") % key
+            assert key in ("text", "tail"), "Text nodes are text or tail, found %s" % key
             if key == "text":
                 # XXX: we cannot use a "bool(node) and node[0] or None" construct here
                 # because node[0] might evaluate to False if it has no child element
@@ -196,7 +193,7 @@ class TreeWalker(_base.NonRecursiveTreeWalker):
     def getParentNode(self, node):
         if isinstance(node, tuple):  # Text node
             node, key = node
-            assert key in ("text", "tail"), _("Text nodes are text or tail, found %s") % key
+            assert key in ("text", "tail"), "Text nodes are text or tail, found %s" % key
             if key == "text":
                 return node
             # else: fallback to "normal" processing

included_dependencies/html5lib/utils.py

@@ -2,6 +2,8 @@ from __future__ import absolute_import, division, unicode_literals
 
 from types import ModuleType
 
+from six import text_type
+
 try:
     import xml.etree.cElementTree as default_etree
 except ImportError:
@@ -9,7 +11,26 @@ except ImportError:
 
 
 __all__ = ["default_etree", "MethodDispatcher", "isSurrogatePair",
-           "surrogatePairToCodepoint", "moduleFactoryFactory"]
+           "surrogatePairToCodepoint", "moduleFactoryFactory",
+           "supports_lone_surrogates"]
+
+
+# Platforms not supporting lone surrogates (\uD800-\uDFFF) should be
+# caught by the below test. In general this would be any platform
+# using UTF-16 as its encoding of unicode strings, such as
+# Jython. This is because UTF-16 itself is based on the use of such
+# surrogates, and there is no mechanism to further escape such
+# escapes.
+try:
+    _x = eval('"\\uD800"')
+    if not isinstance(_x, text_type):
+        # We need this with u"" because of http://bugs.jython.org/issue2039
+        _x = eval('u"\\uD800"')
+        assert isinstance(_x, text_type)
+except:
+    supports_lone_surrogates = False
+else:
+    supports_lone_surrogates = True
 
 
 class MethodDispatcher(dict):

makeplugin.py

@@ -25,7 +25,7 @@ if __name__=="__main__":
     exclude=['*.pyc','*~','*.xcf','*[0-9].png','*.po','*.pot','*default.mo','*Thumbs.db']
     
     os.chdir('calibre-plugin')
-    files=['plugin-defaults.ini','plugin-example.ini','about.txt',
+    files=['plugin-defaults.ini','plugin-example.ini','about.html',
            'images','translations']
     files.extend(glob('*.py'))
     files.extend(glob('plugin-import-name-*.txt'))
@@ -36,7 +36,7 @@ if __name__=="__main__":
 
     os.chdir('../included_dependencies')
     # 'a' for append
-    files=['six.py','bs4','html5lib','chardet']
+    files=['gif.py','six.py','bs4','html5lib','chardet']
     createZipFile("../"+filename,"a",
                   files,
                   exclude=exclude)

setup.py

@@ -25,7 +25,7 @@ setup(
     # Versions should comply with PEP440.  For a discussion on single-sourcing
     # the version across setup.py and the project code, see
     # https://packaging.python.org/en/latest/single_source_version.html
-    version="2.2.2",
+    version="2.2.8",
 
     description='A tool for downloading fanfiction to eBook formats',
     long_description=long_description,

webservice/app.yaml

@@ -1,6 +1,6 @@
 # ffd-retief-hrd fanfictiondownloader fanficfare
 application: fanficfare
-version: 2-2-2
+version: 2-2-8
 runtime: python27
 api_version: 1
 threadsafe: true

webservice/index.html

@@ -46,19 +46,7 @@
           </p>
 	  <h3>Changes:</h3>
 	  <ul>
-            <li>Change the name to FanFicFare.</li>
-            <li>Fixes for literotica changes.</li>
-            <li>Fix error in debug output that uses wrong variable.</li>
-            <li>Change date format for onedirectionfanfiction.com.</li>
-            <li>Explicit html parse for BS4 when reading in existing epub.  For cal1.48.</li>
-            <li>Fix for PI - eFiction Base adapter Anthology Series string.</li>
-            <li>Update translations.</li>
-            <li>Change some menu & gui details to be clearer.</li>
-            <li>Fix for ffnet's latest change to story/cover image.</li>
-            <li>Fix problems found during trawl through all sites -- thehookupzone.net and libraryofmoria.com now use eFiction Base.</li>
-            <li>Remove defunct sites - netraptor.org www.simplyundeniable.com www.wizardtales.net www.checkmated.com stargate-atlantis.org www.henneth-annun.net www.twiwrite.net www.dotmoon.net</li>
-            <li>Add option to check ffnet auth img vs cover img and not use author images as book covers.  Defaults to on, add <b>skip_author_cover:false</b> to <b>[fanfiction.net]</b> section if you like author image covers.</li>
-            <li>Add site tgstorytime.com as an eFiction Base Adapter.</li>
+            <li>Fix add_genre_when_multi_category when genre is empty.</li>
 	  </ul>
 	  <p>
 	    Questions?  Check out our 
@@ -68,8 +56,8 @@
 	    If you have any problems with this application, please
 	    report them in
 	    the <a href="http://groups.google.com/group/fanfic-downloader">FanFicFare Google Group</a>.  The
-	    <a href="http://fanfictiondownloader.appspot.com">FanFictionDownLoader(FFDL)
-	    Version</a> is also available for you to use if necessary.
+	    <a href="http://2-2-6.fanficfare.appspot.com">previous version
+	    </a> is also available for you to use if necessary.
 	  </p>
 	  <div id='error'>
 	    {{ error_message }}
@@ -130,7 +118,7 @@
 	    Once you have calibre installed and running, inside
 	    calibre, you can go to 'Get plugins to enhance calibre' or
 	    'Get new plugins' and
-	    install <a href="http://www.mobileread.com/forums/showthread.php?t=163261">FanFicFare</a>.
+	    install <a href="http://www.mobileread.com/forums/showthread.php?t=259221">FanFicFare</a>.
 	    
 	  </p>
       </div>