Bump versions

Update example URLs for mediaminer.org.
Allow forum.questionablequesting.com now as well as questionablequesting.com.
2026-09-13 12:11:20 +08:00 · 2015-09-17 10:50:46 -05:00 · 2015-09-17 10:46:56 -05:00 · 2015-09-16 11:35:06 -05:00 · 2015-09-14 12:33:06 -05:00 · 2015-09-14 10:22:20 -05:00
84 changed files with 10094 additions and 6079 deletions
@@ -0,0 +1 @@
+include DESCRIPTION.rst
@@ -10,8 +10,10 @@ FFDL, AKA fanficdownloader) project previously hosted as a
 This program is available as a calibre plugin, a command-line
 interface, and a web service.

-*Currently, the project is the middle of being moved from code.google
-to GitHub and being renamed.  For now, continue to use
-FanFictionDownLoader(FFDL).*
+FanFicFare has now been launched.  New versions, features and updates
+will all be in FanFicFare.

-[code.google project]: https://code.google.com/p/fanficdownloader/
+There's additional info in the project [wiki] pages.
+
+[code.google project]: http://google-opensource.blogspot.com/2015/03/farewell-to-google-code.html
+[wiki]: https://github.com/JimmXinu/FanFicFare/wiki
@@ -42,7 +42,7 @@ class FanFicFareBase(InterfaceActionBase):
    description         = _('UI plugin to download FanFiction stories from various sites.')
    supported_platforms = ['windows', 'osx', 'linux']
    author              = 'Jim Miller'
-    version             = (2, 2, 2)
+    version             = (2, 2, 13)
    minimum_calibre_version = (1, 48, 0)

    #: This field defines the GUI plugin class that contains all the code
@@ -0,0 +1,32 @@
+<hr />
+
+<p>Plugin created by Jim Miller, borrowing heavily from Grant Drake's
+  '<a href="http://www.mobileread.com/forums/showthread.php?t=134856">Reading List</a>',
+  '<a href="http://www.mobileread.com/forums/showthread.php?t=126727">Extract ISBN</a>' and
+  '<a href="http://www.mobileread.com/forums/showthread.php?t=134000">Count Pages</a>'
+  plugins.</p>
+
+<p>
+  Calibre officially distributes plugins from the mobileread.com forum site.
+  The official distro channel for this plugin is there: <a href="http://www.mobileread.com/forums/showthread.php?t=259221">FanFicFare</a>
+</p>
+
+<p> I also monitor the
+  <a href="http://groups.google.com/group/fanfic-downloader">general users
+    group</a> for the downloader.  That covers the web application and CLI, too.
+</p>
+
+<p>
+  The source for this plugin is available at it's 
+  <a href="https://github.com/JimmXinu/FanFicFare">project home</a>.
+</p>
+
+<hr />
+
+<p>
+  See the <a href="https://github.com/JimmXinu/FanFicFare/wiki/Supportedsites">list of supported sites</a>.
+</p>
+
+<p>
+  Read the <a href="https://github.com/JimmXinu/FanFicFare/wiki/FAQs">FAQs</a>.
+</p>
@@ -1,28 +0,0 @@
-<hr />
-
-<p>Plugin created by Jim Miller, borrowing heavily from Grant Drake's
-'<a href="http://www.mobileread.com/forums/showthread.php?t=134856">Reading List</a>',
-'<a href="http://www.mobileread.com/forums/showthread.php?t=126727">Extract ISBN</a>' and
-'<a href="http://www.mobileread.com/forums/showthread.php?t=134000">Count Pages</a>'
-plugins.</p>
-
-<p>
-Calibre officially distributes plugins from the mobileread.com forum site.
-The official distro channel for this plugin is there: <a href="http://www.mobileread.com/forums/showthread.php?t=163261">FanFictionDownLoader</a>
-</p>
-
-<p> I also monitor the
-<a href="http://groups.google.com/group/fanfic-downloader">general users
-group</a> for the downloader.  That covers the web application and CLI, too.
-</p>
-
-The source for this plugin is available at it's 
-<a href="https://github.com/JimmXinu/FanFicFare">project home</a>.
-<hr />
-
-<p>
-See the <a href="https://github.com/JimmXinu/FanFicFare/wiki/Supportedsites">list of supported sites</a>.
-</p>
-<p>
-Read the <a href="https://github.com/JimmXinu/FanFicFare/wiki/FAQs">FAQs</a>.
-</p>
@@ -1,5 +1,5 @@
-#!/usr/bin/env python
-# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+# -*- coding: utf-8 -*-
+
 from __future__ import (unicode_literals, division,
                        print_function)

@@ -1,5 +1,5 @@
-#!/usr/bin/env python
-# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+# -*- coding: utf-8 -*-
+
 from __future__ import (unicode_literals, division, absolute_import,
                        print_function)

@@ -1,5 +1,5 @@
-#!/usr/bin/env python
-# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+# -*- coding: utf-8 -*-
+
 from __future__ import (unicode_literals, division, absolute_import,
                        print_function)

@@ -17,12 +17,12 @@ try:
    from PyQt5.Qt import (QDialog, QWidget, QVBoxLayout, QHBoxLayout, QGridLayout,
                          QLabel, QLineEdit, QFont, QWidget, QTextEdit, QComboBox,
                          QCheckBox, QPushButton, QTabWidget, QScrollArea,
-                          QDialogButtonBox, QGroupBox, Qt )
+                          QDialogButtonBox, QGroupBox, QButtonGroup, QRadioButton, Qt)
 except ImportError as e:
    from PyQt4.Qt import (QDialog, QWidget, QVBoxLayout, QHBoxLayout, QGridLayout,
                          QLabel, QLineEdit, QFont, QWidget, QTextEdit, QComboBox,
                          QCheckBox, QPushButton, QTabWidget, QScrollArea,
-                          QDialogButtonBox, QGroupBox, Qt )
+                          QDialogButtonBox, QGroupBox, QButtonGroup, QRadioButton, Qt)
 try:
    from calibre.gui2 import QVariant
    del QVariant
@@ -39,9 +39,15 @@ else:
            return [convert_qvariant(i) for i in x.toList()]
        return x.toPyObject()

-from calibre.gui2.ui import get_gui
 from calibre.gui2 import dynamic, info_dialog, question_dialog
-from calibre.constants import numeric_version as calibre_version
+from calibre.gui2.ui import get_gui
+from calibre.gui2.complete2 import EditWithComplete
+
+try:
+    from calibre.ebooks.covers import generate_cover as cal_generate_cover
+    HAS_CALGC=True
+except:
+    HAS_CALGC=False

 # pulls in translation files for _() strings
 try:
@@ -49,12 +55,14 @@ try:
 except NameError:
    pass # load_translations() added in calibre 1.9

+from calibre.library.field_metadata import FieldMetadata
+field_metadata = FieldMetadata()
+
 # There are a number of things used several times that shouldn't be
 # translated.  This is just a way to make that easier by keeping them
 # out of the _() strings.
 # I'm tempted to override _() to include them...
 no_trans = { 'pini':'personal.ini',
-             'imgset':'\n\n[epub]\ninclude_images:true\nkeep_summary_html:true\nmake_firstimage_cover:true\n\n',
             'gcset':'generate_cover_settings',
             'ccset':'custom_columns_settings',
             'gc':'Generate Cover',
@@ -70,13 +78,18 @@ no_trans = { 'pini':'personal.ini',
             'p':'password',
             }

-from calibre_plugins.fanficfare_plugin.prefs import prefs, PREFS_NAMESPACE
+STD_COLS_SKIP = ['size','cover','news','ondevice','path','series_sort','sort']
+
+from calibre_plugins.fanficfare_plugin.prefs \
+    import (prefs, PREFS_NAMESPACE, updatecalcover_order, calcover_save_options,
+            gencalcover_order, SAVE_YES, SAVE_NO)
+
 from calibre_plugins.fanficfare_plugin.dialogs \
    import (UPDATE, UPDATEALWAYS, collision_order, save_collisions, RejectListDialog,
            EditTextDialog, IniTextDialog, RejectUrlEntry)
    
 from calibre_plugins.fanficfare_plugin.fanficfare.adapters \
-    import getConfigSections
+    import getSiteSections

 from calibre_plugins.fanficfare_plugin.common_utils \
    import ( KeyboardConfigDialog, PrefsViewerDialog )
@@ -84,8 +97,6 @@ from calibre_plugins.fanficfare_plugin.common_utils \
 from calibre_plugins.fanficfare_plugin.fff_util \
    import (test_config)

-from calibre.gui2.complete2 import EditWithComplete  #MultiCompleteLineEdit
-
 class RejectURLList:
    def __init__(self,prefs):
        self.prefs = prefs
@@ -205,10 +216,8 @@ class ConfigWidget(QWidget):
        if 'Reading List' not in plugin_action.gui.iactions:
            self.readinglist_tab.setEnabled(False)

-        self.generatecover_tab = GenerateCoverTab(self, plugin_action)
-        tab_widget.addTab(self.generatecover_tab, 'Generate Cover')
-        if 'Generate Cover' not in plugin_action.gui.iactions:
-            self.generatecover_tab.setEnabled(False)
+        self.calibrecover_tab = CalibreCoverTab(self, plugin_action)
+        tab_widget.addTab(self.calibrecover_tab, _('Calibre Cover'))

        self.countpages_tab = CountPagesTab(self, plugin_action)
        tab_widget.addTab(self.countpages_tab, 'Count Pages')
@@ -234,7 +243,6 @@ class ConfigWidget(QWidget):
        prefs['fileform'] = unicode(self.basic_tab.fileform.currentText())
        prefs['collision'] = save_collisions[unicode(self.basic_tab.collision.currentText())]
        prefs['updatemeta'] = self.basic_tab.updatemeta.isChecked()
-        prefs['updatecover'] = self.basic_tab.updatecover.isChecked()
        prefs['updateepubcover'] = self.basic_tab.updateepubcover.isChecked()
        prefs['keeptags'] = self.basic_tab.keeptags.isChecked()
        prefs['suppressauthorsort'] = self.basic_tab.suppressauthorsort.isChecked()
@@ -247,11 +255,11 @@ class ConfigWidget(QWidget):
        prefs['updatedefault'] = self.basic_tab.updatedefault.isChecked()
        prefs['deleteotherforms'] = self.basic_tab.deleteotherforms.isChecked()
        prefs['adddialogstaysontop'] = self.basic_tab.adddialogstaysontop.isChecked()
-        prefs['includeimages'] = self.basic_tab.includeimages.isChecked()
        prefs['lookforurlinhtml'] = self.basic_tab.lookforurlinhtml.isChecked()
        prefs['checkforseriesurlid'] = self.basic_tab.checkforseriesurlid.isChecked()
        prefs['checkforurlchange'] = self.basic_tab.checkforurlchange.isChecked()
        prefs['injectseries'] = self.basic_tab.injectseries.isChecked()
+        prefs['matchtitleauth'] = self.basic_tab.matchtitleauth.isChecked()
        prefs['smarten_punctuation'] = self.basic_tab.smarten_punctuation.isChecked()
        prefs['reject_always'] = self.basic_tab.reject_always.isChecked()

@@ -264,6 +272,7 @@ class ConfigWidget(QWidget):
            prefs['addtolists'] = self.readinglist_tab.addtolists.isChecked()
            prefs['addtoreadlists'] = self.readinglist_tab.addtoreadlists.isChecked()
            prefs['addtolistsonread'] = self.readinglist_tab.addtolistsonread.isChecked()
+            prefs['autounnew'] = self.readinglist_tab.autounnew.isChecked()

        # personal.ini
        ini = self.personalini_tab.personalini
@@ -273,17 +282,25 @@ class ConfigWidget(QWidget):
            # if they've removed everything, reset to default.
            prefs['personal.ini'] = get_resources('plugin-example.ini')

-        # Generate Covers tab
-        prefs['gcnewonly'] = self.generatecover_tab.gcnewonly.isChecked()
+        prefs['cal_cols_pass_in'] = self.personalini_tab.cal_cols_pass_in.isChecked()
+            
+        # Covers tab
+        prefs['updatecalcover'] = calcover_save_options[unicode(self.calibrecover_tab.updatecalcover.currentText())]
+        # for backward compatibility:
+        prefs['updatecover'] = prefs['updatecalcover'] == SAVE_YES
+        prefs['gencalcover'] = calcover_save_options[unicode(self.calibrecover_tab.gencalcover.currentText())]
+        prefs['calibre_gen_cover'] = self.calibrecover_tab.calibre_gen_cover.isChecked()
+        prefs['plugin_gen_cover'] = self.calibrecover_tab.plugin_gen_cover.isChecked()
+        prefs['gcnewonly'] = self.calibrecover_tab.gcnewonly.isChecked()
        gc_site_settings = {}
-        for (site,combo) in self.generatecover_tab.gc_dropdowns.iteritems():
+        for (site,combo) in self.calibrecover_tab.gc_dropdowns.iteritems():
            val = unicode(convert_qvariant(combo.itemData(combo.currentIndex())))
            if val != 'none':
                gc_site_settings[site] = val
                #print("gc_site_settings[%s]:%s"%(site,gc_site_settings[site]))
        prefs['gc_site_settings'] = gc_site_settings
-        prefs['allow_gc_from_ini'] = self.generatecover_tab.allow_gc_from_ini.isChecked()
-        prefs['gc_polish_cover'] = self.generatecover_tab.gc_polish_cover.isChecked()
+        prefs['allow_gc_from_ini'] = self.calibrecover_tab.allow_gc_from_ini.isChecked()
+        prefs['gc_polish_cover'] = self.calibrecover_tab.gc_polish_cover.isChecked()

        # Count Pages tab
        countpagesstats = []
@@ -308,10 +325,15 @@ class ConfigWidget(QWidget):
            colsnewonly[col] = checkbox.isChecked()
        prefs['std_cols_newonly'] = colsnewonly

+        prefs['set_author_url'] =self.std_columns_tab.set_author_url.isChecked()
+
        # Custom Columns tab
        # error column
        prefs['errorcol'] = unicode(convert_qvariant(self.cust_columns_tab.errorcol.itemData(self.cust_columns_tab.errorcol.currentIndex())))

+        # metadata column
+        prefs['savemetacol'] = unicode(convert_qvariant(self.cust_columns_tab.savemetacol.itemData(self.cust_columns_tab.savemetacol.currentIndex())))
+
        # cust cols tab
        colsmap = {}
        for (col,combo) in self.cust_columns_tab.custcol_dropdowns.iteritems():
@@ -328,12 +350,14 @@ class ConfigWidget(QWidget):
        
        prefs['allow_custcol_from_ini'] = self.cust_columns_tab.allow_custcol_from_ini.isChecked()

-        prefs['imapserver'] = unicode(self.imap_tab.imapserver.text())
-        prefs['imapuser'] = unicode(self.imap_tab.imapuser.text())
-        prefs['imappass'] = unicode(self.imap_tab.imappass.text())
-        prefs['imapfolder'] = unicode(self.imap_tab.imapfolder.text())
+        prefs['imapserver'] = unicode(self.imap_tab.imapserver.text()).strip()
+        prefs['imapuser'] = unicode(self.imap_tab.imapuser.text()).strip()
+        prefs['imappass'] = unicode(self.imap_tab.imappass.text()).strip()
+        prefs['imapfolder'] = unicode(self.imap_tab.imapfolder.text()).strip()
        prefs['imapmarkread'] = self.imap_tab.imapmarkread.isChecked()
        prefs['imapsessionpass'] = self.imap_tab.imapsessionpass.isChecked()
+        prefs['auto_reject_from_email'] = self.imap_tab.auto_reject_from_email.isChecked()
+        prefs['download_from_email_immediately'] = self.imap_tab.download_from_email_immediately.isChecked()
        
        prefs.save_to_db()

@@ -396,21 +420,17 @@ class BasicTab(QWidget):
        horz.addWidget(self.collision)
        self.l.addLayout(horz)

+        horz = QHBoxLayout()
        self.updatemeta = QCheckBox(_('Default Update Calibre &Metadata?'),self)
        self.updatemeta.setToolTip(_("On each download, FanFicFare offers an option to update Calibre's metadata (title, author, URL, tags, custom columns, etc) from the web site. <br />This sets whether that will default to on or off. <br />Columns set to 'New Only' in the column tabs will only be set for new books."))
        self.updatemeta.setChecked(prefs['updatemeta'])
-        self.l.addWidget(self.updatemeta)
+        horz.addWidget(self.updatemeta)

        self.updateepubcover = QCheckBox(_('Default Update EPUB Cover when Updating EPUB?'),self)
        self.updateepubcover.setToolTip(_("On each download, FanFicFare offers an option to update the book cover image <i>inside</i> the EPUB from the web site when the EPUB is updated.<br />This sets whether that will default to on or off."))
        self.updateepubcover.setChecked(prefs['updateepubcover'])
-        self.l.addWidget(self.updateepubcover)
-
-        self.smarten_punctuation = QCheckBox(_('Smarten Punctuation (EPUB only)'),self)
-        self.smarten_punctuation.setToolTip(_("Run Smarten Punctuation from Calibre's Polish Book feature on each EPUB download and update."))
-        self.smarten_punctuation.setChecked(prefs['smarten_punctuation'])
-        if calibre_version >= (0, 9, 39):
-            self.l.addWidget(self.smarten_punctuation)
+        horz.addWidget(self.updateepubcover)
+        self.l.addLayout(horz)

        cali_gb = groupbox = QGroupBox(_("Updating Calibre Options"))
        self.l = QVBoxLayout()
@@ -421,11 +441,6 @@ class BasicTab(QWidget):
        self.deleteotherforms.setChecked(prefs['deleteotherforms'])
        self.l.addWidget(self.deleteotherforms)
        
-        self.updatecover = QCheckBox(_('Update Calibre Cover when Updating Metadata?'),self)
-        self.updatecover.setToolTip(_("Update calibre book cover image from EPUB when metadata is updated.  (EPUB only.)\nDoesn't go looking for new images on 'Update Calibre Metadata Only'."))
-        self.updatecover.setChecked(prefs['updatecover'])
-        self.l.addWidget(self.updatecover)
-
        self.keeptags = QCheckBox(_('Keep Existing Tags when Updating Metadata?'),self)
        self.keeptags.setToolTip(_("Existing tags will be kept and any new tags added.\n%(cmplt)s and %(inprog)s tags will be still be updated, if known.\n%(lul)s tags will be updated if %(lus)s in %(is)s.\n(If Tags is set to 'New Only' in the Standard Columns tab, this has no effect.)")%no_trans)
        self.keeptags.setChecked(prefs['keeptags'])
@@ -466,16 +481,16 @@ class BasicTab(QWidget):
        self.showmarked.setChecked(prefs['showmarked'])
        self.l.addWidget(self.showmarked)

+        self.smarten_punctuation = QCheckBox(_('Smarten Punctuation (EPUB only)'),self)
+        self.smarten_punctuation.setToolTip(_("Run Smarten Punctuation from Calibre's Polish Book feature on each EPUB download and update."))
+        self.smarten_punctuation.setChecked(prefs['smarten_punctuation'])
+        self.l.addWidget(self.smarten_punctuation)
+
        self.autoconvert = QCheckBox(_("Automatically Convert new/update books?"),self)
        self.autoconvert.setToolTip(_("Automatically call calibre's Convert for new/update books.\nConverts to the current output format as chosen in calibre's\nPreferences->Behavior settings."))
        self.autoconvert.setChecked(prefs['autoconvert'])
        self.l.addWidget(self.autoconvert)

-        self.show_est_time = QCheckBox(_("Show estimated time left?"),self)
-        self.show_est_time.setToolTip(_("When a Progress Bar is shown, show a rough estimate of the time left."))
-        self.show_est_time.setChecked(prefs['show_est_time'])
-        self.l.addWidget(self.show_est_time)
-
        gui_gb = groupbox = QGroupBox(_("GUI Options"))
        self.l = QVBoxLayout()
        groupbox.setLayout(self.l)
@@ -495,21 +510,25 @@ class BasicTab(QWidget):
        self.adddialogstaysontop.setChecked(prefs['adddialogstaysontop'])
        self.l.addWidget(self.adddialogstaysontop)

+        self.show_est_time = QCheckBox(_("Show estimated time left?"),self)
+        self.show_est_time.setToolTip(_("When a Progress Bar is shown, show a rough estimate of the time left."))
+        self.show_est_time.setChecked(prefs['show_est_time'])
+        self.l.addWidget(self.show_est_time)
+
        misc_gb = groupbox = QGroupBox(_("Misc Options"))
        self.l = QVBoxLayout()
        groupbox.setLayout(self.l)

-        # this is a cheat to make it easier for users to realize there's a new include_images features.
-        self.includeimages = QCheckBox(_("Include images in EPUBs?"),self)
-        self.includeimages.setToolTip(_("Download and include images in EPUB stories.  This is equivalent to adding:%(imgset)s ...to the top of %(pini)s.  Your settings in %(pini)s will override this.")%no_trans)
-        self.includeimages.setChecked(prefs['includeimages'])
-        self.l.addWidget(self.includeimages)
-
        self.injectseries = QCheckBox(_("Inject calibre Series when none found?"),self)
        self.injectseries.setToolTip(_("If no series is found, inject the calibre series (if there is one) so \nit appears on the FanFicFare title page(not cover)."))
        self.injectseries.setChecked(prefs['injectseries'])
        self.l.addWidget(self.injectseries)

+        self.matchtitleauth = QCheckBox(_("Search by Title/Author(s) for If Story Already Exists?"),self)
+        self.matchtitleauth.setToolTip(_("When checking <i>If Story Already Exists</i> FanFicFare will first match by URL Identifier.  But if not found, it can also search existing books by Title and Author(s)."))
+        self.matchtitleauth.setChecked(prefs['matchtitleauth'])
+        self.l.addWidget(self.matchtitleauth)
+
        rej_gb = groupbox = QGroupBox(_("Reject List"))
        self.l = QVBoxLayout()
        groupbox.setLayout(self.l)
@@ -639,15 +658,25 @@ class PersonalIniTab(QWidget):
        self.ini_button.clicked.connect(self.add_ini_button)
        self.l.addWidget(self.ini_button)
        
+        label = QLabel(_("Changes will only be saved if you click 'OK' to leave Customize FanFicFare."))
+        label.setWordWrap(True)
+        self.l.addWidget(label)
+        
        self.defaults = QPushButton(_('View Defaults')+' (plugin-defaults.ini)', self)
        self.defaults.setToolTip(_("View all of the plugin's configurable settings\nand their default settings."))
        self.defaults.clicked.connect(self.show_defaults)
        self.l.addWidget(self.defaults)

-        label = QLabel(_("Changes will only be saved if you click 'OK' to leave Customize FanFicFare."))
-        label.setWordWrap(True)
-        self.l.addWidget(label)
+        self.cal_cols_pass_in = QCheckBox(_('Pass Calibre Columns into FanFicFare on Update/Overwrite')%no_trans,self)
+        self.cal_cols_pass_in.setToolTip(_("If checked, when updating/overwriting an existing book, FanFicFare will have the Calibre Columns available to use in replace_metadata, title_page, etc.<br>Click the button below to see the Calibre Column namess.")%no_trans)
+        self.cal_cols_pass_in.setChecked(prefs['cal_cols_pass_in'])
+        self.l.addWidget(self.cal_cols_pass_in)
        
+        self.showcalcols = QPushButton(_('Show Calibre Column Names'), self)
+        self.showcalcols.setToolTip(_("FanFicFare can pass the Calibre Columns into the download/update process.<br>This will show you the columns available by name."))
+        self.showcalcols.clicked.connect(self.show_showcalcols)
+        self.l.addWidget(self.showcalcols)
+
        self.l.insertStretch(-1)
        # let edit box fill the space.
        
@@ -673,6 +702,27 @@ class PersonalIniTab(QWidget):
        if d.result() == d.Accepted:
            self.personalini = d.get_plain_text()
                
+    def show_showcalcols(self):
+        lines=[]#[('calibre_std_user_categories',_('User Categories'))]
+        for k,f in field_metadata.iteritems():
+            if f['name'] and k not in STD_COLS_SKIP: # only if it has a human readable name.
+                lines.append(('calibre_std_'+k,f['name']))
+            
+        for k, column in self.plugin_action.gui.library_view.model().custom_columns.iteritems():
+            if k != prefs['savemetacol']:
+                # custom always have name.
+                lines.append(('calibre_cust_'+k[1:],column['name']))
+
+        lines.sort() # sort by key.
+
+        EditTextDialog(self,
+                       '\n'.join(['%s (%s)'%(l,k) for (k,l) in lines]),
+                       icon=self.windowIcon(),
+                       title=_('Calibre Column Entry Names'),
+                       label=_('Label (entry_name)'),
+                       read_only=True,
+                       save_size_name='fff:showcalcols').exec_()
+        
 class ReadingListTab(QWidget):

    def __init__(self, parent_dialog, plugin_action):
@@ -708,6 +758,7 @@ class ReadingListTab(QWidget):
        self.send_lists_box.update_items_cache(reading_lists)
        self.send_lists_box.setText(prefs['send_lists'])
        horz.addWidget(self.send_lists_box)
+        self.send_lists_box.setCursorPosition(0)
        self.l.addLayout(horz)
        
        self.addtoreadlists = QCheckBox(_('Add new/updated stories to "To Read" Reading List(s).'),self)
@@ -724,6 +775,7 @@ class ReadingListTab(QWidget):
        self.read_lists_box.update_items_cache(reading_lists)
        self.read_lists_box.setText(prefs['read_lists'])
        horz.addWidget(self.read_lists_box)
+        self.read_lists_box.setCursorPosition(0)
        self.l.addLayout(horz)
        
        self.addtolistsonread = QCheckBox(_('Add stories back to "Send to Device" Reading List(s) when marked "Read".'),self)
@@ -731,16 +783,26 @@ class ReadingListTab(QWidget):
        self.addtolistsonread.setChecked(prefs['addtolistsonread'])
        self.l.addWidget(self.addtolistsonread)
            
+        self.autounnew = QCheckBox(_('Automatically run Remove "New" Chapter Marks when marking books "Read".'),self)
+        self.autounnew.setToolTip(_('Menu option to remove from "To Read" lists will also remove "(new)" chapter marks created by personal.ini <i>mark_new_chapters</i> setting.'))
+        self.autounnew.setChecked(prefs['autounnew'])
+        self.l.addWidget(self.autounnew)
+            
        self.l.insertStretch(-1)
        
-class GenerateCoverTab(QWidget):
+class CalibreCoverTab(QWidget):

    def __init__(self, parent_dialog, plugin_action):
        self.parent_dialog = parent_dialog
        self.plugin_action = plugin_action
        QWidget.__init__(self)
        
-        self.l = QVBoxLayout()
+        self.gencov_elements=[] ## used to disable/enable when gen
+                                ## cover is off/on.  This is more
+                                ## about being a visual que than real
+                                ## necessary function.
+        
+        topl = self.l = QVBoxLayout()
        self.setLayout(self.l)

        try:
@@ -748,8 +810,116 @@ class GenerateCoverTab(QWidget):
            gc_settings = gc_plugin.get_saved_setting_names()
        except KeyError:
            gc_settings= []
+
            
-        label = QLabel(_('The %(gc)s plugin can create cover images for books using various metadata and configurations.  If you have GC installed, FanFicFare can run GC on new downloads and metadata updates.  Pick a GC setting by site or Default.')%no_trans)
+        label = QLabel(_("The Calibre cover image for a downloaded book can come"
+                         " from the story site(if EPUB and images are enabled), or"
+                         " from either Calibre's built-in random cover generator or"
+                         " the %(gc)s plugin.")%no_trans)
+        label.setWordWrap(True)
+        self.l.addWidget(label)
+        self.l.addSpacing(5)
+        
+        tooltip = _("Update Calibre book cover image from EPUB when Calibre metadata is updated.\n"
+                    "Doesn't go looking for new images on 'Update Calibre Metadata Only'.\n"
+                    "Cover in EPUB could be from site or previously injected into the EPUB.\n"
+                    "This comes before Generate Cover so %(gc)s(Plugin) use the image if configured to.")%no_trans
+        horz = QHBoxLayout()
+        label = QLabel(_('Update Calibre Cover (from EPUB):'))
+        label.setToolTip(tooltip)
+        horz.addWidget(label)
+        self.updatecalcover = QComboBox(self)
+        for i in updatecalcover_order:
+            self.updatecalcover.addItem(i)
+        # back compat.  If has own value, use.
+        if prefs['updatecalcover']:
+            self.updatecalcover.setCurrentIndex(self.updatecalcover.findText(calcover_save_options[prefs['updatecalcover']]))
+        elif prefs['updatecover']: # doesn't have own val, set YES if old value set.
+            self.updatecalcover.setCurrentIndex(self.updatecalcover.findText(calcover_save_options[SAVE_YES]))
+        else: # doesn't have own value, old value not set, NO.
+            self.updatecalcover.setCurrentIndex(self.updatecalcover.findText(calcover_save_options[SAVE_NO]))
+        self.updatecalcover.setToolTip(tooltip)
+        label.setBuddy(self.updatecalcover)
+        horz.addWidget(self.updatecalcover)
+        self.l.addLayout(horz)
+
+        tooltip = _("Generate a Calibre book cover image when Calibre metadata is updated.<br />"
+                    "Defaults to 'Yes, Always' for backward compatibility and because %(gc)s(Plugin)"
+                    " will only run if configured for Default or site.")%no_trans
+        horz = QHBoxLayout()
+        label = QLabel(_('Generate Calibre Cover:'))
+        label.setToolTip(tooltip)
+        horz.addWidget(label)
+        self.gencalcover = QComboBox(self)
+        for i in gencalcover_order:
+            self.gencalcover.addItem(i)
+        # back compat.  If has own value, use.
+        # if prefs['gencalcover']:
+        self.gencalcover.setCurrentIndex(self.gencalcover.findText(calcover_save_options[prefs['gencalcover']]))
+        # elif prefs['gencover']: # doesn't have own val, set YES if old value set.
+        #     self.gencalcover.setCurrentIndex(self.gencalcover.findText(calcover_save_options[SAVE_YES]))
+        # else: # doesn't have own value, old value not set, NO.
+        #     self.gencalcover.setCurrentIndex(self.gencalcover.findText(calcover_save_options[SAVE_NO]))
+            
+        self.gencalcover.setToolTip(tooltip)
+        label.setBuddy(self.gencalcover)
+        horz.addWidget(self.gencalcover)
+        self.l.addLayout(horz)
+        self.gencalcover.currentIndexChanged.connect(self.endisable_elements)
+
+        # can't be local or it's destroyed when __init__ is done and
+        # connected things don't fire.
+        self.gencov_rdgrp = QButtonGroup()
+        self.gencov_gb = QGroupBox()
+        horz = QHBoxLayout()
+        self.gencov_gb.setLayout(horz)
+        
+        self.plugin_gen_cover = QRadioButton(_('Plugin %(gc)s')%no_trans,self)
+        self.plugin_gen_cover.setToolTip(_("Use plugin to create covers.  Additional settings are below."))
+        self.gencov_rdgrp.addButton(self.plugin_gen_cover)
+        # always, new only, when no cover from site, inject yes/no...
+        self.plugin_gen_cover.setChecked(prefs['plugin_gen_cover'])
+        horz.addWidget(self.plugin_gen_cover)
+        self.gencov_elements.append(self.plugin_gen_cover)
+
+        self.calibre_gen_cover = QRadioButton(_('Calibre Generate Cover'),self)
+        self.calibre_gen_cover.setToolTip(_("Call Calibre's Edit Metadata Generate cover"
+                                            " feature to create a random cover each time"
+                                            " a story is downloaded or updated.<br />"
+                                            "Right click or long click the 'Generate cover'"
+                                            " button in Calibre's Edit Metadata to customize."))
+        self.gencov_rdgrp.addButton(self.calibre_gen_cover)
+        # always, new only, when no cover from site, inject yes/no...
+        self.calibre_gen_cover.setChecked(prefs['calibre_gen_cover'])
+        horz.addWidget(self.calibre_gen_cover)
+        self.gencov_elements.append(self.calibre_gen_cover)
+
+        #self.l.addLayout(horz)
+        self.l.addWidget(self.gencov_gb)
+
+        self.gcnewonly = QCheckBox(_("Generate Covers Only for New Books")%no_trans,self)
+        self.gcnewonly.setToolTip(_("Default is to generate a cover any time the calibre metadata is"
+                                    " updated.<br \>Used for both Calibre and Plugin generated covers."))
+        self.gcnewonly.setChecked(prefs['gcnewonly'])
+        self.l.addWidget(self.gcnewonly)
+        self.gencov_elements.append(self.gcnewonly)
+
+        self.gc_polish_cover = QCheckBox(_("Inject/update the cover inside EPUB"),self)
+        self.gc_polish_cover.setToolTip(_("Calibre's Polish feature will be used to inject or update the generated"
+                                          " cover into the EPUB ebook file.<br \>Used for both Calibre and Plugin generated covers."))
+        self.gc_polish_cover.setChecked(prefs['gc_polish_cover'])
+        self.l.addWidget(self.gc_polish_cover)
+        self.gencov_elements.append(self.gc_polish_cover)
+
+        self.gcp_gb = QGroupBox(_("%(gc)s(Plugin) Settings")%no_trans)
+        topl.addWidget(self.gcp_gb)
+        self.l = QVBoxLayout()
+        self.gcp_gb.setLayout(self.l)
+        self.gencov_elements.append(self.gcp_gb)
+
+        self.gencov_rdgrp.buttonClicked.connect(self.endisable_elements)
+            
+        label = QLabel(_('The %(gc)s plugin can create cover images for books using various metadata (including existing cover image).  If you have %(gc)s installed, FanFicFare can run %(gc)s on new downloads and metadata updates.  Pick a %(gc)s setting by site and/or one to use by Default.')%no_trans)
        label.setWordWrap(True)
        self.l.addWidget(label)
        self.l.addSpacing(5)
@@ -765,14 +935,14 @@ class GenerateCoverTab(QWidget):
        
        self.gc_dropdowns = {}

-        sitelist = getConfigSections()
+        sitelist = getSiteSections()
        sitelist.sort()
        sitelist.insert(0,_("Default"))
        for site in sitelist:
            horz = QHBoxLayout()
            label = QLabel(site)
            if site == _("Default"):
-                s = _("On Metadata update, run %(gc)s with this setting, if not selected for specific site.")%no_trans
+                s = _("On Metadata update, run %(gc)s with this setting, if there isn't a more specific setting below.")%no_trans
            else:
                no_trans['site']=site # not ideal, but, meh.
                s = _("On Metadata update, run %(gc)s with this setting for %(site)s stories.")%no_trans
@@ -798,20 +968,34 @@ class GenerateCoverTab(QWidget):
        
        self.sl.insertStretch(-1)
        
-        self.gcnewonly = QCheckBox(_("Run %(gc)s Only on New Books")%no_trans,self)
-        self.gcnewonly.setToolTip(_("Default is to run GC any time the calibre metadata is updated."))
-        self.gcnewonly.setChecked(prefs['gcnewonly'])
-        self.l.addWidget(self.gcnewonly)
-
        self.allow_gc_from_ini = QCheckBox(_('Allow %(gcset)s from %(pini)s to override')%no_trans,self)
-        self.allow_gc_from_ini.setToolTip(_("The %(pini)s parameter %(gcset)s allows you to choose a GC setting based on metadata rather than site, but it's much more complex.<br \>%(gcset)s is ignored when this is off.")%no_trans)
+        self.allow_gc_from_ini.setToolTip(_("The %(pini)s parameter %(gcset)s allows you to choose a %(gc)s setting based on metadata"
+                                            " rather than site, but it's much more complex.<br \>%(gcset)s is ignored when this is off.")%no_trans)
        self.allow_gc_from_ini.setChecked(prefs['allow_gc_from_ini'])
        self.l.addWidget(self.allow_gc_from_ini)
-            
-        self.gc_polish_cover = QCheckBox(_("Use calibre's Polish feature to inject/update the cover"),self)
-        self.gc_polish_cover.setToolTip(_("Calibre's Polish feature will be used to inject or update the generated cover into the ebook, EPUB only."))
-        self.gc_polish_cover.setChecked(prefs['gc_polish_cover'])
-        self.l.addWidget(self.gc_polish_cover)
+
+        # keep at end.
+        self.endisable_elements()
+
+    def endisable_elements(self,button=None):
+        "Clearing house function for setting elements of Calibre"
+        "Cover tab enabled/disabled depending on all factors."
+
+        ## First, cover gen on/off 
+        for e in self.gencov_elements:
+            e.setEnabled(calcover_save_options[unicode(self.gencalcover.currentText())] != SAVE_NO)
+
+        # next, disable plugin settings when using calibre gen cov.
+        if not self.plugin_gen_cover.isChecked():
+            self.gcp_gb.setEnabled(False)
+
+        # disable (but not enable) unsupported options.
+        if not HAS_CALGC:
+            self.calibre_gen_cover.setEnabled(False)
+        if not 'Generate Cover' in self.plugin_action.gui.iactions:
+            self.plugin_gen_cover.setEnabled(False)
+            self.gcp_gb.setEnabled(False)
+        
            
 class CountPagesTab(QWidget):

@@ -1061,7 +1245,6 @@ class CustomColumnsTab(QWidget):
        self.allow_custcol_from_ini.setChecked(prefs['allow_custcol_from_ini'])
        self.l.addWidget(self.allow_custcol_from_ini)
        
-        self.l.addSpacing(5)
        label = QLabel(_("Special column:"))
        label.setWordWrap(True)
        self.l.addWidget(label)
@@ -1080,6 +1263,21 @@ class CustomColumnsTab(QWidget):
        self.errorcol.setCurrentIndex(self.errorcol.findData(prefs['errorcol']))
        horz.addWidget(self.errorcol)
        self.l.addLayout(horz)
+        
+        horz = QHBoxLayout()
+        label = QLabel(_("Saved Metadata Column:"))
+        tooltip=_("If set, FanFicFare will save a copy of all its metadata in this column when the book is downloaded or updated.<br/>The metadata from this column can later be used to update custom columns without having to request the metadata from the server again.<br/>(Long Text columns only.)")
+        label.setToolTip(tooltip)
+        horz.addWidget(label)
+        self.savemetacol = QComboBox(self)
+        self.savemetacol.setToolTip(tooltip)
+        self.savemetacol.addItem('','')
+        for key, column in custom_columns.iteritems():
+            if column['datatype'] in ('comments'):
+                self.savemetacol.addItem(column['name'],key)
+        self.savemetacol.setCurrentIndex(self.savemetacol.findData(prefs['savemetacol']))
+        horz.addWidget(self.savemetacol)
+        self.l.addLayout(horz)

        #print("prefs['custom_cols'] %s"%prefs['custom_cols'])

@@ -1128,6 +1326,17 @@ class StandardColumnsTab(QWidget):
            horz.addWidget(newonlycheck)
            
            self.l.addLayout(horz)
+            
+        self.l.addSpacing(5)
+        label = QLabel(_("Other Standard Column Options"))
+        label.setWordWrap(True)
+        self.l.addWidget(label)
+        self.l.addSpacing(5)
+
+        self.set_author_url = QCheckBox(_('Set Calibre Author URL'),self)
+        self.set_author_url.setToolTip(_("Set Calibre Author URL to Author's URL on story site."))
+        self.set_author_url.setChecked(prefs['set_author_url'])
+        self.l.addWidget(self.set_author_url)
        
        self.l.insertStretch(-1)

@@ -1200,6 +1409,18 @@ class ImapTab(QWidget):
        self.l.addWidget(self.imapmarkread,row,0,1,-1)
        row+=1
            
+        self.auto_reject_from_email = QCheckBox(_('Discard URLs on Reject List'),self)
+        self.auto_reject_from_email.setToolTip(_('If checked, FanFicFare will silently discard story URLs from emails that are on your Reject URL List.<br>Otherwise they will appear and you will see the normal Reject URL dialog.<br>The Emails will still be marked Read if configured to.'))
+        self.auto_reject_from_email.setChecked(prefs['auto_reject_from_email'])
+        self.l.addWidget(self.auto_reject_from_email,row,0,1,-1)
+        row+=1
+        
+        self.download_from_email_immediately = QCheckBox(_('Download from Email Immediately'),self)
+        self.download_from_email_immediately.setToolTip(_('If checked, FanFicFare will start downloading story URLs from emails immediately.<br>Otherwise the usual Download from URLs dialog will appear.'))
+        self.download_from_email_immediately.setChecked(prefs['download_from_email_immediately'])
+        self.l.addWidget(self.download_from_email_immediately,row,0,1,-1)
+        row+=1
+        
        label = QLabel(_("<b>It's safest if you create a separate email account that you use only "
                         "for your story update notices.  FanFicFare and calibre cannot guarantee that "
                         "malicious code cannot get your email password once you've entered it. "
@@ -1,5 +1,5 @@
-#!/usr/bin/env python
-# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+# -*- coding: utf-8 -*-
+
 from __future__ import (unicode_literals, division,
                        print_function)

@@ -79,14 +79,16 @@ UPDATE=_('Update EPUB if New Chapters')
 UPDATEALWAYS=_('Update EPUB Always')
 OVERWRITE=_('Overwrite if Newer')
 OVERWRITEALWAYS=_('Overwrite Always')
-CALIBREONLY=_('Update Calibre Metadata Only')
+CALIBREONLY=_('Update Calibre Metadata from Web Site')
+CALIBREONLYSAVECOL=_('Update Calibre Metadata from Saved Metadata Column')
 collision_order=[SKIP,
                 ADDNEW,
                 UPDATE,
                 UPDATEALWAYS,
                 OVERWRITE,
                 OVERWRITEALWAYS,
-                 CALIBREONLY,]
+                 CALIBREONLY,
+                 CALIBREONLYSAVECOL,]

 # best idea I've had for how to deal with config/pref saving the
 # collision name in english.
@@ -97,6 +99,7 @@ SAVE_UPDATEALWAYS='Update EPUB Always'
 SAVE_OVERWRITE='Overwrite if Newer'
 SAVE_OVERWRITEALWAYS='Overwrite Always'
 SAVE_CALIBREONLY='Update Calibre Metadata Only'
+SAVE_CALIBREONLYSAVECOL='Update Calibre Metadata Only(Saved Column)'
 save_collisions={
    SKIP:SAVE_SKIP,
    ADDNEW:SAVE_ADDNEW,
@@ -112,8 +115,9 @@ save_collisions={
    SAVE_OVERWRITE:OVERWRITE,
    SAVE_OVERWRITEALWAYS:OVERWRITEALWAYS,
    SAVE_CALIBREONLY:CALIBREONLY,
+    SAVE_CALIBREONLYSAVECOL:CALIBREONLYSAVECOL,
    }
-    
+
 anthology_collision_order=[UPDATE,
                           UPDATEALWAYS,
                           OVERWRITEALWAYS]
@@ -122,8 +126,8 @@ gpstyle='QGroupBox {border:0; padding-top:10px; padding-bottom:0px; margin-botto

 class RejectUrlEntry:

-    matchpat=re.compile(r"^(?P<url>[^,]+)(,(?P<fullnote>(((?P<title>.+) by (?P<auth>.+?)( - (?P<note>.+))?)|.*)))?$")
-    
+    matchpat=re.compile(r"^(?P<url>[^,]+?)(,(?P<fullnote>(((?P<title>.+?) by (?P<auth>.+?)( - (?P<note>.+))?)|.*)))?$")
+
    def __init__(self,url_or_line,note=None,title=None,auth=None,
                 addreasontext=None,fromline=False,book_id=None):

@@ -147,7 +151,7 @@ class RejectUrlEntry:
                self.note=note
                self.title=title
                self.auth=auth
-        
+
        if not self.note:
            if addreasontext:
                self.note = addreasontext
@@ -156,14 +160,14 @@ class RejectUrlEntry:
        else:
            if addreasontext:
                self.note = self.note + ' - ' + addreasontext
-                
+
        self.url = getNormalStoryURL(self.url)
        self.valid = self.url != None
-                
+
    def to_line(self):
        # always 'url,'
        return self.url+","+self.fullnote()
-        
+
    def fullnote(self):
        retval = ""
        if self.title and self.auth:
@@ -171,10 +175,10 @@ class RejectUrlEntry:
            retval = retval + "%s by %s"%(self.title,self.auth)
            if self.note:
                retval = retval + " - "
-                
+
        if self.note:
            retval = retval + self.note
-            
+
        return retval

 class NotGoingToDownload(Exception):
@@ -218,7 +222,7 @@ class DroppableQTextEdit(QTextEdit):
            self.append("\n".join(urllist))
            return None
        return QTextEdit.dropEvent(self,event)
-        
+
    def canInsertFromMimeData(self, source):
        if source.hasUrls():
            return True
@@ -230,7 +234,7 @@ class DroppableQTextEdit(QTextEdit):
            self.append(source.text())
        else:
            return QTextEdit.insertFromMimeData(self, source)
-                            
+
 class AddNewDialog(SizePersistedDialog):

    go_signal = pyqtSignal(object, object, object, object)
@@ -238,7 +242,7 @@ class AddNewDialog(SizePersistedDialog):
    def __init__(self, gui, prefs, icon):
        SizePersistedDialog.__init__(self, gui, 'fff:add new dialog')
        self.prefs = prefs
-        
+
        self.setMinimumWidth(300)
        self.l = QVBoxLayout()
        self.setLayout(self.l)
@@ -254,7 +258,8 @@ class AddNewDialog(SizePersistedDialog):
        self.l.addWidget(self.url)

        self.merge = self.newmerge = False
-        
+        self.extraoptions = {}
+
        # elements to hide when doing merge.
        self.mergehide = []
        # elements to show again when doing *update* merge
@@ -277,11 +282,11 @@ class AddNewDialog(SizePersistedDialog):

        self.gbf.setVisible(False)
        self.groupbox.toggled.connect(self.gbf.setVisible)
-        
+
        horz = QHBoxLayout()
        label = QLabel(_('Output &Format:'))
        self.mergehide.append(label)
-        
+
        self.fileform = QComboBox(self)
        self.fileform.addItem('epub')
        self.fileform.addItem('mobi')
@@ -289,7 +294,7 @@ class AddNewDialog(SizePersistedDialog):
        self.fileform.addItem('txt')
        self.fileform.setToolTip(_('Choose output format to create.  May set default from plugin configuration.'))
        self.fileform.activated.connect(self.set_collisions)
-        
+
        horz.addWidget(label)
        label.setBuddy(self.fileform)
        horz.addWidget(self.fileform)
@@ -303,7 +308,7 @@ class AddNewDialog(SizePersistedDialog):
        self.collision.setToolTip("CollisionToolTip")
        # add collision options
        self.set_collisions()
-        i = self.collision.findText(save_collisions[prefs['collision']])
+        i = self.collision.findText(save_collisions[self.prefs['collision']])
        if i > -1:
            self.collision.setCurrentIndex(i)
        self.collisionlabel.setBuddy(self.collision)
@@ -317,17 +322,17 @@ class AddNewDialog(SizePersistedDialog):
        horz = QHBoxLayout()
        self.updatemeta = QCheckBox(_('Update Calibre &Metadata?'),self)
        self.updatemeta.setToolTip(_("Update metadata for existing stories in Calibre from web site?\n(Columns set to 'New Only' in the column tabs will only be set for new books.)"))
-        self.updatemeta.setChecked(prefs['updatemeta'])
+        self.updatemeta.setChecked(self.prefs['updatemeta'])
        horz.addWidget(self.updatemeta)
        self.mergehide.append(self.updatemeta)
        self.mergeupdateshow.append(self.updatemeta)

        self.updateepubcover = QCheckBox(_('Update EPUB Cover?'),self)
        self.updateepubcover.setToolTip(_('Update book cover image from site or defaults (if found) <i>inside</i> the EPUB when EPUB is updated.'))
-        self.updateepubcover.setChecked(prefs['updateepubcover'])
+        self.updateepubcover.setChecked(self.prefs['updateepubcover'])
        horz.addWidget(self.updateepubcover)
        self.mergehide.append(self.updateepubcover)
-        
+
        self.gbl.addLayout(horz)

        self.button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
@@ -335,7 +340,7 @@ class AddNewDialog(SizePersistedDialog):
        self.button_box.rejected.connect(self.reject)
        self.l.addWidget(self.button_box)

-    # invoke the 
+    # invoke the
    def ok_clicked(self):
        self.dialog_closing(None) # save persistent size.
        self.hide()
@@ -354,7 +359,10 @@ class AddNewDialog(SizePersistedDialog):
                    extrapayload=None):
        # rather than mutex in fff_plugin, just bail here if it's
        # already in use.
-        if self.isVisible(): return
+        if self.isVisible():
+            if url_list_text: # add to open box.
+               self.url.setText( '\n'.join([self.get_urlstext(), url_list_text]) )
+            return

        try:
            self.go_signal.disconnect()
@@ -368,7 +376,7 @@ class AddNewDialog(SizePersistedDialog):
        self.extrapayload = extrapayload

        self.groupbox.setVisible(not(self.merge and self.newmerge))
-        
+
        if self.merge:
            self.toplabel.setText(_('Story URLs for anthology, one per line:'))
            self.url.setToolTip(_('URLs for stories to include in the anthology, one per line.\nWill take URLs from clipboard, but only valid URLs.'))
@@ -390,7 +398,7 @@ class AddNewDialog(SizePersistedDialog):
        # Need to re-able after hiding/showing
        self.setAcceptDrops(True)
        self.url.setFocus()
-            
+
        if self.prefs['adddialogstaysontop']:
            QDialog.setWindowFlags ( self, Qt.Dialog | Qt.WindowStaysOnTopHint )
        else:
@@ -407,18 +415,18 @@ class AddNewDialog(SizePersistedDialog):
        i = self.collision.findText(save_collisions[self.prefs['collision']])
        if i > -1:
            self.collision.setCurrentIndex(i)
-            
+
        self.updatemeta.setChecked(self.prefs['updatemeta'])
-            
+
        if not self.merge:
            self.updateepubcover.setChecked(self.prefs['updateepubcover'])
-                
+
        self.url.setText(url_list_text)
        if url_list_text:
            self.button_box.button(QDialogButtonBox.Ok).setFocus()
        # restore saved size.
        self.resize_dialog()
-        
+
        if show: # so anthology update can be modal still.
            self.show()
        #self.resize(self.sizeHint())
@@ -427,16 +435,23 @@ class AddNewDialog(SizePersistedDialog):
        prev=self.collision.currentText()
        self.collision.clear()
        if self.merge:
-            order = anthology_collision_order
+            order = list(anthology_collision_order)
        else:
-            order = collision_order
+            order = list(collision_order)
+            ## Remove options that aren't valid.
+            if self.fileform.currentText() != 'epub':
+                order.remove(UPDATE)
+                order.remove(UPDATEALWAYS)
+            if self.prefs['savemetacol'] == '':
+                order.remove(CALIBREONLYSAVECOL)
+
        for o in order:
-            if self.merge or self.fileform.currentText() == 'epub' or o not in [UPDATE,UPDATEALWAYS]:
-                self.collision.addItem(o)
+            self.collision.addItem(o)
+
        i = self.collision.findText(prev)
        if i > -1:
            self.collision.setCurrentIndex(i)
-        
+
    def get_fff_options(self):
        retval =  {
            'fileform': unicode(self.fileform.currentText()),
@@ -445,14 +460,14 @@ class AddNewDialog(SizePersistedDialog):
            'updateepubcover': self.updateepubcover.isChecked(),
            'smarten_punctuation':self.prefs['smarten_punctuation']
                }
-        
+
        if self.merge:
            retval['fileform']=='epub'
            retval['updateepubcover']=True
            if self.newmerge:
                retval['updatemeta']=True
                retval['collision']=ADDNEW
-            
+
        return dict(retval.items() + self.extraoptions.items() )

    def get_urlstext(self):
@@ -462,21 +477,21 @@ class AddNewDialog(SizePersistedDialog):
 class FakeLineEdit():
    def __init__(self):
        pass
-    
+
    def text(self):
        pass
-    
+
 class CollectURLDialog(SizePersistedDialog):
    '''
    Collect single url for get urls.
    '''
-    def __init__(self, gui, title, url_text, anthology=False, indiv=True): 
+    def __init__(self, gui, title, url_text, anthology=False, indiv=True):
        SizePersistedDialog.__init__(self, gui, 'fff:get story urls')
        self.status=False
        self.anthology=False

        self.setMinimumWidth(300)
-        
+
        self.l = QVBoxLayout()
        self.setLayout(self.l)

@@ -546,16 +561,16 @@ class UserPassDialog(QDialog):
        else:
            self.setWindowTitle(_('User/Password'))
            self.l.addWidget(QLabel(_("%s requires you to login to download this story.")%site),0,0,1,2)
-        
+
            self.l.addWidget(QLabel(_("User:")),1,0)
            self.user = QLineEdit(self)
            self.l.addWidget(self.user,1,1)
-   
+
        self.l.addWidget(QLabel(_("Password:")),2,0)
        self.passwd = QLineEdit(self)
        self.passwd.setEchoMode(QLineEdit.Password)
        self.l.addWidget(self.passwd,2,1)
-   
+
        self.ok_button = QPushButton(_('OK'), self)
        self.ok_button.clicked.connect(self.ok)
        self.l.addWidget(self.ok_button,3,0)
@@ -601,6 +616,9 @@ class LoopProgressDialog(QProgressDialog):
        from calibre_plugins.fanficfare_plugin.prefs import prefs
        self.show_est_time = prefs['show_est_time']

+        self.setLabelText('%s %d / %d' % (self.status_prefix, self.i, len(self.book_list)))
+        self.setValue(self.i)
+
        ## self.do_loop does QTimer.singleShot on self.do_loop also.
        ## A weird way to do a loop, but that was the example I had.
        QTimer.singleShot(0, self.do_loop)
@@ -627,7 +645,7 @@ class LoopProgressDialog(QProgressDialog):
            ## collision spec passed into getadapter by partial from fff_plugin
            ## no retval only if it exists, but collision is SKIP
            self.foreach_function(book)
-            
+
        except NotGoingToDownload as d:
            book['good']=False
            book['comment']=unicode(d)
@@ -638,10 +656,10 @@ class LoopProgressDialog(QProgressDialog):
            book['comment']=unicode(e)
            logger.error("Exception: %s:%s"%(book,unicode(e)))
            traceback.print_exc()
-            
+
        self.updateStatus()
        self.i += 1
-            
+
        if self.i >= len(self.book_list) or self.wasCanceled():
            return self.do_when_finished()
        else:
@@ -732,7 +750,7 @@ class UpdateExistingDialog(SizePersistedDialog):
        self.prefs = prefs
        self.setWindowTitle(header)
        self.setWindowIcon(icon)
-        
+
        layout = QVBoxLayout(self)
        self.setLayout(layout)
        title_layout = ImageTitleLayout(self, 'images/icon.png',
@@ -775,7 +793,7 @@ class UpdateExistingDialog(SizePersistedDialog):

        gbf.setVisible(False)
        groupbox.toggled.connect(gbf.setVisible)
-        
+
        label = QLabel(_('Output &Format:'))
        gbl.addWidget(label)
        self.fileform = QComboBox(self)
@@ -783,19 +801,19 @@ class UpdateExistingDialog(SizePersistedDialog):
        self.fileform.addItem('mobi')
        self.fileform.addItem('html')
        self.fileform.addItem('txt')
-        self.fileform.setCurrentIndex(self.fileform.findText(prefs['fileform']))
+        self.fileform.setCurrentIndex(self.fileform.findText(self.prefs['fileform']))
        self.fileform.setToolTip(_('Choose output format to create.  May set default from plugin configuration.'))
        self.fileform.activated.connect(self.set_collisions)
        label.setBuddy(self.fileform)
        gbl.addWidget(self.fileform)
-        
+
        label = QLabel(_('Update Mode:'))
        gbl.addWidget(label)
        self.collision = QComboBox(self)
        self.collision.setToolTip(_("What sort of update to perform.  May set default from plugin configuration."))
        # add collision options
        self.set_collisions()
-        i = self.collision.findText(save_collisions[prefs['collision']])
+        i = self.collision.findText(save_collisions[self.prefs['collision']])
        if i > -1:
            self.collision.setCurrentIndex(i)
        label.setBuddy(self.collision)
@@ -803,23 +821,23 @@ class UpdateExistingDialog(SizePersistedDialog):

        self.updatemeta = QCheckBox(_('Update Calibre &Metadata?'),self)
        self.updatemeta.setToolTip(_("Update metadata for existing stories in Calibre from web site?\n(Columns set to 'New Only' in the column tabs will only be set for new books.)"))
-        self.updatemeta.setChecked(prefs['updatemeta'])
+        self.updatemeta.setChecked(self.prefs['updatemeta'])
        gbl.addWidget(self.updatemeta)
-                
+
        self.updateepubcover = QCheckBox(_('Update EPUB Cover?'),self)
        self.updateepubcover.setToolTip(_('Update book cover image from site or defaults (if found) <i>inside</i> the EPUB when EPUB is updated.'))
-        self.updateepubcover.setChecked(prefs['updateepubcover'])
+        self.updateepubcover.setChecked(self.prefs['updateepubcover'])
        gbl.addWidget(self.updateepubcover)


-        
+
        button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
        button_box.accepted.connect(self.accept)
        button_box.rejected.connect(self.reject)
        options_layout.addWidget(button_box)
-        
+
        layout.addLayout(options_layout)
-        
+
        # Cause our dialog size to be restored from prefs or created on first usage
        self.resize_dialog()
        self.books_table.populate_table(books)
@@ -827,14 +845,22 @@ class UpdateExistingDialog(SizePersistedDialog):
    def set_collisions(self):
        prev=self.collision.currentText()
        self.collision.clear()
-        for o in collision_order:
-            if o not in [ADDNEW,SKIP] and \
-                    (self.fileform.currentText() == 'epub' or o not in [UPDATE,UPDATEALWAYS]):
-                self.collision.addItem(o)        
+        order = list(collision_order)
+        order.remove(ADDNEW)
+        order.remove(SKIP)
+        if self.fileform.currentText() != 'epub':
+            order.remove(UPDATE)
+            order.remove(UPDATEALWAYS)
+        if self.prefs['savemetacol'] == '':
+            order.remove(CALIBREONLYSAVECOL)
+
+        for o in order:
+            self.collision.addItem(o)
+
        i = self.collision.findText(prev)
        if i > -1:
            self.collision.setCurrentIndex(i)
-        
+
    def remove_from_list(self):
        self.books_table.remove_selected_rows()

@@ -901,16 +927,16 @@ class StoryListTableWidget(QTableWidget):
        status_cell = IconWidgetItem(None,icon,val)
        status_cell.setData(Qt.UserRole, val)
        self.setItem(row, 0, status_cell)
-        
+
        title_cell = ReadOnlyTableWidgetItem(book['title'])
        title_cell.setData(Qt.UserRole, row)
        self.setItem(row, 1, title_cell)
-        
+
        self.setItem(row, 2, AuthorTableWidgetItem(", ".join(book['author']), ", ".join(book['author_sort'])))
-        
+
        url_cell = ReadOnlyTableWidgetItem(book['url'])
        self.setItem(row, 3, url_cell)
-        
+
        comment_cell = ReadOnlyTableWidgetItem(book['comment'])
        self.setItem(row, 4, comment_cell)

@@ -988,16 +1014,17 @@ class RejectListTableWidget(QTableWidget):
        self.setItem(row, 0, url_cell)
        self.setItem(row, 1, ReadOnlyTableWidgetItem(rej.title))
        self.setItem(row, 2, ReadOnlyTableWidgetItem(rej.auth))
-        
+
        note_cell = EditWithComplete(self,sort_func=lambda x:1)
-        
+
        items = [rej.note]+self.rejectreasons
        note_cell.update_items_cache(items)
        note_cell.show_initial_value(rej.note)
        note_cell.set_separator(None)
        note_cell.setToolTip(_('Select or Edit Reject Note.'))
        self.setCellWidget(row, 3, note_cell)
-        
+        note_cell.setCursorPosition(0)
+
    def remove_selected_rows(self):
        self.setFocus()
        rows = self.selectionModel().selectedRows()
@@ -1029,10 +1056,10 @@ class RejectListDialog(SizePersistedDialog):
                 show_all_reasons=True,
                 save_size_name='fff:reject list dialog'):
        SizePersistedDialog.__init__(self, gui, save_size_name)
-      
+
        self.setWindowTitle(header)
        self.setWindowIcon(get_icon(icon))
-      
+
        layout = QVBoxLayout(self)
        self.setLayout(layout)
        title_layout = ImageTitleLayout(self, icon, header,
@@ -1048,7 +1075,7 @@ class RejectListDialog(SizePersistedDialog):
        rejects_layout.addLayout(button_layout)
        spacerItem = QtGui.QSpacerItem(20, 40, QtGui.QSizePolicy.Minimum, QtGui.QSizePolicy.Expanding)
        button_layout.addItem(spacerItem)
-        
+
        self.remove_button = QtGui.QToolButton(self)
        self.remove_button.setToolTip(_('Remove selected URLs from the list'))
        self.remove_button.setIcon(get_icon('list_remove.png'))
@@ -1060,21 +1087,22 @@ class RejectListDialog(SizePersistedDialog):

        if show_all_reasons:
            self.reason_edit = EditWithComplete(self,sort_func=lambda x:1)
-            
+
            items = ['']+rejectreasons
            self.reason_edit.update_items_cache(items)
            self.reason_edit.show_initial_value('')
            self.reason_edit.set_separator(None)
            self.reason_edit.setToolTip(_("This will be added to whatever note you've set for each URL above."))
-            
+
            horz = QHBoxLayout()
            label = QLabel(_("Add this reason to all URLs added:"))
            label.setToolTip(_("This will be added to whatever note you've set for each URL above."))
            horz.addWidget(label)
            horz.addWidget(self.reason_edit)
+            self.reason_edit.setCursorPosition(0)
            horz.insertStretch(-1)
            layout.addLayout(horz)
-                    
+
        options_layout = QHBoxLayout()

        if show_delete:
@@ -1082,14 +1110,14 @@ class RejectListDialog(SizePersistedDialog):
            self.deletebooks.setToolTip(_("Delete the selected books after adding them to the Rejected URLs list."))
            self.deletebooks.setChecked(True)
            options_layout.addWidget(self.deletebooks)
-        
+
        button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
        button_box.accepted.connect(self.accept)
        button_box.rejected.connect(self.reject)
        options_layout.addWidget(button_box)
-      
+
        layout.addLayout(options_layout)
-      
+
        # Cause our dialog size to be restored from prefs or created on first usage
        self.resize_dialog()
        self.rejects_table.populate_table(reject_list)
@@ -1122,7 +1150,7 @@ class RejectListDialog(SizePersistedDialog):
        except:
            # doesn't have self.reason_edit when editing existing list.
            return None
-    
+
    def get_deletebooks(self):
        return self.deletebooks.isChecked()

@@ -1130,6 +1158,7 @@ class EditTextDialog(SizePersistedDialog):

    def __init__(self, parent, text,
                 icon=None, title=None, label=None, tooltip=None,
+                 read_only=False,
                 rejectreasons=[],reasonslabel=None,
                 save_size_name='fff:edit text dialog',
                 ):
@@ -1143,9 +1172,10 @@ class EditTextDialog(SizePersistedDialog):
        if icon:
            self.setWindowIcon(icon)
        self.l.addWidget(self.label)
-        
+
        self.textedit = QTextEdit(self)
        self.textedit.setLineWrapMode(QTextEdit.NoWrap)
+        self.textedit.setReadOnly(read_only)
        self.textedit.setText(text)
        self.l.addWidget(self.textedit)

@@ -1155,13 +1185,13 @@ class EditTextDialog(SizePersistedDialog):

        if rejectreasons or reasonslabel:
            self.reason_edit = EditWithComplete(self,sort_func=lambda x:1)
-            
+
            items = ['']+rejectreasons
            self.reason_edit.update_items_cache(items)
            self.reason_edit.show_initial_value('')
            self.reason_edit.set_separator(None)
            self.reason_edit.setToolTip(reasonslabel)
-            
+
            if reasonslabel:
                horz = QHBoxLayout()
                label = QLabel(reasonslabel)
@@ -1171,12 +1201,13 @@ class EditTextDialog(SizePersistedDialog):
                self.l.addLayout(horz)
            else:
                self.l.addWidget(self.reason_edit)
-            
+            self.reason_edit.setCursorPosition(0)
+
        button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
        button_box.accepted.connect(self.accept)
        button_box.rejected.connect(self.reject)
        self.l.addWidget(button_box)
-        
+
        # Cause our dialog size to be restored from prefs or created on first usage
        self.resize_dialog()

@@ -1185,7 +1216,7 @@ class EditTextDialog(SizePersistedDialog):

    def get_reason_text(self):
        return unicode(self.reason_edit.currentText()).strip()
-    
+
 class IniTextDialog(SizePersistedDialog):

    def __init__(self, parent, text,
@@ -1195,9 +1226,9 @@ class IniTextDialog(SizePersistedDialog):
                 save_size_name='fff:ini text dialog',
                 ):
        SizePersistedDialog.__init__(self, parent, save_size_name)
-        
+
        self.keys=dict()
-        
+
        self.l = QVBoxLayout()
        self.setLayout(self.l)
        self.label = QLabel(label)
@@ -1206,7 +1237,7 @@ class IniTextDialog(SizePersistedDialog):
        if icon:
            self.setWindowIcon(icon)
        self.l.addWidget(self.label)
-        
+
        self.textedit = QTextEdit(self)

        highlighter = IniHighlighter(self.textedit,
@@ -1215,7 +1246,7 @@ class IniTextDialog(SizePersistedDialog):
                                     entries=get_valid_entries(),
                                     entry_keywords=get_valid_entry_keywords(),
                                     )
-        
+
        self.textedit.setLineWrapMode(QTextEdit.NoWrap)
        try:
            self.textedit.setFont(QFont("Courier",
@@ -1233,41 +1264,41 @@ class IniTextDialog(SizePersistedDialog):
        if use_find:

            findtooltip=_('Search for string in edit box.')
-            
+
            horz = QHBoxLayout()
            label = QLabel(_('Find:'))

            label.setToolTip(findtooltip)
-                
+
            # Button to search the document for something
            self.findButton = QtGui.QPushButton(_('Find'),self)
            self.findButton.clicked.connect(self.find)
            self.findButton.setToolTip(findtooltip)
-        
+
            # The field into which to type the query
            self.findField = QLineEdit(self)
            self.findField.setToolTip(findtooltip)
            self.findField.returnPressed.connect(self.findButton.setFocus)
-            
+
            # Case Sensitivity option
            self.caseSens = QtGui.QCheckBox(_('Case sensitive'),self)
            self.caseSens.setToolTip(_("Search for case sensitive string; don't treat Harry, HARRY and harry all the same."))
-            
+
            horz.addWidget(label)
            horz.addWidget(self.findField)
            horz.addWidget(self.findButton)
            horz.addWidget(self.caseSens)
-            
+
            self.l.addLayout(horz)

            self.addCtrlKeyPress(QtCore.Qt.Key_F,self.findFocus)
            self.addCtrlKeyPress(QtCore.Qt.Key_G,self.find)
-        
+
        button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
        button_box.accepted.connect(self.accept)
        button_box.rejected.connect(self.reject)
        self.l.addWidget(button_box)
-        
+
        # Cause our dialog size to be restored from prefs or created on first usage
        self.resize_dialog()

@@ -1283,9 +1314,9 @@ class IniTextDialog(SizePersistedDialog):
                        _('Go back to fix errors?'),
                        errors)
            retry = d.exec_() == d.Accepted
-            
+
            # print("retry:%s"%retry)
-        
+
        if retry:
            lineno=d.get_lineno()
            if lineno:
@@ -1307,19 +1338,19 @@ class IniTextDialog(SizePersistedDialog):
            return func()
        else:
            return SizePersistedDialog.keyPressEvent(self, event)
-    
+
    def get_plain_text(self):
        return unicode(self.textedit.toPlainText())

    def findFocus(self):
        # print("findFocus called")
        self.findField.setFocus()
-        self.findField.selectAll()        
-        
+        self.findField.selectAll()
+
    def find(self):

        #print("find self.lastStart:%s"%self.lastStart)
-        
+
        # Grab the parent's text
        text = self.textedit.toPlainText()

@@ -1334,7 +1365,7 @@ class IniTextDialog(SizePersistedDialog):
        # last starting position
        self.lastStart = text.find(query,self.lastStart + 1)
        # If the find() method didn't return -1 (not found)
-        
+
        if self.lastStart >= 0:
            end = self.lastStart + len(query)
            self.moveCursor(self.lastStart,end)
@@ -1342,7 +1373,7 @@ class IniTextDialog(SizePersistedDialog):
            # Make the next search start from the begining again
            self.lastStart = 0
            self.textedit.moveCursor(self.textedit.textCursor().Start)
-                    
+
    def moveCursor(self,start,end):

        # We retrieve the QTextCursor object from the parent's QTextEdit
@@ -1359,7 +1390,7 @@ class IniTextDialog(SizePersistedDialog):
        self.textedit.setTextCursor(cursor)

    def select_line(self,lineno):
-        
+
        # We retrieve the QTextCursor object from the parent's QTextEdit
        cursor = self.textedit.textCursor()

@@ -1385,8 +1416,8 @@ class ViewLog(SizePersistedDialog):

    def get_lineno(self):
        return self.lineno
-        
-    def __init__(self, parent, title, errors, 
+
+    def __init__(self, parent, title, errors,
                 save_size_name='fff:view log dialog',):
        SizePersistedDialog.__init__(self, parent,save_size_name)
        self.l = l = QVBoxLayout()
@@ -1407,9 +1438,9 @@ class ViewLog(SizePersistedDialog):
            label.setToolTip(_('Click to go to line %s')%lineno)
            label.mouseReleaseEvent = partial(self.label_clicked, lineno=lineno)
            self.l.addWidget(label)
-        
+
        # html='<p>'+'</p><p>'.join([ '(lineno: %s) %s'%e for e in errors ])+'</p>'
-        
+
        # self.tb = QTextBrowser(self)
        # self.tb.setFont(QFont("Courier",
        #                       parent.font().pointSize()+1))
@@ -1427,13 +1458,13 @@ class ViewLog(SizePersistedDialog):
        saveanyway = QPushButton(_('Save Anyway'), self)
        saveanyway.clicked.connect(self.reject)
        horz.addWidget(saveanyway)
-        
+
        l.addLayout(horz)
        self.setModal(False)
        self.setWindowTitle(title)
        self.setWindowIcon(QIcon(I('debug.png')))
        #self.show()
-        
+
        # Cause our dialog size to be restored from prefs or created on first usage
        self.resize_dialog()

@@ -1454,7 +1485,7 @@ class EmailPassDialog(QDialog):

        self.setWindowTitle(_('Password'))
        self.l.addWidget(QLabel(_("Enter Email Password for %s:")%user),0,0,1,2)
-   
+
        # self.l.addWidget(QLabel(_("Password:")),1,0)
        self.passwd = QLineEdit(self)
        self.passwd.setEchoMode(QLineEdit.Password)
@@ -1,5 +1,5 @@
-#!/usr/bin/env python
-# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+# -*- coding: utf-8 -*-
+
 from __future__ import (unicode_literals, division, absolute_import,
                        print_function)

@@ -13,24 +13,24 @@ logger = logging.getLogger(__name__)
 import time, os, copy, threading, re, platform, sys
 from StringIO import StringIO
 from functools import partial
-from datetime import datetime, time
+from datetime import datetime, time, date
 from string import Template
 import urllib
 import email
 import traceback

 try:
-    from PyQt5.Qt import (QApplication, QMenu, QTimer)
+    from PyQt5.Qt import (QApplication, QMenu, QTimer, QCursor, Qt)
    from PyQt5.QtCore import QBuffer
 except ImportError as e:
-    from PyQt4.Qt import (QApplication, QMenu, QTimer)
+    from PyQt4.Qt import (QApplication, QMenu, QTimer, QCursor, Qt)
    from PyQt4.QtCore import QBuffer

 from calibre.constants import numeric_version as calibre_version

 from calibre.ptempfile import PersistentTemporaryFile, PersistentTemporaryDirectory, remove_dir
 from calibre.ebooks.metadata import MetaInformation
-from calibre.ebooks.metadata.meta import get_metadata
+from calibre.ebooks.metadata.meta import get_metadata as calibre_get_metadata
 from calibre.gui2 import error_dialog, warning_dialog, question_dialog, info_dialog
 from calibre.gui2.dialogs.message_box import ViewLog
 from calibre.gui2.dialogs.confirm_delete import confirm
@@ -48,20 +48,46 @@ try:
 except NameError:
    pass # load_translations() added in calibre 1.9

-from calibre_plugins.fanficfare_plugin.common_utils import (set_plugin_icon_resources, get_icon,
-                                                            create_menu_action_unique, get_library_uuid)
+try:
+    # should be present from cal2.3.0 on.
+    from calibre.ebooks.covers import generate_cover as cal_generate_cover
+    HAS_CALGC=True
+except:
+    HAS_CALGC=False

-from calibre_plugins.fanficfare_plugin.fanficfare import adapters, exceptions
-from calibre_plugins.fanficfare_plugin.fanficfare.epubutils import get_dcsource, get_dcsource_chaptercount, get_story_url_from_html
-from calibre_plugins.fanficfare_plugin.fanficfare.geturls import get_urls_from_page, get_urls_from_html, get_urls_from_text, get_urls_from_imap
+from calibre.library.field_metadata import FieldMetadata
+field_metadata = FieldMetadata()
+    
+from calibre_plugins.fanficfare_plugin.common_utils import (
+    set_plugin_icon_resources, get_icon, create_menu_action_unique,
+    get_library_uuid)
+
+from calibre_plugins.fanficfare_plugin.fanficfare import (
+    adapters, exceptions)
+
+from calibre_plugins.fanficfare_plugin.fanficfare.epubutils import (
+    get_dcsource, get_dcsource_chaptercount, get_story_url_from_html,
+    reset_orig_chapters_epub)
+
+from calibre_plugins.fanficfare_plugin.fanficfare.geturls import (
+    get_urls_from_page, get_urls_from_html,get_urls_from_text,
+    get_urls_from_imap)
+
+from calibre_plugins.fanficfare_plugin.fff_util import (
+    get_fff_adapter, get_fff_config, get_fff_personalini)
+
+from calibre_plugins.fanficfare_plugin.config import (
+    permitted_values, rejecturllist, STD_COLS_SKIP)
+
+from calibre_plugins.fanficfare_plugin.prefs import (
+    prefs, SAVE_YES, SAVE_NO, SAVE_YES_IF_IMG, SAVE_YES_UNLESS_IMG)

-from calibre_plugins.fanficfare_plugin.fff_util import (get_fff_adapter, get_fff_config, get_fff_personalini)
-from calibre_plugins.fanficfare_plugin.config import (permitted_values, rejecturllist)
-from calibre_plugins.fanficfare_plugin.prefs import prefs
 from calibre_plugins.fanficfare_plugin.dialogs import (
    AddNewDialog, UpdateExistingDialog,
-    LoopProgressDialog, UserPassDialog, AboutDialog, CollectURLDialog, RejectListDialog, EmailPassDialog,
+    LoopProgressDialog, UserPassDialog, AboutDialog, CollectURLDialog,
+    RejectListDialog, EmailPassDialog,
    OVERWRITE, OVERWRITEALWAYS, UPDATE, UPDATEALWAYS, ADDNEW, SKIP, CALIBREONLY,
+    CALIBREONLYSAVECOL,
    NotGoingToDownload, RejectUrlEntry )

 # because calibre immediately transforms html into zip and don't want
@@ -310,6 +336,12 @@ class FanFicFarePlugin(InterfaceAction):
                                                                      image='minusminus.png',
                                                                      triggered=partial(self.update_lists,add=False))

+            self.menu.addSeparator()
+            self.get_list_action = self.create_menu_item_ex(self.menu, _('Remove "New" Chapter Marks from Selected books'),
+                                                            unique_name='Remove "(new)" chapter marks created by personal.ini <i>mark_new_chapters</i> setting.',
+                                                            image='edit-undo.png',
+                                                            triggered=self.unnew_books)
+
            self.menu.addSeparator()
            self.get_list_action = self.create_menu_item_ex(self.menu, _('Get Story URLs from Selected Books'),
                                                            unique_name='Get URLs from Selected Books',
@@ -350,7 +382,7 @@ class FanFicFarePlugin(InterfaceAction):
        # get_resources will return a dictionary mapping names to bytes. Names that
        # are not found in the zip file will not be in the returned dictionary.

-        text = get_resources('about.txt')
+        text = get_resources('about.html')
        AboutDialog(self.gui,self.qaction.icon(),self.version + text).exec_()

    def create_menu_item_ex(self, parent_menu, menu_text, image=None, tooltip=None,
@@ -390,6 +422,8 @@ class FanFicFarePlugin(InterfaceAction):
                return

            self.update_reading_lists(self.gui.library_view.get_selected_ids(),add)
+            if not add and prefs['autounnew']:
+                self.unnew_books()

    def get_urls_from_imap_menu(self):

@@ -413,20 +447,46 @@ class FanFicFarePlugin(InterfaceAction):
            if prefs['imapsessionpass']:
                self.imap_pass = imap_pass

+        self.busy_cursor()
+        self.gui.status_bar.show_message(_('Fetching Story URLs from Email...'))
        url_list = get_urls_from_imap(prefs['imapserver'],
                                      prefs['imapuser'],
                                      imap_pass,
                                      prefs['imapfolder'],
                                      prefs['imapmarkread'],)

-        if url_list:
-            self.add_dialog("\n".join(url_list),merge=False)
-        else:
-            info_dialog(self.gui, _('Get Story URLs from Email'),
-                        _('No Valid Story URLs Found in Unread Emails.'),
-                        show=True,
-                        show_copy_button=False)
+        ## reject will now be redundant with reject check inside
+        ## prep_downloads because of change-able story URLs.
+        ## Keeping here to because it's the far more common case.
+        reject_list=set()
+        if prefs['auto_reject_from_email']:
+            # need to normalize for reject list.
+            reject_list = set([x for x in url_list if rejecturllist.check(adapters.getNormalStoryURLSite(x)[0])])
+        url_list = url_list - reject_list

+        self.gui.status_bar.show_message(_('No Valid Story URLs Found in Unread Emails.'),3000)
+        self.restore_cursor()
+
+        if prefs['download_from_email_immediately']:
+            ## do imap fetch w/o GUI elements
+            if url_list:
+                self.prep_downloads(self.add_new_dialog.get_fff_options(),
+                                    "\n".join(url_list))
+            else:
+                self.gui.status_bar.show_message(_('Finished Fetching Story URLs from Email.'),3000)
+                
+        else:
+            if url_list:
+                self.add_dialog("\n".join(url_list),merge=False)
+            else:            
+                msg = _('No Valid Story URLs Found in Unread Emails.')
+                if reject_list:
+                    msg = msg + '<p>'+(_('(%d Story URLs Skipped, on Rejected URL List)')%len(reject_list))+'</p>'
+                info_dialog(self.gui, _('Get Story URLs from Email'),
+                            msg,
+                            show=True,
+                            show_copy_button=False)
+    
    def get_urls_from_page_menu(self,anthology=False):

        urltxt = ""
@@ -444,8 +504,14 @@ class FanFicFarePlugin(InterfaceAction):
            return
        url = u"%s"%d.url.text()

+        self.busy_cursor()
+        self.gui.status_bar.show_message(_('Fetching Story URLs from Page...'))
+
        url_list = self.get_urls_from_page(url)

+        self.gui.status_bar.show_message(_('Finished Fetching Story URLs from Page.'),3000)
+        self.restore_cursor()
+        
        if url_list:
            self.add_dialog("\n".join(url_list),merge=d.anthology,anthology_url=url)
        else:
@@ -510,6 +576,70 @@ class FanFicFarePlugin(InterfaceAction):
                        show=True,
                        show_copy_button=False)

+    def unnew_books(self):
+        '''Get list of URLs from existing books.'''
+        if not self.is_library_view():
+            self.gui.status_bar.show_message(_('Can only UnNew books in library'),
+                                             3000)
+            return
+        
+        if not self.gui.current_view().selectionModel().selectedRows() :
+            self.gui.status_bar.show_message(_('No Selected Books to Get URLs From'),
+                                             3000)
+            return
+
+        book_list = map( partial(self.make_book_id_only),
+                         self.gui.library_view.get_selected_ids() )
+
+        tdir = PersistentTemporaryDirectory(prefix='fanficfare_')
+        LoopProgressDialog(self.gui,
+                           book_list,
+                           partial(self.get_unnew_books_loop, db=self.gui.current_db, tdir=tdir),
+                           partial(self.get_unnew_books_finish, tdir=tdir),
+                           init_label=_("UnNewing books..."),
+                           win_title=_("UnNew Books"),
+                           status_prefix=_("Books UnNewed"))
+
+    def get_unnew_books_loop(self,book,db=None,tdir=None):
+        
+        if book['calibre_id'] and db.has_format(book['calibre_id'],'EPUB',index_is_id=True):
+            tmp = PersistentTemporaryFile(prefix='%s-'%book['calibre_id'],
+                                          suffix='.epub',
+                                          dir=tdir)
+            db.copy_format_to(book['calibre_id'],'EPUB',tmp,index_is_id=True)
+            
+            unnewtmp = PersistentTemporaryFile(prefix='unnew-%s-'%book['calibre_id'],
+                                               suffix='.epub',
+                                               dir=tdir)
+            book['changed']=reset_orig_chapters_epub(tmp,unnewtmp)
+            if book['changed']:
+                db.add_format_with_hooks(book['calibre_id'],
+                                         'EPUB',
+                                         unnewtmp,
+                                         index_is_id=True)
+                if prefs['deleteotherforms']:
+                    fmts = db.formats(book['calibre_id'], index_is_id=True).split(',')
+                    for fmt in fmts:
+                        if fmt.lower() != formmapping['epub'].lower():
+                            logger.debug("deleteotherforms remove f:"+fmt)
+                            db.remove_format(book['calibre_id'], fmt, index_is_id=True)#, notify=False
+                elif prefs['autoconvert']:
+                    ## 'Convert Book'.auto_convert_auto_add doesn't convert if
+                    ## the format is already there.
+                    fmt = calibre_prefs['output_format']
+                    # delete if there, but not if the format we just made.
+                    if fmt.lower() != 'epub' and db.has_format(book['calibre_id'],fmt,index_is_id=True):
+                        logger.debug("autoconvert remove f:"+fmt)
+                        db.remove_format(book['calibre_id'], fmt, index_is_id=True)#, notify=False
+                
+    def get_unnew_books_finish(self, book_list, tdir=None):
+        remove_dir(tdir)
+        if prefs['autoconvert']:
+            changed_ids = [ x['calibre_id'] for x in book_list if x['changed'] ]
+            if changed_ids:
+                self.gui.status_bar.show_message(_('Starting auto conversion of %d books.')%(len(changed_ids)), 3000)
+                self.gui.iactions['Convert Books'].auto_convert_auto_add(changed_ids)
+
    def reject_list_urls(self):
        if self.is_library_view():
            book_list = map( partial(self.make_book_id_only),
@@ -630,6 +760,9 @@ class FanFicFarePlugin(InterfaceAction):
            remove_dir(tdir)
            return

+        self.busy_cursor()
+        self.gui.status_bar.show_message(_('Fetching Story URLs for Series...'))
+        
        # get list from identifiers:url/uri if present, but only if
        # it's *not* a valid story URL.
        mergeurl = self.get_story_url(db,book_id)
@@ -638,6 +771,9 @@ class FanFicFarePlugin(InterfaceAction):

        url_list_text = "\n".join(url_list)

+        self.gui.status_bar.show_message(_('Finished Fetching Story URLs for Series.'),3000)
+        self.restore_cursor()
+        
        #print("urlmapfile:%s"%urlmapfile)

        # AddNewDialog collects URLs, format and presents buttons.
@@ -759,10 +895,10 @@ class FanFicFarePlugin(InterfaceAction):
        # No need to do anything with perfs here, but we could.
        prefs

-    def make_id_searchstr(self,url):
+    def do_id_search(self,url):
        # older idents can be uri vs url and have | instead of : after
        # http, plus many sites are now switching to https.
-        return 'identifiers:"~ur(i|l):~^%s$"'%re.sub(r'https?\\\:','https?(\:|\|)',re.escape(url))
+        return self.gui.current_db.search_getting_ids('identifiers:"~ur(i|l):~^%s$"'%re.sub(r'https?\\\:','https?(\:|\|)',re.escape(url)),None)

    def prep_downloads(self, options, books, merge=False, extrapayload=None):
        '''Fetch metadata for stories from servers, launch BG job when done.'''
@@ -777,6 +913,7 @@ class FanFicFarePlugin(InterfaceAction):
        options['version'] = self.version
        logger.debug(self.version)
        options['personal.ini'] = get_fff_personalini()
+        options['savemetacol'] = prefs['savemetacol']

        #print("prep_downloads:%s"%books)

@@ -798,6 +935,43 @@ class FanFicFarePlugin(InterfaceAction):
        # LoopProgressDialog calls start_download_job at the end which goes
        # into the BG, or shows list if no 'good' books.

+    def reject_url(self,merge,book):
+        url = book['url']
+        if not merge and rejecturllist.check(url): # skip reject list when merging.
+            rejnote = rejecturllist.get_full_note(url)
+            if prefs['reject_always'] or question_dialog(self.gui, _('Reject URL?'),'''
+                      <h3>%s</h3>
+                      <p>%s</p>
+                      <p>"<b>%s</b>"</p>
+                      <p>%s</p>
+                      <p>%s</p>'''%(
+                    _('Reject URL?'),
+                    _('<b>%s</b> is on your Reject URL list:')%url,
+                    rejnote,
+                    _("Click '<b>Yes</b>' to Reject."),
+                    _("Click '<b>No</b>' to download anyway.")),
+                               show_copy_button=False):
+                book['comment'] = _("Story on Reject URLs list (%s).")%rejnote
+                book['good']=False
+                book['icon']='rotate-right.png'
+                book['status'] = _('Rejected')
+                return True
+            else:
+                if question_dialog(self.gui, _('Remove Reject URL?'),'''
+                          <h3>%s</h3>
+                          <p>%s</p>
+                          <p>"<b>%s</b>"</p>
+                          <p>%s</p>
+                          <p>%s</p>'''%(
+                        _("Remove URL from Reject List?"),
+                        _('<b>%s</b> is on your Reject URL list:')%url,
+                        rejnote,
+                        _("Click '<b>Yes</b>' to remove it from the list,"),
+                        _("Click '<b>No</b>' to leave it on the list.")),
+                                   show_copy_button=False):
+                    rejecturllist.remove(url)
+        return False
+            
    def prep_download_loop(self,book,
                           options={'fileform':'epub',
                                    'collision':ADDNEW,
@@ -814,40 +988,11 @@ class FanFicFarePlugin(InterfaceAction):
        logger.debug("url:%s"%url)
        mi = None

-        if not merge: # skip reject list when merging.
-            if rejecturllist.check(url):
-                rejnote = rejecturllist.get_full_note(url)
-                if prefs['reject_always'] or question_dialog(self.gui, _('Reject URL?'),'''
-                          <h3>%s</h3>
-                          <p>%s</p>
-                          <p>"<b>%s</b>"</p>
-                          <p>%s</p>
-                          <p>%s</p>'''%(
-                        _('Reject URL?'),
-                        _('<b>%s</b> is on your Reject URL list:')%url,
-                        rejnote,
-                        _("Click '<b>Yes</b>' to Reject."),
-                        _("Click '<b>No</b>' to download anyway.")),
-                                   show_copy_button=False):
-                    book['comment'] = _("Story on Reject URLs list (%s).")%rejnote
-                    book['good']=False
-                    book['icon']='rotate-right.png'
-                    book['status'] = _('Rejected')
-                    return
-                else:
-                    if question_dialog(self.gui, _('Remove Reject URL?'),'''
-                              <h3>%s</h3>
-                              <p>%s</p>
-                              <p>"<b>%s</b>"</p>
-                              <p>%s</p>
-                              <p>%s</p>'''%(
-                            _("Remove URL from Reject List?"),
-                            _('<b>%s</b> is on your Reject URL list:')%url,
-                            rejnote,
-                            _("Click '<b>Yes</b>' to remove it from the list,"),
-                            _("Click '<b>No</b>' to leave it on the list.")),
-                                       show_copy_button=False):
-                        rejecturllist.remove(url)
+        ## Check reject list.  Redundant with below for when story URL
+        ## changes, but also kept here to avoid network hit in most
+        ## common case where given url is story url.
+        if self.reject_url(merge,book):
+            return

        # The current database shown in the GUI
        # db is an instance of the class LibraryDatabase2 from database.py
@@ -868,9 +1013,9 @@ class FanFicFarePlugin(InterfaceAction):
            # book has already been flagged bad for whatever reason.
            return

-        skip_date_update = False
-
        adapter = get_fff_adapter(url,fileform)
+        ## chapter range for title_chapter_range_pattern
+        adapter.setChaptersRange(book['begin'],book['end'])
        ## save and share cookiejar and pagecache between all
        ## downloads.
        if 'pagecache' not in options:
@@ -880,67 +1025,101 @@ class FanFicFarePlugin(InterfaceAction):
            options['cookiejar'] = adapter.get_empty_cookiejar()
        adapter.set_cookiejar(options['cookiejar'])

-        # reduce foreground sleep time for ffnet when few books.
-        if 'ffnetcount' in options and \
-                adapter.getConfig('tweak_fg_sleep') and \
-                adapter.getSiteDomain() == 'www.fanfiction.net':
-            minslp = float(adapter.getConfig('min_fg_sleep'))
-            maxslp = float(adapter.getConfig('max_fg_sleep'))
-            dwnlds = float(adapter.getConfig('max_fg_sleep_at_downloads'))
-            m = (maxslp-minslp) / (dwnlds-1)
-            b = minslp - m
-            slp = min(maxslp,m*float(options['ffnetcount'])+b)
-            #print("m:%s b:%s = %s"%(m,b,slp))
-            adapter.set_sleep(slp)
+        if collision in (CALIBREONLY, CALIBREONLYSAVECOL):
+            ## Getting metadata from configured column.
+            custom_columns = self.gui.library_view.model().custom_columns
+            if ( collision in (CALIBREONLYSAVECOL) and
+                 prefs['savemetacol'] != '' and 
+                 prefs['savemetacol'] in custom_columns ):

-        ## three tries, that's enough if both user/pass & is_adult needed,
-        ## or a couple tries of one or the other
-        for x in range(0,2):
-            try:
-                adapter.getStoryMetadataOnly(get_cover=False)
-            except exceptions.FailedToLogin, f:
-                logger.warn("Login Failed, Need Username/Password.")
-                userpass = UserPassDialog(self.gui,url,f)
-                userpass.exec_() # exec_ will make it act modal
-                if userpass.status:
-                    adapter.username = userpass.user.text()
-                    adapter.password = userpass.passwd.text()
-
-            except exceptions.AdultCheckRequired:
-                if question_dialog(self.gui, _('Are You an Adult?'), '<p>'+
-                                   _("%s requires that you be an adult.  Please confirm you are an adult in your locale:")%url,
-                                   show_copy_button=False):
-                    adapter.is_adult=True
-
-        # let other exceptions percolate up.
-        story = adapter.getStoryMetadataOnly(get_cover=False)
-
-        series = story.getMetadata('series')
-        if not merge and series and prefs['checkforseriesurlid']:
-            # try to find *series anthology* by *seriesUrl* identifier url or uri first.
-            searchstr = self.make_id_searchstr(story.getMetadata('seriesUrl'))
-            identicalbooks = db.search_getting_ids(searchstr, None)
-            # print("searchstr:%s"%searchstr)
-            # print("identicalbooks:%s"%identicalbooks)
-            if len(identicalbooks) > 0 and question_dialog(self.gui, _('Skip Story?'),'''
-                                                              <h3>%s</h3>
-                                                              <p>%s</p>
-                                                              <p>%s</p>
-                                                              <p>%s</p>
-                                                           '''%(
-                                                           _('Skip Anthology Story?'),
-                                                           _('"<b>%s</b>" is in series "<b><a href="%s">%s</a></b>" that you have an anthology book for.')%(story.getMetadata('title'),story.getMetadata('seriesUrl'),series[:series.index(' [')]),
-                                                           _("Click '<b>Yes</b>' to Skip."),
-                                                           _("Click '<b>No</b>' to download anyway.")),
-                                                           show_copy_button=False):
-                book['comment'] = _("Story in Series Anthology(%s).")%series
-                book['title'] = story.getMetadata('title')
-                book['author'] = [story.getMetadata('author')]
-                book['good']=False
-                book['icon']='rotate-right.png'
-                book['status'] = _('Skipped')
+                savedmeta_book_id = book['calibre_id']
+                # won't have calibre_id if update by URL vs book.
+                if not savedmeta_book_id:
+                    identicalbooks = self.do_id_search(url)
+                    if len(identicalbooks) == 1:
+                        savedmeta_book_id = identicalbooks.pop()
+                        
+                if savedmeta_book_id:
+                    label = custom_columns[prefs['savemetacol']]['label']
+                    savedmetadata = db.get_custom(savedmeta_book_id, label=label, index_is_id=True)
+                else:
+                    savedmetadata = None
+                    
+                if savedmetadata:
+                    # sets flag inside story so getStoryMetadataOnly won't hit server.
+                    adapter.setStoryMetadata(savedmetadata)
+                
+            # let other exceptions percolate up.
+            story = adapter.getStoryMetadataOnly(get_cover=False)
+        else:
+            # reduce foreground sleep time for ffnet when few books.
+            if 'ffnetcount' in options and \
+                    adapter.getConfig('tweak_fg_sleep') and \
+                    adapter.getSiteDomain() == 'www.fanfiction.net':
+                minslp = float(adapter.getConfig('min_fg_sleep'))
+                maxslp = float(adapter.getConfig('max_fg_sleep'))
+                dwnlds = float(adapter.getConfig('max_fg_sleep_at_downloads'))
+                m = (maxslp-minslp) / (dwnlds-1)
+                b = minslp - m
+                slp = min(maxslp,m*float(options['ffnetcount'])+b)
+                #print("m:%s b:%s = %s"%(m,b,slp))
+                adapter.set_sleep(slp)
+    
+            ## three tries, that's enough if both user/pass & is_adult needed,
+            ## or a couple tries of one or the other
+            for x in range(0,2):
+                try:
+                    adapter.getStoryMetadataOnly(get_cover=False)
+                except exceptions.FailedToLogin, f:
+                    logger.warn("Login Failed, Need Username/Password.")
+                    userpass = UserPassDialog(self.gui,url,f)
+                    userpass.exec_() # exec_ will make it act modal
+                    if userpass.status:
+                        adapter.username = userpass.user.text()
+                        adapter.password = userpass.passwd.text()
+    
+                except exceptions.AdultCheckRequired:
+                    if question_dialog(self.gui, _('Are You an Adult?'), '<p>'+
+                                       _("%s requires that you be an adult.  Please confirm you are an adult in your locale:")%url,
+                                       show_copy_button=False):
+                        adapter.is_adult=True
+    
+            # let other exceptions percolate up.
+            story = adapter.getStoryMetadataOnly(get_cover=False)
+            book['title'] = story.getMetadata('title')
+            book['author'] = [story.getMetadata('author')]
+            book['url'] = story.getMetadata('storyUrl')
+    
+            ## Check reject list.  Redundant with below for when story
+            ## URL changes, but also kept here to avoid network hit in
+            ## most common case where given url is story url.
+            if self.reject_url(merge,book):
                return

+            series = story.getMetadata('series')
+            if not merge and series and prefs['checkforseriesurlid']:
+                # try to find *series anthology* by *seriesUrl* identifier url or uri first.
+                identicalbooks = self.do_id_search(story.getMetadata('seriesUrl'))
+                # print("identicalbooks:%s"%identicalbooks)
+                if len(identicalbooks) > 0 and question_dialog(self.gui, _('Skip Story?'),'''
+                                                                  <h3>%s</h3>
+                                                                  <p>%s</p>
+                                                                  <p>%s</p>
+                                                                  <p>%s</p>
+                                                               '''%(
+                                                               _('Skip Anthology Story?'),
+                                                               _('"<b>%s</b>" is in series "<b><a href="%s">%s</a></b>" that you have an anthology book for.')%(story.getMetadata('title'),story.getMetadata('seriesUrl'),series[:series.index(' [')]),
+                                                               _("Click '<b>Yes</b>' to Skip."),
+                                                               _("Click '<b>No</b>' to download anyway.")),
+                                                               show_copy_button=False):
+                    book['comment'] = _("Story in Series Anthology(%s).")%series
+                    book['title'] = story.getMetadata('title')
+                    book['author'] = [story.getMetadata('author')]
+                    book['url'] = story.getMetadata('storyUrl')
+                    book['good']=False
+                    book['icon']='rotate-right.png'
+                    book['status'] = _('Skipped')
+                    return

        ################################################################################################################################################33

@@ -950,10 +1129,14 @@ class FanFicFarePlugin(InterfaceAction):

        # all_metadata duplicates some data, but also includes extra_entries, etc.
        book['all_metadata'] = story.getAllMetadata(removeallentities=True)
-
+        if prefs['savemetacol'] != '':
+            # get metadata to save in configured column.
+            book['savemetacol'] = story.dump_html_metadata()
+        
        book['title'] = story.getMetadata("title", removeallentities=True)
        book['author_sort'] = book['author'] = story.getList("author", removeallentities=True)
        book['publisher'] = story.getMetadata("site")
+        book['url'] = story.getMetadata("storyUrl")
        book['tags'] = story.getSubjectTags(removeallentities=True)
        if story.getMetadata("description"):
            book['comments'] = sanitize_comments_html(story.getMetadata("description"))
@@ -977,7 +1160,7 @@ class FanFicFarePlugin(InterfaceAction):
            book['timestamp'] = None # need *something* there for calibre.

        if not merge:# skip all the collision code when d/ling for merging.
-            if collision in (CALIBREONLY):
+            if collision in (CALIBREONLY, CALIBREONLYSAVECOL):
                book['icon'] = 'metadata.png'
                book['status'] = _('Meta')

@@ -995,11 +1178,9 @@ class FanFicFarePlugin(InterfaceAction):
                logger.debug("from URL(%s)"%url)

                # try to find by identifier url or uri first.
-                searchstr = self.make_id_searchstr(url)
-                identicalbooks = db.search_getting_ids(searchstr, None)
-                # print("searchstr:%s"%searchstr)
+                identicalbooks = self.do_id_search(url)
                # print("identicalbooks:%s"%identicalbooks)
-                if len(identicalbooks) < 1:
+                if len(identicalbooks) < 1 and prefs['matchtitleauth']:
                    # find dups
                    authlist = story.getList("author", removeallentities=True)
                    mi = MetaInformation(story.getMetadata("title", removeallentities=True),
@@ -1018,7 +1199,7 @@ class FanFicFarePlugin(InterfaceAction):
                    raise NotGoingToDownload(_("More than one identical book by Identifer URL or title/author(s)--can't tell which book to update/overwrite."),"minusminus.png")

                ## changed: add new book when CALIBREONLY if none found.
-                if collision == CALIBREONLY and not identicalbooks:
+                if collision in (CALIBREONLY, CALIBREONLYSAVECOL) and not identicalbooks:
                    collision = ADDNEW
                    options['collision'] = ADDNEW

@@ -1073,7 +1254,7 @@ class FanFicFarePlugin(InterfaceAction):
                                return

            if book_id != None and collision != ADDNEW:
-                if collision in (CALIBREONLY):
+                if collision in (CALIBREONLY, CALIBREONLYSAVECOL):
                    book['comment'] = _('Metadata collected.')
                    # don't need temp file created below.
                    return
@@ -1091,9 +1272,6 @@ class FanFicFarePlugin(InterfaceAction):
                        if chaptercount == urlchaptercount:
                            if collision == UPDATE:
                                raise NotGoingToDownload(_("Already contains %d chapters.")%chaptercount,'edit-undo.png')
-                            else:
-                                # UPDATEALWAYS
-                                skip_date_update = True
                        elif chaptercount > urlchaptercount:
                            raise NotGoingToDownload(_("Existing epub contains %d chapters, web site only has %d. Use Overwrite to force update.") % (chaptercount,urlchaptercount),'dialog_error.png')
                        elif chaptercount == 0:
@@ -1132,6 +1310,52 @@ class FanFicFarePlugin(InterfaceAction):
                    #print("calibre_series:%s [%s]"%book['calibre_series'])

        if book['good']: # there shouldn't be any !'good' books at this point.
+
+            ## Filling calibre_std_* and calibre_cust_* metadata
+            book['calibre_columns']={}
+            if prefs['cal_cols_pass_in']:
+                # std columns
+                mi = db.get_metadata(book['calibre_id'],index_is_id=True)
+                # book['calibre_columns']['calibre_std_identifiers']=\
+                #     {'val':', '.join(["%s:%s"%(k,v) for (k,v) in mi.get_identifiers().iteritems()]),
+                #                      'label':_('Ids')}
+                for k in mi.standard_field_keys():
+                # for k in mi:
+                    if k in STD_COLS_SKIP:
+                        continue
+                    (label,value,v,fmd) = mi.format_field_extended(k)
+                    if not label and k in field_metadata:
+                        label=field_metadata[k]['name']
+                    key='calibre_std_'+k
+                    
+                    # if k == 'user_categories':
+                    #     value=u', '.join(mi.get(k))
+                    #     label=_('User Categories')
+        
+                    if label: # only if it has a human readable name.
+                        if value is None or not book['calibre_id']:
+                            ## if existing book, populate existing calibre column
+                            ## values in metadata, else '' to hide.
+                            value=''                    
+                        book['calibre_columns'][key]={'val':value,'label':label}
+                        #logger.debug("%s(%s): %s"%(label,key,value))
+        
+                # custom columns
+                for k, column in self.gui.library_view.model().custom_columns.iteritems():
+                    if k != prefs['savemetacol']:
+                        key='calibre_cust_'+k[1:]
+                        label=column['name']
+                        value=db.get_custom(book['calibre_id'],
+                                            label=column['label'],
+                                            index_is_id=True)
+                        # custom always have name.
+                        if value is None or not book['calibre_id']:
+                            ## if existing book, populate existing calibre column
+                            ## values in metadata, else '' to hide.
+                            value=''
+                        book['calibre_columns'][key]={'val':value,'label':label}
+                        # logger.debug("%s(%s): %s"%(label,key,value))
+            
            # if still 'good', make a temp file to write the output to.
            # For HTML format users, make the filename inside the zip something reasonable.
            # For crazy long titles/authors, limit it to 200chars.
@@ -1143,17 +1367,6 @@ class FanFicFarePlugin(InterfaceAction):
            logger.debug("outfile:"+tmp.name)
            book['outfile'] = tmp.name

-            # cookiejar = PersistentTemporaryFile(prefix=story.formatFileName("${title}-${author}-",allowunsafefilename=False)[:100],
-            #                                     suffix='.cookiejar',
-            #                                     dir=options['tdir'])
-            # adapter.save_cookiejar(cookiejar.name)
-            # book['cookiejar'] = cookiejar.name
-            # pagecache = PersistentTemporaryFile(prefix=story.formatFileName("${title}-${author}-",allowunsafefilename=False)[:100],
-            #                                     suffix='.pagecache',
-            #                                     dir=options['tdir'])
-            # adapter.save_pagecache(pagecache.name)
-            # book['pagecache'] = pagecache.name
-
        return

    def start_download_job(self,book_list,
@@ -1170,7 +1383,7 @@ class FanFicFarePlugin(InterfaceAction):

        ## No need to BG process when CALIBREONLY!  Fake it.
        #print("options:%s"%options)
-        if options['collision'] == CALIBREONLY:
+        if options['collision'] in (CALIBREONLY, CALIBREONLYSAVECOL):
            class NotJob(object):
                def __init__(self,result):
                    self.failed=False
@@ -1255,10 +1468,10 @@ class FanFicFarePlugin(InterfaceAction):
        logger.debug("add/update %s %s"%(book['title'],book['url']))
        mi = self.make_mi_from_book(book)

-        if options['collision'] != CALIBREONLY:
+        if options['collision'] not in (CALIBREONLY, CALIBREONLYSAVECOL):
            self.add_book_or_update_format(book,options,prefs,mi)

-        if options['collision'] == CALIBREONLY or \
+        if options['collision'] in (CALIBREONLY, CALIBREONLYSAVECOL) or \
                ( (options['updatemeta'] or book['added']) and book['good'] ):
            try:
                self.update_metadata(db, book['calibre_id'], book, mi, options)
@@ -1285,7 +1498,7 @@ class FanFicFarePlugin(InterfaceAction):
        failed_list = filter(lambda x : not x['good'] , book_list)
        failed_ids = [ x['calibre_id'] for x in failed_list ]

-        if options['collision'] != CALIBREONLY and \
+        if options['collision'] not in (CALIBREONLY, CALIBREONLYSAVECOL) and \
                (prefs['addtolists'] or prefs['addtoreadlists']):
            self.update_reading_lists(all_ids,add=True)

@@ -1349,7 +1562,7 @@ class FanFicFarePlugin(InterfaceAction):
            if countpagesstats:
                cp_plugin.count_statistics(all_ids,countpagesstats)

-        if prefs['autoconvert'] and options['collision'] != CALIBREONLY:
+        if prefs['autoconvert'] and options['collision'] not in (CALIBREONLY, CALIBREONLYSAVECOL):
            self.gui.status_bar.show_message(_('Starting auto conversion of %d books.')%(len(all_ids)), 3000)
            self.gui.iactions['Convert Books'].auto_convert_auto_add(all_ids)

@@ -1609,15 +1822,6 @@ class FanFicFarePlugin(InterfaceAction):
            if not oldmi.languages:
                mi.languages=['en']

-        if options['fileform'] == 'epub' and prefs['updatecover']:
-            existingepub = db.format(book_id,'EPUB',index_is_id=True, as_file=True)
-            epubmi = get_metadata(existingepub,'EPUB')
-            if epubmi.cover_data[1] is not None:
-                try:
-                    db.set_cover(book_id, epubmi.cover_data[1])
-                except:
-                    logger.info("Failed to set_cover, skipping")
-
        # implement 'newonly' flags here by setting to the current
        # value again.
        if not book['added']:
@@ -1640,6 +1844,11 @@ class FanFicFarePlugin(InterfaceAction):
        #print("all_metadata: %s"%book['all_metadata'])
        custom_columns = self.gui.library_view.model().custom_columns

+        # save metadata to configured column
+        if 'savemetacol' in book and prefs['savemetacol'] != '' and prefs['savemetacol'] in custom_columns:
+            label = custom_columns[prefs['savemetacol']]['label']
+            self.set_custom(db, book_id, 'comment', book['savemetacol'], label=label, commit=True)
+
        #print("prefs['custom_cols'] %s"%prefs['custom_cols'])
        for col, meta in prefs['custom_cols'].iteritems():
            #print("setting %s to %s"%(col,meta))
@@ -1740,7 +1949,7 @@ class FanFicFarePlugin(InterfaceAction):

        # set author link if found.  All current adapters have authorUrl, except anonymous on AO3.
        # Moved down so author's already in the DB.
-        if 'authorUrl' in book['all_metadata']:
+        if 'authorUrl' in book['all_metadata'] and prefs['set_author_url']:
            authurls = book['all_metadata']['authorUrl'].split(", ")
            authorlist = [ a.replace('&',';') for a in book['author'] ]
            authorids = db.new_api.get_item_ids('authors',authorlist)
@@ -1757,73 +1966,105 @@ class FanFicFarePlugin(InterfaceAction):

        db.commit()

-        if 'Generate Cover' in self.gui.iactions and (book['added'] or not prefs['gcnewonly']):
-
-            #logger.debug("Do Generate Cover added:%s gcnewonly:%s"%(book['added'],prefs['gcnewonly']))
-
-            # force a refresh if generating cover so complex composite
-            # custom columns are current and correct
-            db.refresh_ids([book_id])
-
-            gc_plugin = self.gui.iactions['Generate Cover']
-            setting_name = None
-            if prefs['allow_gc_from_ini']:
-                if not configuration: # might already have it from allow_custcol_from_ini
-                    configuration = get_fff_config(book['url'],options['fileform'])
-
-                # template => regexp to match => GC Setting to use.
-                # generate_cover_settings:
-                # ${category} => Buffy:? the Vampire Slayer => Buffy
-                # for line in configuration.getConfig('generate_cover_settings').splitlines():
-                #     if "=>" in line:
-                #         (template,regexp,setting) = map( lambda x: x.strip(), line.split("=>") )
-                for (template,regexp,setting) in configuration.get_generate_cover_settings():
-                    value = Template(template).safe_substitute(book['all_metadata']).encode('utf8')
-                    # print("%s(%s) => %s => %s"%(template,value,regexp,setting))
-                    if re.search(regexp,value):
-                        setting_name = setting
-                        break
+        # updating calibre cover from book.
+        if options['fileform'] == 'epub' and (
+            (prefs['updatecover'] and not prefs['updatecalcover']) ## backward compat
+            or prefs['updatecalcover'] == SAVE_YES ## yes, always
+            or (prefs['updatecalcover'] == SAVE_YES_IF_IMG ## yes, if image.
+                and book['all_metadata']['cover_image'] )): # in ('specific','first','default','old')
+            existingepub = db.format(book_id,'EPUB',index_is_id=True, as_file=True)
+            epubmi = calibre_get_metadata(existingepub,'EPUB')
+            if epubmi.cover_data[1] is not None:
+                try:
+                    db.set_cover(book_id, epubmi.cover_data[1])
+                except:
+                    logger.info("Failed to set_cover, skipping")

+        # First, should cover generation happen at all?
+        if (book['added'] or not prefs['gcnewonly']) and ( # skip if not new book and gcnewonly is True
+            prefs['gencalcover'] == SAVE_YES ## yes, always
+            or (prefs['gencalcover'] == SAVE_YES_UNLESS_IMG ## yes, unless image.
+                and book['all_metadata']['cover_image'] not in ('specific','first','default')) ):
+        
+            cover_generated = False # flag for polish below.
+            # Yes, should do gencov.  Which?
+            if prefs['calibre_gen_cover'] and HAS_CALGC:
+                # calibre's builtin, if available.
+                cdata = cal_generate_cover(mi)
+                db.set_cover(book_id, cdata)
+                cover_generated = True
+            elif prefs['plugin_gen_cover'] and 'Generate Cover' in self.gui.iactions:
+                # plugin, if available.
+    
+                #logger.debug("Do Generate Cover added:%s gcnewonly:%s"%(book['added'],prefs['gcnewonly']))
+    
+                # force a refresh if generating cover so complex composite
+                # custom columns are current and correct
+                db.refresh_ids([book_id])
+    
+                gc_plugin = self.gui.iactions['Generate Cover']
+                setting_name = None
+                if prefs['allow_gc_from_ini']:
+                    if not configuration: # might already have it from allow_custcol_from_ini
+                        configuration = get_fff_config(book['url'],options['fileform'])
+    
+                    # template => regexp to match => GC Setting to use.
+                    # generate_cover_settings:
+                    # ${category} => Buffy:? the Vampire Slayer => Buffy
+                    # for line in configuration.getConfig('generate_cover_settings').splitlines():
+                    #     if "=>" in line:
+                    #         (template,regexp,setting) = map( lambda x: x.strip(), line.split("=>") )
+                    for (template,regexp,setting) in configuration.get_generate_cover_settings():
+                        value = Template(template).safe_substitute(book['all_metadata']).encode('utf8')
+                        print("%s(%s) => %s => %s"%(template,value,regexp,setting))
+                        if re.search(regexp,value):
+                            setting_name = setting
+                            break
+    
+                    if setting_name:
+                        logger.debug("Generate Cover Setting from generate_cover_settings(%s)"%setting_name)
+                        if setting_name not in gc_plugin.get_saved_setting_names():
+                            logger.info("GC Name %s not found, discarding! (check personal.ini for typos)"%setting_name)
+                            setting_name = None
+    
+                if not setting_name and book['all_metadata']['site'] in prefs['gc_site_settings']:
+                    setting_name =  prefs['gc_site_settings'][book['all_metadata']['site']]
+                    logger.debug("Generate Cover Setting from site(%s)"%setting_name)
+    
+                if not setting_name and 'Default' in prefs['gc_site_settings']:
+                    setting_name =  prefs['gc_site_settings']['Default']
+                    logger.debug("Generate Cover Setting from Default(%s)"%setting_name)
+    
                if setting_name:
-                    logger.debug("Generate Cover Setting from generate_cover_settings(%s)"%setting_name)
-                    if setting_name not in gc_plugin.get_saved_setting_names():
-                        logger.info("GC Name %s not found, discarding! (check personal.ini for typos)"%setting_name)
-                        setting_name = None
-
-            if not setting_name and book['all_metadata']['site'] in prefs['gc_site_settings']:
-                setting_name =  prefs['gc_site_settings'][book['all_metadata']['site']]
-                logger.debug("Generate Cover Setting from site(%s)"%setting_name)
-
-            if not setting_name and 'Default' in prefs['gc_site_settings']:
-                setting_name =  prefs['gc_site_settings']['Default']
-                logger.debug("Generate Cover Setting from Default(%s)"%setting_name)
-
-            if setting_name:
-                logger.debug("Running Generate Cover with settings %s."%setting_name)
-                realmi = db.get_metadata(book_id, index_is_id=True)
-                gc_plugin.generate_cover_for_book(realmi,saved_setting_name=setting_name)
-
-                if prefs['gc_polish_cover'] and \
-                        options['fileform'] == "epub":
-                    # set cover inside epub from calibre's polish feature
-                    from calibre.ebooks.oeb.polish.main import polish, ALL_OPTS
-                    from calibre.utils.logging import Log
-                    from collections import namedtuple
-
-                    # Couldn't find a better way to get the cover path.
-                    cover_path = os.path.join(db.library_path, db.path(book_id, index_is_id=True), 'cover.jpg')
-                    data = {'cover':cover_path}
-                    #print("cover_path:%s"%cover_path)
-                    opts = ALL_OPTS.copy()
-                    opts.update(data)
-                    O = namedtuple('Options', ' '.join(ALL_OPTS.iterkeys()))
-                    opts = O(**opts)
-
-                    log = Log(level=Log.DEBUG)
-                    outfile = db.format_abspath(book_id, formmapping[options['fileform']], index_is_id=True)
-                    #print("polish cover outfile:%s"%outfile)
-                    polish({outfile:outfile}, opts, log, logger.info)
-
+                    logger.debug("Running Generate Cover with settings %s."%setting_name)
+                    realmi = db.get_metadata(book_id, index_is_id=True)
+                    gc_plugin.generate_cover_for_book(realmi,saved_setting_name=setting_name)
+                    cover_generated = True
+    
+            if cover_generated and prefs['gc_polish_cover'] and \
+                    options['fileform'] == "epub":
+                # set cover inside epub from calibre's polish feature
+                from calibre.ebooks.oeb.polish.main import polish, ALL_OPTS
+                from calibre.utils.logging import Log
+                from collections import namedtuple
+    
+                # Couldn't find a better way to get the cover path.
+                cover_path = os.path.join(db.library_path,
+                                          db.path(book_id, index_is_id=True),
+                                          'cover.jpg')
+                data = {'cover':cover_path}
+                #print("cover_path:%s"%cover_path)
+                opts = ALL_OPTS.copy()
+                opts.update(data)
+                O = namedtuple('Options', ' '.join(ALL_OPTS.iterkeys()))
+                opts = O(**opts)
+    
+                log = Log(level=Log.DEBUG)
+                outfile = db.format_abspath(book_id,
+                                            formmapping[options['fileform']],
+                                            index_is_id=True)
+                #print("polish cover outfile:%s"%outfile)
+                polish({outfile:outfile}, opts, log, logger.info)

    def get_clean_reading_lists(self,lists):
        if lists == None or lists.strip() == "" :
@@ -1916,6 +2157,7 @@ class FanFicFarePlugin(InterfaceAction):
        book['site'] = ''
        book['added'] = False
        book['pubdate'] = None
+        book['publisher'] = None
        return book

    def convert_urls_to_books(self, urls):
@@ -2018,7 +2260,7 @@ class FanFicFarePlugin(InterfaceAction):
            existingepub = None
            if path == None and db.has_format(book_id,'EPUB',index_is_id=True):
                existingepub = db.format(book_id,'EPUB',index_is_id=True, as_file=True)
-                mi = get_metadata(existingepub,'EPUB')
+                mi = calibre_get_metadata(existingepub,'EPUB')
                identifiers = mi.get_identifiers()
                if 'url' in identifiers:
                    # print("url from get_metadata:%s"%identifiers['url'].replace('|',':'))
@@ -2172,6 +2414,12 @@ class FanFicFarePlugin(InterfaceAction):

        return book

+    def busy_cursor(self):
+        QApplication.setOverrideCursor(QCursor(Qt.WaitCursor))
+
+    def restore_cursor(self):
+        QApplication.restoreOverrideCursor()
+    
 def split_text_to_urls(urls):
    # remove dups while preserving order.
    dups=set()
@@ -1,5 +1,5 @@
-#!/usr/bin/env python
-# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+# -*- coding: utf-8 -*-
+
 from __future__ import (unicode_literals, division, absolute_import,
                        print_function)

@@ -18,25 +18,17 @@ from calibre_plugins.fanficfare_plugin.fanficfare.configurable import Configurat
 from calibre_plugins.fanficfare_plugin.prefs import prefs

 def get_fff_personalini():
-    if prefs['includeimages']:
-        # this is a cheat to make it easier for users.
-        return '''[epub]
-include_images:true
-keep_summary_html:true
-make_firstimage_cover:true
-''' + prefs['personal.ini']
-    else:
-        return prefs['personal.ini']
+    return prefs['personal.ini']

 def get_fff_config(url,fileform="epub",personalini=None):
    if not personalini:
        personalini = get_fff_personalini()
-    site='unknown'
+    sections=['unknown']
    try:
-        site = adapters.getConfigSectionFor(url)
+        sections = adapters.getConfigSectionsFor(url)
    except Exception as e:
-        logger.debug("Failed trying to get ini config for url(%s): %s, using section [%s] instead"%(url,e,site))
-    configuration = Configuration(site,fileform)
+        logger.debug("Failed trying to get ini config for url(%s): %s, using section %s instead"%(url,e,sections))
+    configuration = Configuration(sections,fileform)
    configuration.readfp(StringIO(get_resources("plugin-defaults.ini")))
    configuration.readfp(StringIO(personalini))

@@ -1,5 +1,5 @@
-#!/usr/bin/env python
-# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+# -*- coding: utf-8 -*-
+
 from __future__ import (unicode_literals, division,
                        print_function)

@@ -1,5 +1,5 @@
-#!/usr/bin/env python
-# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+# -*- coding: utf-8 -*-
+
 from __future__ import (unicode_literals, division, absolute_import,
                        print_function)

@@ -11,6 +11,7 @@ import logging
 logger = logging.getLogger(__name__)

 import time, traceback
+from StringIO import StringIO
    
 from calibre.utils.ipc.server import Server
 from calibre.utils.ipc.job import ParallelJob
@@ -99,7 +100,7 @@ def do_download_for_worker(book,options,notification=lambda x,y:x):
    with fffbase:
        
        from calibre_plugins.fanficfare_plugin.dialogs import (NotGoingToDownload,
-                OVERWRITE, OVERWRITEALWAYS, UPDATE, UPDATEALWAYS, ADDNEW, SKIP, CALIBREONLY)
+                OVERWRITE, OVERWRITEALWAYS, UPDATE, UPDATEALWAYS, ADDNEW, SKIP, CALIBREONLY, CALIBREONLYSAVECOL)
        from calibre_plugins.fanficfare_plugin.fanficfare import adapters, writers, exceptions
        from calibre_plugins.fanficfare_plugin.fanficfare.epubutils import get_update_data
        
@@ -151,9 +152,12 @@ def do_download_for_worker(book,options,notification=lambda x,y:x):
            outfile = book['outfile']
    
            ## No need to download at all.  Shouldn't ever get down here.
-            if options['collision'] in (CALIBREONLY):
+            if options['collision'] in (CALIBREONLY, CALIBREONLYSAVECOL):
                logger.info("Skipping CALIBREONLY 'update' down inside worker--this shouldn't be happening...")
                book['comment'] = 'Metadata collected.'
+                book['all_metadata'] = story.getAllMetadata(removeallentities=True)
+                if options['savemetacol'] != '':
+                    book['savemetacol'] = story.dump_html_metadata()
                
            ## checks were done earlier, it's new or not dup or newer--just write it.
            elif options['collision'] in (ADDNEW, SKIP, OVERWRITE, OVERWRITEALWAYS) or \
@@ -161,23 +165,20 @@ def do_download_for_worker(book,options,notification=lambda x,y:x):
    
                # preserve logfile even on overwrite.
                if 'epub_for_update' in book:
-                    (urlignore,
-                     chaptercountignore,
-                     oldchaptersignore,
-                     oldimgsignore,
-                     oldcoverignore,
-                     calibrebookmarkignore,
-                     # only logfile set in adapter, so others aren't used.
-                     adapter.logfile) = get_update_data(book['epub_for_update'])
-    
+                    
+                    adapter.logfile = get_update_data(book['epub_for_update'])[6]
                    # change the existing entries id to notid so
                    # write_epub writes a whole new set to indicate overwrite.
                    if adapter.logfile:
                        adapter.logfile = adapter.logfile.replace("span id","span notid")
                
                logger.info("write to %s"%outfile)
+                inject_cal_cols(book,story,configuration)
                writer.writeStory(outfilename=outfile, forceOverwrite=True)
                book['comment'] = 'Download %s completed, %s chapters.'%(options['fileform'],story.getMetadata("numChapters"))
+                book['all_metadata'] = story.getAllMetadata(removeallentities=True)
+                if options['savemetacol'] != '':
+                    book['savemetacol'] = story.dump_html_metadata()
                
            ## checks were done earlier, just update it.
            elif 'epub_for_update' in book and options['collision'] in (UPDATE, UPDATEALWAYS):
@@ -191,12 +192,17 @@ def do_download_for_worker(book,options,notification=lambda x,y:x):
                 adapter.oldimgs,
                 adapter.oldcover,
                 adapter.calibrebookmark,
-                 adapter.logfile) = get_update_data(book['epub_for_update'])
+                 adapter.logfile,
+                 adapter.oldchaptersmap,
+                 adapter.oldchaptersdata) = get_update_data(book['epub_for_update'])[0:9]
    
                # dup handling from fff_plugin needed for anthology updates.
                if options['collision'] == UPDATE:
                    if chaptercount == urlchaptercount:
                        book['comment']=_("Already contains %d chapters.  Reuse as is.")%chaptercount
+                        book['all_metadata'] = story.getAllMetadata(removeallentities=True)
+                        if options['savemetacol'] != '':
+                            book['savemetacol'] = story.dump_html_metadata()
                        book['outfile'] = book['epub_for_update'] # for anthology merge ops.
                        return book
    
@@ -211,11 +217,15 @@ def do_download_for_worker(book,options,notification=lambda x,y:x):
                logger.info("Do update - epub(%d) vs url(%d)" % (chaptercount, urlchaptercount))
                logger.info("write to %s"%outfile)
    
+                inject_cal_cols(book,story,configuration)
                writer.writeStory(outfilename=outfile, forceOverwrite=True)
                
                book['comment'] = _('Update %s completed, added %s chapters for %s total.')%\
                    (options['fileform'],(urlchaptercount-chaptercount),urlchaptercount)
-            
+                book['all_metadata'] = story.getAllMetadata(removeallentities=True)
+                if options['savemetacol'] != '':
+                    book['savemetacol'] = story.dump_html_metadata()
+
            if options['smarten_punctuation'] and options['fileform'] == "epub" \
                    and calibre_version >= (0, 9, 39):
                # for smarten punc
@@ -249,3 +259,22 @@ def do_download_for_worker(book,options,notification=lambda x,y:x):
            
        #time.sleep(10)
    return book
+
+## calibre's columns for an existing book are pased in and injected
+## into the story's metadata.  For convenience, we also add labels and
+## valid_entries for them in a special [injected] section that has
+## even less precedence than [defaults]
+def inject_cal_cols(book,story,configuration):
+    configuration.remove_section('injected')
+    if 'calibre_columns' in book:
+        injectini = ['[injected]']
+        extra_valid = []
+        for k, v in book['calibre_columns'].iteritems():
+            story.setMetadata(k,v['val'])
+            injectini.append('%s_label:%s'%(k,v['label']))
+            extra_valid.append(k)
+        if extra_valid: # if empty, there's nothing to add.
+            injectini.append("add_to_extra_valid_entries:,"+','.join(extra_valid))
+            configuration.readfp(StringIO('\n'.join(injectini)))
+            #print("added:\n%s\n"%('\n'.join(injectini)))
+    
@@ -235,9 +235,18 @@ connect_timeout:60.0
 ## To use this, make sure you go to the Generate Cover tab in FanFicFare
 ## config and check 'Allow generate_cover_settings from personal.ini
 ## to override'
+## You can check for an existing cover image in the metadata
+## cover_image.  cover_image can be:
+## specific -- The site has an image that is specifically this story's cover
+## first -- The first image in the story or story desc is used as cover
+## default -- A default_cover_image was used
+## old -- There was already a cover image in the epub.
+## This is useful because GC plugin can use the existing image.
 #generate_cover_settings:
 # ${category} => Buffy:? [tT]he Vampire Slayer => BuffyCover
 # ${category} => Star Trek => StarTrekCover
+# ${cover_image} => => CompleteCover
+# ${cover_image} => (specific|first|default) => CoverWithOrigImage

 ## If set false, the summary will have all html stripped.
 ## Both this and include_images must be true to get images in the
@@ -252,6 +261,11 @@ keep_summary_html:true
 ## kept.  Some sites(AO3) use <a title="Translation">Foreign language</a>
 #keep_title_attr: false

+## If a chapter range was given, use this pattern for the book title.
+## replace_metadata and include/exclude will be applied *after* this.
+## Set to empty value to disable.
+title_chapter_range_pattern:${title} (Ch ${first}-${last})
+
 ## Don't like the numbers at the start of chapter titles on some
 ## sites?  You can use strip_chapter_numbers to strip them off.  Just
 ## want to make them all look the same?  Strip them off, then add them
@@ -284,12 +298,31 @@ chapter_title_strip_pattern:^[0-9]+[\.: -]+
 ## etc
 #chapter_title_strip_pattern:^([0-9]+[\.: -]+)?(Chapter *[0-9]+[\.:, -]*)?

-## Uses a python template substitution.  The ${index} is the 'chapter'
-## number and ${title} is the chapter title, after applying
-## chapter_title_strip_pattern.  Those are the only variables available.
-## "The Beginning" => "1. The Beginning" 
+## If true, when updating an epub that already has old chapters, new
+## chapters will be marked in the TOC and chapter header by using
+## chapter_title_new_pattern and chapter_title_addnew_pattern to set the chapter.
+mark_new_chapters:false
+
+## chapter title patterns use python template substitution.  The
+## ${index} is the 'chapter' number and ${title} is the chapter title,
+## after applying chapter_title_strip_pattern.  Those are the only
+## variables available.
+
+## The basic pattern used when not using add_chapter_numbers or
+## mark_new_chapters
+chapter_title_def_pattern:${title}
+
+## Pattern used with add_chapter_numbers, but not mark_new_chapters
 chapter_title_add_pattern:${index}. ${title}

+## Pattern used with mark_new_chapters, but not add_chapter_numbers
+## (new) is just text and can be changed.
+chapter_title_new_pattern:(new) ${title}
+
+## Pattern used with add_chapter_numbers and mark_new_chapters
+## (new) is just text and can be changed.
+chapter_title_addnew_pattern:${index}. (new) ${title}
+
 ## Uses a python template substitution.  The ${title} is the default
 ## title of a new anthology, <series name> in the case of a series, or
 ## the first book title otherwise.  This is only applied to new
@@ -321,6 +354,77 @@ sort_ships:false
 ## User-agent
 user_agent:FFF/2.X

+## Added for [base_xenforoforum], but can be used with other sites,
+## too.  Limit the 'description' to the first X *characters*
+## collected.  Character count includes HTML tags, so it can be
+## non-intuitive.
+#description_limit:1000
+
+[base_efiction]
+## At the time of writing, eFiction Base adapters allow downloading
+## the whole story in bulk using the 'Print' feature. If 'bulk_load'
+## is set to 'true', both metadata and chapters can be loaded in one
+## step
+bulk_load:true
+
+[base_xenforoforum]
+## Currently only forums.spacebattles.com and forums.sufficientvelocity.com
+
+cover_exclusion_regexp:/clear.png
+
+## I saw lots of chapters name simply '1.1' etc during testing.
+strip_chapter_numbers:false
+
+## Copy title to tagsfromtitle for parsing tags.
+add_to_extra_valid_entries:,tagsfromtitle,forumtags
+
+## '.NOREPL' tells the system to *not* apply title's
+## in/exclude/replace_metadata -- Only works on include_in_ lines.
+include_in_tagsfromtitle:title.NOREPL
+
+tagsfromtitle_label:Tags from Title
+forumtags_label:Tags from Forum
+
+## might want to do this, maybe not.  Will often include category, but
+## also often include non-category stuff.
+# include_in_category:tagsfromtitle 
+
+add_to_include_metadata_pre:
+# only keep tagsfromtitle with ( or [ in.
+ tagsfromtitle=~[\[\(]
+ 
+add_to_replace_metadata:
+# remove anything outside () or []
+ tagsfromtitle=>^.*?([\(\[]([^\]\)]+)[\)\]]).*?$=>\1
+# remove () []
+ tagsfromtitle=>[\(\)\[\]]=>
+# change (spaces)slash(or semicolon)(spaces) to comma
+ tagsfromtitle=> *[/;] *=>,
+ tagsfromtitle=> [xX] =>,
+ 
+# remove [] or () blocks and leading/trailing spaces/dashes/colons
+ title=>[-: ]*[\(\[]([^\]\)]+)[\)\]][-: ]*=>
+# remove 'Thread' and the next word, usually "Thread 2", "Thread
+# four", "Thread iv", etc
+ title,tagsfromtitle=>[-: ]*[Tt]hread [^ ]+[-: ]*=>
+
+add_to_extra_titlepage_entries:,tagsfromtitle,forumtags
+
+## '.SPLIT' tells the system to split by ','
+add_to_include_subject_tags:,tagsfromtitle.SPLIT
+## Or to also include forumtags add this to personal.ini:
+#add_to_include_subject_tags:,forumtags
+
+## base_xenforoforum reads Published and Updated datetimes from
+## Threadmarks if used, or from the posted & updated times of the
+## 'first' post if no threadmarks.
+datePublished_format:%%Y-%%m-%%d %%H:%%M:%%S
+dateUpdated_format:%%Y-%%m-%%d %%H:%%M:%%S
+
+## Only take the first X characters of the 'first' post to use as
+## the description.
+description_limit:500
+
 ## Each output format has a section that overrides [defaults]
 [html]

@@ -442,12 +546,12 @@ output_css:
 ## include images from img tags in the body and summary of
 ## stories.  Images will be converted to jpg for size if possible.
 ## include_images is *only* available in epub and html output format.
-#include_images:false
+include_images:true

 ## If set, the first image found will be made the cover image.  If
 ## keep_summary_html is true, any images in summary will be before any
 ## in chapters.
-#make_firstimage_cover: false
+make_firstimage_cover: true

 ## If set, the epub will never have a cover, even include_images is on
 ## and the site has specific cover images.
@@ -569,6 +673,8 @@ extratags: FanFiction,Testing,HTML
 ## doesn't like that.  If do_update_hook is uncommented and set true,
 ## the adapter will discard all existing chapters from the newest one
 ## on when updating to enforce accurate chapters.
+## Starting July 2015, FFF stores chapter URLs in the chapter files.
+## Stories downloaded after that shouldn't need this setting anymore.
 #do_update_hook:false

 ## AO3 adapter defines a few extra metadata entries.
@@ -584,6 +690,18 @@ hits_label:Hits
 collections_label:Collections
 bookmarks_label:Bookmarks

+## AO3 doesn't have anything it calls 'genre'.  The adapter used to be
+## hardcoded to include the site specific metadata freeformtags &
+## ao3categories in the standard metadata field genre.  By making it
+## configurable, users can change it.
+include_in_genre: freeformtags, ao3categories
+
+## AO3 uses the word 'category' differently than most sites.  The
+## adapter used to be hardcoded to include the site specific metadata
+## fandom in the standard metadata field category.  By making it
+## configurable, users can change it.
+include_in_category:fandoms
+
 ## freeformtags was previously typo'ed as freefromtags.  This way,
 ## freefromtags will still work for people who've used it.
 include_in_freefromtags:freeformtags
@@ -595,7 +713,7 @@ include_in_freefromtags:freeformtags
 #extra_subject_tags:fandoms,freeformtags,ao3categories

 ## AO3 chapters can include several different types of notes.  We've
-## traditional included them all in the chapter text, but this allows
+## traditionally included them all in the chapter text, but this allows
 ## you to customize which you include.  Copy this parameter to your
 ## personal.ini and list the ones you don't want.
 #exclude_notes:authorheadnotes,chaptersummary,chapterheadnotes,chapterfootnotes,authorfootnotes
@@ -745,11 +863,6 @@ extraships:Spike/Buffy
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true

-## Virtually all eFiction-based sites allow downloading the whole story in
-## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
-## metadata and chapters can be loaded in one step
-bulk_load:true
-
 [dramione.org]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -930,11 +1043,6 @@ extraships:Harry Potter/Hermione Granger
 #username:YourName
 #password:yourpassword

-## Virtually all eFiction-based sites allow downloading the whole story in
-## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
-## metadata and chapters can be loaded in one step
-bulk_load:true
-
 extra_valid_entries: readings,romance
 extra_titlepage_entries: readings,romance
 readings_label: Readings
@@ -1024,6 +1132,12 @@ extra_valid_entries:size
 # don't show twitter icon.
 cover_exclusion_regexp:/res/css/bir.png

+[forums.spacebattles.com]
+## see [base_xenforoforum]
+
+[forums.sufficientvelocity.com]
+## see [base_xenforoforum]
+
 [grangerenchanted.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -1070,6 +1184,18 @@ extracategories:In Death
 ## cover image.  This lets you exclude them.
 cover_exclusion_regexp:/public/style_emoticons/.*

+[it-could-happen.net]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+## RPF == http://en.wikipedia.org/wiki/Real_person_fiction
+## Site dedicated to these categories/characters/ships
+extracategories:Glee RPF
+extracharacters:Darren Criss, Chris Colfer
+extraships:Darren Criss/Chris Colfer
+
 [ksarchive.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:Star Trek
@@ -1089,11 +1215,6 @@ eroticatags_label:Erotica Tags
 extra_titlepage_entries: eroticatags

 [lotrfanfiction.com]
-## Virtually all eFiction-based sites allow downloading the whole story in
-## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
-## metadata and chapters can be loaded in one step
-bulk_load:true
-
 extra_valid_entries: readings
 readings_label: Readings

@@ -1147,6 +1268,12 @@ extracategories:NCIS
 extracategories:Buffy: The Vampire Slayer
 extracharacters:Willow

+[ninelives.dark-solace.org]
+## Site dedicated to these categories/characters/ships
+extracategories:The Walking Dead
+extracharacters:Carol,Daryl
+extraships:Carol/Daryl
+
 [nocturnal-light.net]
 ## Extra metadata that this adapter knows about.  See [dramione.org]
 ## for examples of how to use them.
@@ -1343,11 +1470,6 @@ extracategories:Transgender
 ## confirm they are adult for adult content.
 #is_adult:true

-## Virtually all eFiction-based sites allow downloading the whole story in
-## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
-## metadata and chapters can be loaded in one step
-bulk_load:true
-
 [thehexfiles.net]
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter
@@ -1367,11 +1489,6 @@ extraships:Harry Potter/Draco Malfoy
 ## personal.ini, not defaults.ini.
 #is_adult:true

-## Virtually all eFiction-based sites allow downloading the whole story in
-## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
-## metadata and chapters can be loaded in one step
-bulk_load:true
-
 ## Site dedicated to these categories/characters/ships
 extracategories:Criminal Minds

@@ -1381,11 +1498,6 @@ extracategories:Criminal Minds
 ## personal.ini, not defaults.ini.
 #is_adult:true

-## Virtually all eFiction-based sites allow downloading the whole story in
-## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
-## metadata and chapters can be loaded in one step
-bulk_load:true
-
 extra_valid_entries: readings,challenge
 extra_titlepage_entries: readings,challenge
 challenge_label: Challenge
@@ -1454,6 +1566,8 @@ extra_titlepage_entries:readings,awards
 awards_label:Awards
 readings_label:Readings

+## some sites include images that we don't ever want becoming the
+## cover image.  This lets you exclude them.
 cover_exclusion_regexp:art/.*Awards.jpg

 [voracity2.e-fic.com]
@@ -1650,6 +1764,10 @@ dateUpdated_format:%%Y-%%m-%%d %%H:%%M:%%S
 ## to on, but can be switched off if it is found to cause problems.
 fix_fimf_blockquotes:true

+## some sites include images that we don't ever want becoming the
+## cover image.  This lets you exclude them.
+cover_exclusion_regexp:/images/emoticons/
+
 ## Site dedicated to these categories/characters/ships
 extracategories:My Little Pony: Friendship is Magic

@@ -1733,12 +1851,38 @@ extraships:InuYasha/Kagome
 ## Site dedicated to these categories/characters/ships
 extracategories:Lord of the Rings

-## Virtually all eFiction-based sites allow downloading the whole story in
-## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
-## metadata and chapters can be loaded in one step
-bulk_load:true
+[www.masseffect2.in]
+## Site dedicated to this fandom.
+extracategories: Mass Effect
+
+## Ucoz.com engine, upon which MassEffect2.in is based, imposes an unspecified limit on request frequency.
+## Reports vary from `5 requests per second' to `2 requests per second for more than 10 per minute'.
+## With default settings, a several-hours IP ban may follow, so set it higher.
+slow_down_sleep_time: 2
+
+## Whether to exclude editor signature from the bottom of chapter text.
+exclude_editor_signature: false
+
+## Stories on the site almost never have cover image, and for the stories which do,
+## this may be adjusted in `personal.ini' before downloading.
+never_make_cover: true
+
+## Titles for ratings identified by 1- or 2-letter codes from `ERATING system'
+## (`система Р.Е.Й.Т.И.Н.Г.').  MassEffect2.in and some other sites adopted it,
+## but changed titles and update them occasionally.
+rating_titles: R=RESTRICTED (16+), E=EXEMPT (18+), I=ART HOUSE, T=To every, A=IN=Иной мир, Nn=Новый мир, G=О\, Господи!
+adult_ratings: E,R

 [www.mediaminer.org]
+dateUpdated_format:%%Y-%%m-%%d %%H:%%M:%%S
+## Note that mediaminer doesn't give datePublished on the story's
+## index page--it's collected from the earliest uploaded chapter.  So
+## it's not available when only fetching metadata.
+datePublished_format:%%Y-%%m-%%d %%H:%%M:%%S
+
+## some sites include images that we don't ever want becoming the
+## cover image.  This lets you exclude them.
+cover_exclusion_regexp:/img/rss.png

 [www.midnightwhispers.ca]
 ## Some sites do not require a login, but do require the user to
@@ -1806,6 +1950,13 @@ extracategories:Psych
 ## Site dedicated to these categories/characters/ships
 extracategories:Queer as Folk

+[quotev.com]
+extra_valid_entries:pages,readers,reads,favorites
+pages_label:Pages
+readers_label:Readers
+reads_label:Reads
+favorites_label:Favorites
+
 [www.restrictedsection.org]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -1,5 +1,5 @@
-#!/usr/bin/env python
-# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+# -*- coding: utf-8 -*-
+
 from __future__ import (unicode_literals, division, absolute_import,
                        print_function)

@@ -15,9 +15,32 @@ import copy
 from calibre.utils.config import JSONConfig
 from calibre.gui2.ui import get_gui

-from calibre_plugins.fanficfare_plugin.dialogs import UPDATE
+from calibre_plugins.fanficfare_plugin.dialogs import SAVE_UPDATE
 from calibre_plugins.fanficfare_plugin.common_utils import get_library_uuid

+# Show translated strings, but save the same string in prefs so your
+# prefs are the same in different languages.
+YES=_('Yes, Always')
+SAVE_YES='Yes'
+YES_IF_IMG=_('Yes, if EPUB has a cover image')
+SAVE_YES_IF_IMG='Yes, if img'
+YES_UNLESS_IMG=_('Yes, unless FanFicFare found a cover image')
+SAVE_YES_UNLESS_IMG='Yes, unless img'
+NO=_('No')
+SAVE_NO='No'
+calcover_save_options = {
+    YES:SAVE_YES,
+    SAVE_YES:YES,
+    YES_IF_IMG:SAVE_YES_IF_IMG,
+    SAVE_YES_IF_IMG:YES_IF_IMG,
+    YES_UNLESS_IMG:SAVE_YES_UNLESS_IMG,
+    SAVE_YES_UNLESS_IMG:YES_UNLESS_IMG,
+    NO:SAVE_NO,
+    SAVE_NO:NO,
+    }
+updatecalcover_order=[YES,YES_IF_IMG,NO]
+gencalcover_order=[YES,YES_UNLESS_IMG,NO]
+
 # if don't have any settings for FanFicFarePlugin, copy from
 # predecessor FanFictionDownLoaderPlugin.
 FFDL_PREFS_NAMESPACE = 'FanFictionDownLoaderPlugin'
@@ -28,6 +51,7 @@ PREFS_KEY_SETTINGS = 'settings'
 # take from here.
 default_prefs = {}
 default_prefs['personal.ini'] = get_resources('plugin-example.ini')
+default_prefs['cal_cols_pass_in'] = False
 default_prefs['rejecturls'] = ''
 default_prefs['rejectreasons'] = '''Sucked
 Boring
@@ -35,7 +59,6 @@ Dup from another site'''
 default_prefs['reject_always'] = False

 default_prefs['updatemeta'] = True
-default_prefs['updatecover'] = False
 default_prefs['updateepubcover'] = False
 default_prefs['keeptags'] = False
 default_prefs['suppressauthorsort'] = False
@@ -46,14 +69,14 @@ default_prefs['autoconvert'] = False
 default_prefs['urlsfromclip'] = True
 default_prefs['updatedefault'] = True
 default_prefs['fileform'] = 'epub'
-default_prefs['collision'] = UPDATE
+default_prefs['collision'] = SAVE_UPDATE
 default_prefs['deleteotherforms'] = False
 default_prefs['adddialogstaysontop'] = False
-default_prefs['includeimages'] = False
 default_prefs['lookforurlinhtml'] = False
 default_prefs['checkforseriesurlid'] = True
 default_prefs['checkforurlchange'] = True
 default_prefs['injectseries'] = False
+default_prefs['matchtitleauth'] = True
 default_prefs['smarten_punctuation'] = False
 default_prefs['show_est_time'] = False

@@ -62,7 +85,13 @@ default_prefs['read_lists'] = ''
 default_prefs['addtolists'] = False
 default_prefs['addtoreadlists'] = False
 default_prefs['addtolistsonread'] = False
+default_prefs['autounnew'] = False

+default_prefs['updatecalcover'] = None
+default_prefs['gencalcover'] = SAVE_YES
+default_prefs['updatecover'] = False
+default_prefs['calibre_gen_cover'] = False
+default_prefs['plugin_gen_cover'] = True
 default_prefs['gcnewonly'] = False
 default_prefs['gc_site_settings'] = {}
 default_prefs['allow_gc_from_ini'] = True
@@ -72,11 +101,13 @@ default_prefs['countpagesstats'] = []
 default_prefs['wordcountmissing'] = False

 default_prefs['errorcol'] = ''
+default_prefs['savemetacol'] = ''
 default_prefs['custom_cols'] = {}
 default_prefs['custom_cols_newonly'] = {}
 default_prefs['allow_custcol_from_ini'] = True

 default_prefs['std_cols_newonly'] = {}
+default_prefs['set_author_url'] = True

 default_prefs['imapserver'] = ''
 default_prefs['imapuser'] = ''
@@ -84,6 +115,8 @@ default_prefs['imappass'] = ''
 default_prefs['imapsessionpass'] = False
 default_prefs['imapfolder'] = 'INBOX'
 default_prefs['imapmarkread'] = True
+default_prefs['auto_reject_from_email'] = False
+default_prefs['download_from_email_immediately'] = False

 def set_library_config(library_config,db):
    db.prefs.set_namespaced(PREFS_NAMESPACE,
@@ -0,0 +1,6 @@
+# Translations
+
+If you're interested in helping provide translations for this project,
+please use the
+[Transifex](https://www.transifex.com/projects/p/calibre-plugins/resources/)
+website to add translations to this, or other calibre plugins that support it.
@@ -134,6 +134,13 @@ import adapter_fhsarchivecom
 import adapter_fanfictionjunkiesde
 import adapter_devianthearts
 import adapter_tgstorytimecom
+import adapter_itcouldhappennet
+import adapter_forumsspacebattlescom
+import adapter_forumssufficientvelocitycom
+import adapter_questionablequestingcom
+import adapter_ninelivesdarksolaceorg
+import adapter_masseffect2in
+import adapter_quotevcom

 ## This bit of complexity allows adapters to be added by just adding
 ## importing.  It eliminates the long if/else clauses we used to need
@@ -193,19 +200,35 @@ def getAdapter(config,url,anyurl=False):
    # No adapter found.
    raise exceptions.UnknownSite( url, [cls.getSiteDomain() for cls in __class_list] )

-def getConfigSections():
+def getSiteSections():
+    # doesn't include base sections. Sections rather than site DNS because of squidge/peja
    return [cls.getConfigSection() for cls in __class_list]

+def getConfigSections():
+    # does include base sections.
+    sections = set()
+    for cls in __class_list:
+        sections.update(cls.getConfigSections())
+    return sections
+
+def get_bulk_load_sites():
+    # for now, all eFiction Base adapters are assumed to allow bulk_load.
+    sections = set()
+    for cls in filter( lambda x : issubclass(x,base_efiction_adapter.BaseEfictionAdapter),
+                       __class_list):
+        sections.update( [ x.replace('www.','') for x in cls.getConfigSections() ] )
+    return sections
+
 def getSiteExamples():
    l=[]
    for cls in sorted(__class_list, key=lambda x : x.getConfigSection()):
        l.append((cls.getConfigSection(),cls.getSiteExampleURLs().split()))
    return l

-def getConfigSectionFor(url):
+def getConfigSectionsFor(url):
    (cls,fixedurl) = getClassFor(url)
    if cls:
-        return cls.getConfigSection()
+        return cls.getConfigSections()

    # No adapter found.
    raise exceptions.UnknownSite( url, [cls.getSiteDomain() for cls in __class_list] )
@@ -230,7 +230,6 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
        fandoms = a.findAll('a',{'class':"tag"})
        for fandom in fandoms:
            self.story.addToList('fandoms',fandom.string)
-            self.story.addToList('category',fandom.string)
 		
        a = metasoup.find('dd',{'class':"warning tags"})
        if a != None:
@@ -243,7 +242,6 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
            genres = a.findAll('a',{'class':"tag"})
            for genre in genres:
                self.story.addToList('freeformtags',genre.string)
-                self.story.addToList('genre',genre.string)
                
        a = metasoup.find('dd',{'class':"category tags"})
        if a != None:
@@ -251,7 +249,6 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
            for genre in genres:
                if genre != "Gen":
                    self.story.addToList('ao3categories',genre.string)
-                    self.story.addToList('genre',genre.string)
 		
        a = metasoup.find('dd',{'class':"character tags"})
        if a != None:
@@ -16,321 +16,26 @@
 #

 # Software: eFiction
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
+from base_efiction_adapter import BaseEfictionAdapter

-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
+class DarkSolaceOrgAdapter(BaseEfictionAdapter):

-from base_adapter import BaseSiteAdapter,  makeDate
+    @staticmethod
+    def getSiteDomain():
+        return 'dark-solace.org'

+    @classmethod
+    def getPathToArchive(self):
+        return '/elysian'
+
+    @classmethod
+    def getSiteAbbrev(self):
+        return 'dksl'
+        
+    @classmethod
+    def getDateFormat(self):
+        return "%B %d, %Y"
+                           
 def getClass():
    return DarkSolaceOrgAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class DarkSolaceOrgAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/elysian/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','dksl')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%B %d, %Y"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'dark-solace.org'
-        
-    @classmethod
-    def getAcceptDomains(cls):
-        return ['www.dark-solace.org','dark-solace.org']
-
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://"+cls.getSiteDomain()+"/elysian/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://")+"(www\.)?"+re.escape(self.getSiteDomain()+"/elysian/viewstory.php?sid=")+r"\d+$"
-
-    ## Login seems to be reasonably standard across eFiction sites.
-    def needToLoginCheck(self, data):
-        if 'This story contains adult content not suitable for children' in data \
-                or "That password doesn't match the one in our database" in data \
-                or "Registered Users Only" in data:
-            return True
-        else:
-            return False
-        
-    def performLogin(self, url):
-        params = {}
-
-        if self.password:
-            params['penname'] = self.username
-            params['password'] = self.password
-        else:
-            params['penname'] = self.getConfig("username")
-            params['password'] = self.getConfig("password")
-        params['action'] = 'login'
-        params['submit'] = 'Submit'
    
-        loginUrl = 'http://www.' + self.getSiteDomain() + '/elysian/user.php'
-        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
-                                                              params['penname']))
-    
-        d = self._postUrl(loginUrl, params)
-    
-        if "Member Account" not in d : #User Account Page
-            logger.info("Failed to login to URL %s as %s, or have no authorization to access the story" % (loginUrl, params['penname']))
-            raise exceptions.FailedToLogin(url,params['penname'])
-            return False
-        else:
-            return True
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-        
-        if self.is_adult or self.getConfig("is_adult"):
-            # Weirdly, different sites use different warning numbers.
-            # If the title search below fails, there's a good chance
-            # you need a different number.  print data at that point
-            # and see what the 'click here to continue' url says.
-            addurl = "&ageconsent=ok&warning=5"
-        else:
-            addurl=""
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url+'&index=1'+addurl
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        if self.needToLoginCheck(data):
-            # need to log in for this one.
-            self.performLogin(url)
-            data = self._fetchUrl(url)
-            
-        m = re.search(r"'viewstory.php\?sid=\d+((?:&amp;ageconsent=ok)?&amp;warning=\d+)'",data)
-        if m != None:
-            if self.is_adult or self.getConfig("is_adult"):
-                # We tried the default and still got a warning, so
-                # let's pull the warning number from the 'continue'
-                # link and reload data.
-                addurl = m.group(1)
-                # correct stupid &amp; error in url.
-                addurl = addurl.replace("&amp;","&")
-                url = self.url+'&index=1'+addurl
-                logger.debug("URL 2nd try: "+url)
-
-                try:
-                    data = self._fetchUrl(url)
-                except urllib2.HTTPError, e:
-                    if e.code == 404:
-                        raise exceptions.StoryDoesNotExist(self.url)
-                    else:
-                        raise e    
-            else:
-                raise exceptions.AdultCheckRequired(self.url)
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        ## Title and author
-        div = soup.find('div', {'id' : 'pagetitle'})
-        
-        aut = div.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',aut['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/elysian/'+aut['href'])
-        self.story.setMetadata('author',aut.string)
-        aut.extract()
-
-        # first a tag in pagetitle is title
-        self.story.setMetadata('title',stripHTML(div.find('a')))
-        div.find('a').extract()
-        # only thing left in div(pagetitle) now should be 'by' and rating.
-        rating = stripHTML(div)
-        if '[' in rating:
-            self.story.setMetadata('rating', rating[rating.index('[')+1:-1])
-
-        for chapa in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+
-                                                       self.story.getMetadata('storyId')+'&chapter=\d+')):
-            self.chapterUrls.append((stripHTML(chapa),'http://'+self.host+'/elysian/'+chapa['href']))
-        
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
-        storylink = asoup.find('a', href=re.compile(r'viewstory.php\?sid='+
-                                                    self.story.getMetadata('storyId')+'($|[^\d])'))
-        # author's story list is paginated if there's a pagelinks div.
-        # Only need to look in it if the story wasn't on the first page.
-        pagelinks = asoup.find('div',{'id':'pagelinks'})
-        if pagelinks and storylink==None:
-            authpageslist = pagelinks.findAll('a',href=re.compile(r'action=storiesby'))
-            for page in authpageslist[1:]: # skip first, already checked above.
-                asoup = bs.BeautifulSoup(self._fetchUrl('http://'+self.host+'/elysian/'+page['href']))
-                storylink = asoup.find('a', href=re.compile(r'viewstory.php\?sid='+
-                                                            self.story.getMetadata('storyId')+'($|[^\d])'))
-                if storylink:
-                    break
-
-        if not storylink:
-            raise exceptions.FailedToDownload("Unable to find story metadata on author's page(s)")
-        
-        metalist = storylink.parent.parent
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-                
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
-        labels = metalist.findAll('span', {'class' : 'label'})
-        for labelspan in labels:
-            label = labelspan.text
-            value = labelspan.nextSibling
-
-            if 'Summary' in label:
-                ## Everything until the next span class='label'
-                svalue = ""
-                while value and not (defaultGetattr(value,'class') == 'label' or "Chapters: " in stripHTML(value)):
-                    svalue += str(value)
-                    value = value.nextSibling
-                self.setDescription(url,svalue)
-                #self.story.setMetadata('description',stripHTML(svalue))
-
-            if 'Word count' in label:
-                self.story.setMetadata('numWords', value)
-
-            if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-                for cat in cats:
-                    self.story.addToList('category',cat.string)
-
-            if 'Characters' in label:
-                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-                for char in chars:
-                    self.story.addToList('characters',char.string)
-
-            if 'Genre' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
-                for genre in genres:
-                    self.story.addToList('genre',genre.string)
-
-            if 'Warnings' in label:
-                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
-                for warning in warnings:
-                    self.story.addToList('warnings',warning.string)
-
-            if 'Completed' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-            
-            if 'Updated' in label:
-                # there's a stray [ at the end.
-                #value = value[0:-1]
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-
-        try:
-            # Find Series name from series URL.
-            a = metalist.find('a', href=re.compile(r"series.php\?seriesid=\d+"))
-            series_name = a.string
-            series_url = 'http://'+self.host+'/elysian/'+a['href']
-
-            # use BeautifulSoup HTML parser to make everything easier to find.
-            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
-            storylink = seriessoup.find('a', href=re.compile(r'viewstory.php\?sid='+
-                                                             self.story.getMetadata('storyId')+'($|[^\d])'))
-            if storylink and storylink.parent and storylink.parent['class'] != 'title': # in case of links inside story summaries.
-                storylink = None
-
-            offset = 0
-            # series story list is paginated if there's a pagelinks div.
-            # Only need to look in it if the story wasn't on the first page.
-            pagelinks = seriessoup.find('div',{'id':'pagelinks'})
-            if pagelinks and storylink==None:
-                authpageslist = pagelinks.findAll('a',href=re.compile(r'offset='))
-                for page in authpageslist[1:]: # skip first, already checked above.
-                    seriessoup = bs.BeautifulSoup(self._fetchUrl('http://'+self.host+'/elysian/'+page['href']))
-                    storylink = seriessoup.find('a', href=re.compile(r'viewstory.php\?sid='+
-                                                                     self.story.getMetadata('storyId')+'($|[^\d])'))
-                    if storylink and storylink.parent and storylink.parent['class'] != 'title': # in case of links inside story summaries.
-                        storylink = None
-                    if storylink:
-                        offset = int(page['href'].split('=')[-1]) # offset is last.
-                        break
-
-            # for reasons I don't understand, searching for story
-            # links by regex wasn't working reliably.  It was missing
-            # the javascript links sometimes.  This is cleaner anyway.
-            for i, div in enumerate(seriessoup.findAll('div', {'class':'title'})):
-                a = div.find('a') # first a is story link.
-                # skip 'report this' and 'TOC' links
-                if a == storylink:
-                    self.setSeries(series_name, 1+i+offset)
-                    self.story.setMetadata('seriesUrl',series_url)
-                    break
-            
-        except Exception, e:
-            logger.debug("Series parsing failed: %s"%e)
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulSoup(self._fetchUrl(url))
-        
-        div = soup.find('div', {'id' : 'story'})
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)
@@ -38,8 +38,9 @@ class DramioneOrgAdapter(BaseSiteAdapter):
    def __init__(self, config, url):
        BaseSiteAdapter.__init__(self, config, url)

-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
+        self.decode = ["utf8",
+                       "Windows-1252",]
+                               # 1252 is a superset of iso-8859-1.
                               # Most sites that claim to be
                               # iso-8859-1 (and some that claim to be
                               # utf8) are really windows-1252.
@@ -104,7 +104,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
            data = self._fetchUrl(url)
            #logger.debug("\n===================\n%s\n===================\n"%data)
            soup = self.make_soup(data)
-        except urllib2.HTTPError, e:
+        except urllib2.HTTPError as e:
            if e.code == 404:
                raise exceptions.StoryDoesNotExist(url)
            else:
@@ -135,11 +135,15 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
                                                chapcount+1)
                logger.debug('=Trying newer chapter: %s' % tryurl)
                newdata = self._fetchUrl(tryurl)
-                if "not found. Please check to see you are not using an outdated url." \
-                        not in newdata:
+                if "not found. Please check to see you are not using an outdated url." not in newdata \
+                        and "This request takes too long to process, it is timed out by the server." not in newdata:
                    logger.debug('=======Found newer chapter: %s' % tryurl)
                    soup = self.make_soup(newdata)
-            except:
+            except urllib2.HTTPError as e:
+                if e.code == 503:
+                    raise e
+            except e:
+                logger.warn("Caught an exception reading URL: %s sleeptime(%s) Exception %s."%(unicode(url),sleeptime,unicode(e)))
                pass
            
        # Find authorid and URL from... author url.
@@ -135,7 +135,7 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
        self.story.setMetadata('title', stripHTML(titleh4.a))

        # Find authorid and URL from... author url.
-        a = soup.find('span',{'class':'author'}).find('a', href=re.compile(r"^/author/\d+"))
+        a = soup.find('span',{'class':'author'}).find('a', href=re.compile(r"^/a/"))
        self.story.setMetadata('authorId',a['href'].split('/')[2])
        self.story.setMetadata('authorUrl','http://'+self.host+a['href'])
        self.story.setMetadata('author',a.string)
@@ -176,7 +176,13 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
            if coverurl.startswith('//'): # fix for img urls missing 'http:'
                coverurl = "http:"+coverurl
            if get_cover:
-                self.setCoverImage(self.url,coverurl)
+                # try setting from href, if fails, try using the img src
+                if self.setCoverImage(self.url,coverurl)[0] == "failedtoload":
+                    img = storyImage.find('img')
+                    # try src, then data-src, then leave None.
+                    coverurl = img.get('src',img.get('data-src',None))
+                    if coverurl:
+                        self.setCoverImage(self.url,coverurl)

            coverSource = storyImage.find('a', {'class':'source'})
            if coverSource:
@@ -0,0 +1,35 @@
+#  -*- coding: utf-8 -*-
+
+# Copyright 2015 FanFicFare team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from base_xenforoforum_adapter import BaseXenForoForumAdapter
+
+def getClass():
+    return ForumsSpacebattlesComAdapter
+
+class ForumsSpacebattlesComAdapter(BaseXenForoForumAdapter):
+
+    def __init__(self, config, url):
+        BaseXenForoForumAdapter.__init__(self, config, url)
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','fsb')
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'forums.spacebattles.com'
+
@@ -0,0 +1,34 @@
+#  -*- coding: utf-8 -*-
+
+# Copyright 2015 FanFicFare team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from base_xenforoforum_adapter import BaseXenForoForumAdapter
+
+def getClass():
+    return ForumsSufficientVelocityComAdapter
+
+class ForumsSufficientVelocityComAdapter(BaseXenForoForumAdapter):
+
+    def __init__(self, config, url):
+        BaseXenForoForumAdapter.__init__(self, config, url)
+
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','fsv')
+
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'forums.sufficientvelocity.com'
@@ -176,7 +176,7 @@ class GrangerEnchantedCom(BaseSiteAdapter):
            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
            
        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
+        soup = self.make_soup(data)
        # print data

        # Now go hunting for all the meta data and the chapter list.
@@ -217,11 +217,10 @@ class GrangerEnchantedCom(BaseSiteAdapter):
            if 'Summary' in label:
                ## Everything until the next span class='label'
                svalue = ""
-                while not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
+                while value and not defaultGetattr(value,'class') == 'label' and '<span class="label">' not in unicode(value):
+                    svalue += unicode(value)
                    value = value.nextSibling
                self.setDescription(url,svalue)
-                #self.story.setMetadata('description',stripHTML(svalue))

            if 'Rated' in label:
                self.story.setMetadata('rating', value)
@@ -271,7 +270,7 @@ class GrangerEnchantedCom(BaseSiteAdapter):
            series_url = 'http://'+self.host+'/'+self.section+'/'+a['href']

            # use BeautifulSoup HTML parser to make everything easier to find.
-            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            seriessoup = self.make_soup(self._fetchUrl(series_url))
            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
            i=1
@@ -300,8 +299,7 @@ class GrangerEnchantedCom(BaseSiteAdapter):

        logger.debug('Getting chapter text from: %s' % url)

-        soup = bs.BeautifulSoup(self._fetchUrl(url),
-                                selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        soup = self.make_soup(self._fetchUrl(url))
        
        div = soup.find('div', {'id' : 'story1'})

@@ -98,6 +98,26 @@ class HPFandomNetAdapterAdapter(BaseSiteAdapter): # XXX

        # Now go hunting for all the meta data and the chapter list.
        
+        # set first URL
+        a = soup.find('a', href=re.compile(r"viewstory.php\?sid=\d+"))
+        self._setURL('http://' + self.getSiteDomain() + '/eff/'+a['href'])
+        if self.parsedUrl.query.split('=',)[1] != self.story.getMetadata('storyId'):
+            self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+            url = self.url
+            logger.debug("reset URL: "+url)
+            try:
+                data = self._fetchUrl(url)
+            except urllib2.HTTPError, e:
+                if e.code == 404:
+                    raise exceptions.StoryDoesNotExist(self.url)
+                else:
+                    raise e
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            soup = bs.BeautifulSoup(data)
+            
+#        self.story.setMetadata('storyId', re.compile(self.getSiteURLPattern()).match(a).group('storyId'))
+        
        # Find authorid and URL from... author url.
        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
        self.story.setMetadata('authorId',a['href'].split('=')[1])
@@ -0,0 +1,46 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2015 FanFicFare team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Software: eFiction
+import re
+from base_efiction_adapter import BaseEfictionAdapter
+
+class ItCouldHappenNetSiteAdapter(BaseEfictionAdapter):
+
+    @staticmethod
+    def getSiteDomain():
+        return 'it-could-happen.net'
+
+    @classmethod
+    def getSiteAbbrev(seluuf):
+        return 'ich'
+
+    @classmethod
+    def getDateFormat(self):
+        return "%B %d, %Y"
+
+    def handleMetadataPair(self, key, value):
+        # This site is all one 'category' as it's usually defined and
+        # uses Category for what is usually genre.
+        if key == 'Categories':
+            for val in re.split("\s*,\s*", value):
+                self.story.addToList('genre', val)
+        else:
+            super(ItCouldHappenNetSiteAdapter, self).handleMetadataPair(key, value)
+
+def getClass():
+    return ItCouldHappenNetSiteAdapter
@@ -0,0 +1,719 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2015 FanFicFare team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import bs4
+import datetime
+import logging
+import re
+import urllib2
+
+from ..htmlcleanup import removeEntities, stripHTML
+from .. import exceptions as exceptions
+from base_adapter import BaseSiteAdapter, makeDate
+
+
+_logger = logging.getLogger(__name__)
+
+
+def getClass():
+    """Returns adapter class defined in this module."""
+    return MassEffect2InAdapter
+
+
+class ParsingError(Exception):
+    """Indicates an error while parsing web page content."""
+    def __init__(self, message):
+        Exception.__init__(self)
+        self.message = message
+
+    def __str__(self):
+        return self.message
+
+
+class MassEffect2InAdapter(BaseSiteAdapter):
+    """
+    Provides support for MassEffect2.in site as story source.
+    Can be used as a template for sites build upon Ucoz.com engine (until no base class extracted).
+    Specializations:
+        1) Russian content (date format, genre names, etc.);
+        2) original `E.R.A.T.I.N.G.' rating scale, used by masseffect2.in
+           and some affiliated sites, denoted with images;
+        3) editor signatures an an option to remove them.
+    """
+
+    WORD_PATTERN = re.compile(u'\w+', re.UNICODE)
+    DOCUMENT_ID_PATTERN = re.compile(u'\d+-\d+-\d+-\d+')
+    SITE_LANGUAGE = u'Russian'
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["utf8"]
+
+        self.story.setMetadata('siteabbrev', 'me2in')
+        self.story.setMetadata('storyId', self._getDocumentId(self.url))
+
+        self._setURL(self._makeDocumentUrl(self.story.getMetadata('storyId')))
+
+        self._chapters = {}
+        self._parsingConfiguration = None
+
+    # Must be @staticmethod, not @classmethod!
+    @staticmethod
+    def getSiteDomain():
+        return 'www.masseffect2.in'
+
+    @classmethod
+    def getSiteExampleURLs(cls):
+        return u' '.join([cls._makeDocumentUrl('19-1-0-1234'),
+                          cls._makeDocumentUrl('24-1-0-4321')])
+
+    def getSiteURLPattern(self):
+        return re.escape(self._makeDocumentUrl('')) + self.DOCUMENT_ID_PATTERN.pattern
+
+    def use_pagecache(self):
+        """Allows use of downloaded page cache.  It is essential for this
+        adapter, because the site does not offers chapter URL list, and many
+        pages have to be fetched and parsed repeatedly."""
+        return True
+
+    def extractChapterUrlsAndMetadata(self):
+        """Extracts chapter URLs and story metadata.  Actually downloads all
+        chapters, which is not exactly right, but necessary due to technical
+        limitations of the site."""
+
+        def followChapters(starting, forward=True):
+            if forward:
+                url = starting.getNextChapterUrl()
+            else:
+                url = starting.getPreviousChapterUrl()
+            if url:
+                url = self._makeDocumentUrl(self._getDocumentId(url))
+                following = self._makeChapter(url)
+                # Do not follow links to related, but different stories (prequels or sequels).
+                if not following.isFromStory(starting.getHeading()):
+                    return
+                if forward:
+                    yield following
+                for chapter in followChapters(following, forward):
+                    yield chapter
+                if not forward:
+                    yield following
+
+        try:
+            startingChapter = self._makeChapter(self.url)
+        except urllib2.HTTPError, error:
+            if error.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            raise
+
+        # We only have one date for each chapter and assume the oldest one
+        # to be publication date and the most recent one to be update date.
+        datePublished = datetime.datetime.max
+        dateUpdated = datetime.datetime.min
+        wordCount = 0
+        # We aim at counting chapters, not chapter parts.
+        chapterCount = 0
+        storyInProgress = False
+
+        chapters = \
+            list(followChapters(startingChapter, forward=False)) + \
+            [startingChapter] + \
+            list(followChapters(startingChapter, forward=True))
+
+        headings = [chapter.getHeading() for chapter in chapters]
+        largestCommonPrefix = _getLargestCommonPrefix(*headings)
+        prefixLength = len(largestCommonPrefix)
+        storyTitleEnd, chapterTitleStart = prefixLength, prefixLength
+        match = re.search(u'[:\.\s]*(?P<chapter>глава\s+)?$', largestCommonPrefix, re.IGNORECASE | re.UNICODE)
+        if match:
+            storyTitleEnd -= len(match.group())
+            label = match.group('chapter')
+            if label:
+                chapterTitleStart -= len(label)
+        storyTitle = largestCommonPrefix[:storyTitleEnd]
+        self.story.setMetadata('title', storyTitle)
+
+        garbagePattern = re.compile(u'(?P<start>^)?[:\.\s]*(?(start)|$)', re.UNICODE)
+        indexPattern = re.compile(u'(?:глава\s)?(?:(?<!\d)(?P<index>\d{1,3})(?=\D|$))', re.IGNORECASE | re.UNICODE)
+
+        for chapter in chapters:
+            url = chapter.getUrl()
+            self._chapters[url] = chapter
+            _logger.debug(u"Processing chapter `%s'.", url)
+
+            try:
+                authorName = chapter.getAuthorName()
+                if authorName:
+                    self.story.extendList('author', [authorName])
+                    authorId = chapter.getAuthorId()
+                    if authorId:
+                        authorUrl = 'http://%s/index/%s' % (self.getSiteDomain(), authorId)
+                    else:
+                        authorId = u''
+                        authorUrl = u''
+                    self.story.extendList('authorId', [authorId])
+                    self.story.extendList('authorUrl', [authorUrl])
+
+                if not self.story.getMetadata('rating'):
+                    ratingTitle = chapter.getRatingTitle()
+                    if ratingTitle:
+                        self.story.setMetadata('rating', ratingTitle)
+
+                if not self.story.getMetadata('description'):
+                    summary = chapter.getSummary()
+                    if summary:
+                        self.story.setMetadata('description', summary)
+
+                datePublished = min(datePublished, chapter.getDate())
+                dateUpdated = max(dateUpdated, chapter.getDate())
+
+                self.story.extendList('genre', chapter.getGenres())
+                self.story.extendList('characters', chapter.getCharacters())
+                self.story.extendList('ships', chapter.getPairings())
+
+                wordCount += self._getWordCount(chapter.getTextElement())
+
+                # Chapter status usually represents the story status, so we want the last chapter status.
+                # Some chapters may have no status attribute.
+                chapterInProgress = chapter.isInProgress()
+                if chapterInProgress is not None:
+                    storyInProgress = chapterInProgress
+
+                # If any chapter is adult, consider the whole story adult.
+                if chapter.isAdult():
+                    self.story.setMetadata('is_adult', True)
+                    warning = chapter.getWarning()
+                    if warning:
+                        self.story.extendList('warnings', [warning])
+
+                chapterTitle = re.sub(garbagePattern, u'', chapter.getHeading()[chapterTitleStart:])
+
+                match = re.search(indexPattern, chapterTitle)
+                if match:
+                    index = int(match.group('index'))
+                    chapterCount = max(chapterCount, index)
+                else:
+                    chapterCount += 1
+
+                self.chapterUrls.append((chapterTitle, url))
+            except ParsingError, error:
+                raise exceptions.FailedToDownload(u"Failed to download chapter `%s': %s" % (url, error))
+
+        # Some metadata are handled separately due to format conversions.
+        self.story.setMetadata('status', 'In Progress' if storyInProgress else 'Completed')
+        self.story.setMetadata('datePublished', datePublished)
+        self.story.setMetadata('dateUpdated', dateUpdated)
+        self.story.setMetadata('numWords', str(wordCount))
+        self.story.setMetadata('numChapters', chapterCount)
+
+        # Site-specific metadata.
+        self.story.setMetadata('language', self.SITE_LANGUAGE)
+
+    def getChapterText(self, url):
+        """Grabs the text for an individual chapter."""
+        if url not in self._chapters:
+            raise exceptions.FailedToDownload(u"No chapter `%s' present!" % url)
+        chapter = self._chapters[url]
+        return self.utf8FromSoup(url, chapter.getTextElement())
+
+    def _makeChapter(self, url):
+        """Creates a chapter object given a URL."""
+        document = self.make_soup(self._fetchUrl(url))
+        chapter = Chapter(self._getParsingConfiguration(), url, document)
+        return chapter
+
+    def _getWordCount(self, element):
+        """Returns word count in plain text extracted from chapter body."""
+        text = stripHTML(element)
+        count = len(re.findall(self.WORD_PATTERN, text))
+        return count
+
+    def _getParsingConfiguration(self):
+        if not self._parsingConfiguration:
+            self._parsingConfiguration = {}
+
+            adultRatings = self.getConfigList('adult_ratings')
+            if not adultRatings:
+                raise exceptions.PersonalIniFailed(
+                    u"Missing `adult_ratings' setting", u"MassEffect2.in", u"?")
+            adultRatings = set(adultRatings)
+            self._parsingConfiguration['adultRatings'] = adultRatings
+
+            ratingTitleDescriptions = self.getConfigList('rating_titles')
+            if ratingTitleDescriptions:
+                ratingTitles = {}
+                for ratingDescription in ratingTitleDescriptions:
+                    parts = ratingDescription.split(u'=')
+                    if len(parts) < 2:
+                        _logger.warning(
+                            u"Invalid `rating_titles' setting, missing `=' in `%s'."
+                            % ratingDescription)
+                        continue
+                    labels = parts[:-1]
+                    title = parts[-1]
+                    for label in labels:
+                        ratingTitles[label] = title
+                        # Duplicate label aliasing in adult rating set.
+                        if label in adultRatings:
+                            adultRatings.add(*labels)
+                self._parsingConfiguration['adultRatings'] = list(adultRatings)
+                self._parsingConfiguration['ratingTitles'] = ratingTitles
+            else:
+                raise exceptions.PersonalIniFailed(
+                    u"Missing `rating_titles' setting", u"MassEffect2.in", u"?")
+
+            self._parsingConfiguration['excludeEditorSignature'] = \
+                self.getConfig('exclude_editor_signature', False)
+
+        return self._parsingConfiguration
+
+    def _getDocumentId(self, url):
+        """Extract document ID from MassEffect2.in URL."""
+        match = re.search(self.DOCUMENT_ID_PATTERN, url)
+        if not match:
+            raise ValueError(u"Failed to extract document ID from `'" % url)
+        documentId = url[match.start():match.end()]
+        return documentId
+
+    @classmethod
+    def _makeDocumentUrl(cls, documentId):
+        """Make a chapter URL given a document ID."""
+        return 'http://%s/publ/%s' % (cls.getSiteDomain(), documentId)
+
+
+class Chapter(object):
+    """Represents a lazily-parsed chapter of a story."""
+    def __init__(self, configuration, url, document):
+        self._configuration = configuration
+        self._url = url
+        self._document = document
+        # Lazy-loaded:
+        self._heading = None
+        self._date = None
+        self._author = None
+        self._attributes = None
+        self._textElement = None
+        self._infoBar = None
+
+    def getHeading(self):
+        return self._extractHeading()
+
+    def getSummary(self):
+        attributes = self.__getAttributes()
+        if 'summary' in attributes:
+            return attributes['summary']
+
+    def getAuthorId(self):
+        author = self._getAuthor()
+        if author:
+            return author['id']
+
+    def getAuthorName(self):
+        author = self._getAuthor()
+        if author:
+            return author['name']
+
+    def getDate(self):
+        return self.__getDate()
+
+    def getRatingTitle(self):
+        attributes = self.__getAttributes()
+        if 'rating' in attributes:
+            return attributes['rating']['title']
+
+    def isAdult(self):
+        attributes = self.__getAttributes()
+        if 'rating' in attributes and attributes['rating']['isAdult']:
+            return True
+        if 'warning' in attributes:
+            return True
+        return False
+
+    def getWarning(self):
+        attributes = self.__getAttributes()
+        if 'warning' in attributes:
+            return attributes['warning']
+
+    def getCharacters(self):
+        return self._getListAttribute('characters')
+
+    def getPairings(self):
+        return self._getListAttribute('pairings')
+
+    def getGenres(self):
+        return self._getListAttribute('genres')
+
+    def isInProgress(self):
+        attributes = self.__getAttributes()
+        if 'isInProgress' in attributes:
+            return attributes['isInProgress']
+
+    def getUrl(self):
+        return self._url
+
+    def getTextElement(self):
+        return self._getTextElement()
+
+    def getPreviousChapterUrl(self):
+        """Download chapters following `Previous chapter' links.
+        Returns a list of chapters' URLs."""
+        return self._getSiblingChapterUrl({'class': 'fl tal'})
+
+    def getNextChapterUrl(self):
+        """Download chapters following `Next chapter' links.
+        Returns a list of chapters' URLs."""
+        return self._getSiblingChapterUrl({'class': 'tar fr'})
+
+    def isFromStory(self, storyTitle, prefixThreshold=-1):
+        """Check if this chapter is from a story different from the given one.
+        Prefix threshold specifies how long common story title prefix shall be
+        for chapters from one story: negative value means implementation-defined
+        optimum, zero inhibits the check, and positive value adjusts threshold."""
+
+        def getFirstWord(string):
+            match = re.search(u'^\s*\w+', string, re.UNICODE)
+            return string[match.start():match.end()]
+
+        thisStoryTitle = self.getHeading()
+        if prefixThreshold != 0:
+            if prefixThreshold < 0:
+                prefixThreshold = min(
+                    len(getFirstWord(storyTitle)), len(getFirstWord(thisStoryTitle)))
+            else:
+                prefixThreshold = min(
+                    prefixThreshold, len(storyTitle), len(thisStoryTitle))
+            result = len(_getLargestCommonPrefix(storyTitle, thisStoryTitle)) >= prefixThreshold
+            return result
+        else:
+            return storyTitle != thisStoryTitle
+
+    def _getListAttribute(self, name):
+        """Return an attribute value as a list or an empty list if the attribute is absent."""
+        attributes = self.__getAttributes()
+        if name in attributes:
+            return attributes[name]
+        return []
+
+    def _extractHeading(self):
+        """Extract header text from the document."""
+        return stripHTML(
+            self._document.find('div', {'class': 'eTitle'}).string)
+
+    def __getHeading(self):
+        """Lazily parse and return heading."""
+        if not self._heading:
+            self._heading = self._extractHeading()
+        return self._heading
+
+    def _getAuthor(self):
+        """Lazily parse and return author's information."""
+        if not self._author:
+            self._author = self._parseAuthor()
+        return self._author
+
+    def _parseAuthor(self):
+        """Locate and parse chapter author's information to a dictionary with author's `id' and `name'."""
+        try:
+            authorLink = self._getInfoBarElement() \
+                .find('i', {'class': 'icon-user'}) \
+                .findNextSibling('a')
+        except AttributeError:
+            raise ParsingError(u'Failed to locate author link.')
+        match = re.search(u'(8-\d+)', authorLink['onclick'])
+        if not match:
+            raise ParsingError(u'Failed to extract author ID.')
+        authorId = match.group(0)
+        authorName = stripHTML(authorLink.text)
+        return {
+            'id': authorId,
+            'name': authorName
+        }
+
+    def __getDate(self):
+        """Lazily parse chapter date."""
+        if not self._date:
+            self._date = self._parseDate()
+        return self._date
+
+    def _parseDate(self):
+        """Locate and parse chapter date."""
+        try:
+            dateText = self._getInfoBarElement() \
+                .find('i', {'class': 'icon-eye'}) \
+                .findPreviousSibling(text=True) \
+                .strip(u'| \n')
+        except AttributeError:
+            raise ParsingError(u'Failed to locate date.')
+        date = makeDate(dateText, '%d.%m.%Y')
+        return date
+
+    def _getInfoBarElement(self):
+        """Locate informational bar element, containing chapter date and author, on the page."""
+        if not self._infoBar:
+            self._infoBar = self._document.find('td', {'class': 'eDetails2'})
+            if not self._infoBar:
+                raise ParsingError(u'No informational bar found.')
+        return self._infoBar
+
+    def __getAttributes(self):
+        """Lazily parse attributes."""
+        if not self._attributes:
+            self._attributes = self._parseAttributes()
+        return self._attributes
+
+    def _parseAttributes(self):
+        """Parse chapter attribute block and return it as a dictionary with standard entries."""
+
+        attributes = {}
+        attributesText = u''
+        try:
+            starter = self._document \
+                .find('div', {'class': 'comm-div'}) \
+                .findNextSibling('div', {'class': 'cb'})
+            bound = starter.findNextSibling('div', {'class': 'cb'})
+
+            def processElement(element):
+                """Return textual representation an *inline* element of chapter attribute block."""
+                result = u''
+                if isinstance(element, bs4.Tag):
+                    if element.name in ('b', 'strong', 'font', 'br'):
+                        result += u"\n"
+                    if element.name == 's':
+                        result += u"<s>%s</s>" % stripHTML(element)
+                    else:
+                        result += stripHTML(element)
+                else:
+                    result += removeEntities(element)
+                return result
+
+            elements = starter.nextSiblingGenerator()
+            for element in elements:
+                if isinstance(element, bs4.Tag):
+                    if element == bound:
+                        break
+                    else:
+                        if element.name in ('div', 'p'):
+                            attributesText += u"\n"
+                            for child in element.childGenerator():
+                                attributesText += processElement(child)
+                            continue
+                attributesText += processElement(element)
+
+            elements = starter.nextGenerator()
+            for element in elements:
+                if isinstance(element, bs4.Tag):
+                    if element == bound:
+                        break
+                    elif element.name == 'img':
+                        rating = self._parseRatingFromImage(element)
+                        if rating:
+                            attributes['rating'] = rating
+                            break
+        except AttributeError or TypeError:
+            raise ParsingError(u'Failed to locate and collect attributes.')
+
+        separators = u"\r\n :;."
+        freestandingText = u''
+        for line in attributesText.split(u'\n'):
+            if line.count(u':') != 1:
+                freestandingText += line
+                continue
+            key, value = line.split(u':', 1)
+            key = key.strip(separators).lower()
+            value = value.strip().strip(separators)
+            parsed = self._parseAttribute(key, value)
+            for parsedKey, parsedValue in parsed.iteritems():
+                attributes[parsedKey] = parsedValue
+
+        freestandingText = freestandingText.strip()
+        if 'summary' not in attributes and freestandingText:
+            attributes['summary'] = freestandingText
+
+        if 'rating' not in attributes:
+            _logger.warning(u"Failed to locate or recognize rating for `%s'!", self.getUrl())
+
+        return attributes
+
+    # Most, but not all, URLs of rating icons match this.
+    RATING_LABEL_PATTERN = re.compile(u'/(?P<rating>[ERATINnG]+)\.png$')
+
+    def _parseRatingFromImage(self, element):
+        """Given an image element, try to parse story rating from it."""
+        # Although deprecated, `has_key()' is required here.
+        if not element.has_attr('src'):
+            return
+        source = element['src']
+        if 'REITiNG' in source:
+            match = re.search(self.RATING_LABEL_PATTERN, source)
+            if not match:
+                return
+            label = match.group('rating')
+            if label in self._configuration['ratingTitles']:
+                return {
+                    'label': label,
+                    'title': self._configuration['ratingTitles'][label],
+                    'isAdult': label in self._configuration['adultRatings']
+                }
+            else:
+                _logger.warning(u"No title found for rating label `%s'!" % label)
+        # TODO: conduct a research on such abnormal URLs.
+        elif source == 'http://www.masseffect2.in/_fr/10/1360399.png':
+            label = 'Nn'
+            return {
+                'label': 'Nn',
+                'title': self._configuration['ratingTitles'][label],
+                'isAdult': label in self._configuration['adultRatings']
+            }
+
+    # Various `et cetera' and `et al' forms in Russian texts.
+    # Intended to be used with whole strings!
+    ETC_PATTERN = re.compile(
+        u'''[и&]\s(?:
+              (?:т\.?\s?[пд]?\.?)|
+              (?:др(?:угие|\.)?)|
+              (?:пр(?:очие|\.)?)|
+              # Note: identically looking letters `K' and `o'
+              # below are from Latin and Cyrillic alphabets.
+              (?:ко(?:мпания)?|[KК][oо°])
+            )$
+        ''',
+        re.IGNORECASE + re.UNICODE + re.VERBOSE)
+
+    # `Author's Notes' and its variants in Russian.
+    ANNOTATION_PATTERN = re.compile(u'аннотация|описание|(?:(?:за|при)мечание\s)?(?:от\s)?автора', re.UNICODE)
+
+    def _parseAttribute(self, key, value):
+        """
+        Parse a single a single record in chapter attributes for chapter metadata.
+        Return a dictionary of canonical attributes and values (i. e. multiple attributes may be discovered).
+        """
+
+        def refineCharacter(name):
+            """Refines character name from stop-words and distortions."""
+            strippedName = name.strip()
+            nameOnly = re.sub(self.ETC_PATTERN, u'', strippedName)
+            # TODO: extract canonical name (even ME-specific?).
+            canonicalName = nameOnly
+            return canonicalName
+
+        if re.match(u'жанры?', key, re.UNICODE):
+            genres = filter(bool, map(unicode.strip, re.split(u'[,;/]', value)))
+            return {'genres': genres}
+        elif key == u'статус':
+            isInProgress = value == u'в процессе'
+            return {'isInProgress': isInProgress}
+        elif key == u'персонажи':
+            participants = map(refineCharacter, re.split(u'[,;]', value))
+            characters = []
+            pairings = []
+            for participant in participants:
+                if u'/' in participant:
+                    pairings.append(participant)
+                else:
+                    characters.append(participant)
+            return {
+                'characters': characters,
+                'pairings': pairings
+            }
+        elif key == u'предупреждение':
+            return {'warning': value}
+        elif re.match(self.ANNOTATION_PATTERN, key):
+            if not value.endswith(u'.'):
+                value += u'.'
+            # Capitalize would make value[1:] lowercase, which we don't want.
+            value = value[:1].upper() + value[1:]
+            return {'summary': value}
+        else:
+            _logger.info(u"Unrecognized attribute `%s' ignored.", key)
+            return {}
+
+    def _getTextElement(self):
+        """Locate chapter body text element on the page."""
+        if not self._textElement:
+            self._textElement = self.__collectTextElements()
+        return self._textElement
+
+    def __collectTextElements(self):
+        """Return all elements containing parts of chapter text (which may be
+        <p>aragraphs, <div>isions or plain text nodes) under a single root."""
+        starter = self._document.find('div', {'id': u'article'})
+        if starter is None:
+            # FIXME: This will occur if the method is called more than once.
+            # The reason is elements appended to `root' are removed from the document.
+            # BS 4.4 implements cloning via `copy.copy()', but supporting it for BS 4.3
+            # would be error-prone (due to relying on BS internals) and is not needed.
+            if self._textElement:
+                _logger.debug(u"You may not call this function more than once!")
+            raise ParsingError(u'Failed to locate text.')
+        collection = [starter]
+        for element in starter.nextSiblingGenerator():
+            if element is None:
+                break
+            if isinstance(element, bs4.Tag) and element.name == 'tr':
+                break
+            collection.append(element)
+        root = bs4.Tag(name='td')
+        for element in collection:
+            root.append(element)
+
+        if self._configuration['excludeEditorSignature']:
+            root = self._excludeEditorSignature(root)
+
+        return root
+
+    def _getSiblingChapterUrl(self, selector):
+        """Locate a link to a sibling chapter, either previous or next one, and return its URL."""
+        block = self._document \
+            .find('td', {'class': 'eDetails1'}) \
+            .find('div', selector)
+        if not block:
+            return
+        link = block.find('a')
+        if not link:
+            return
+        return link['href']
+
+    # Editor signature always starts with something like this.
+    SIGNED_PATTERN = re.compile(u'отредактирова(?:но|ла?)[:.\s]', re.IGNORECASE + re.UNICODE)
+
+    def _excludeEditorSignature(self, root):
+        """Exclude editor signature from within `root' element."""
+        for textNode in root.findAll(text=True):
+            if re.match(self.SIGNED_PATTERN, textNode.string):
+                editorLink = textNode.findNext('a')
+                if editorLink:
+                    editorLink.extract()
+                # Seldom editor link has inner formatting, which is sibling DOM-wise.
+                editorName = textNode.findNext('i')
+                if editorName:
+                    editorName.extract()
+                textNode.extract()
+                # We could try removing container element, but there is a risk
+                # of removing text ending with it.  Better play safe here.
+                break
+        return root
+
+
+def _getLargestCommonPrefix(*args):
+    """Returns largest common prefix of all unicode arguments, ignoring case.
+    :rtype : unicode
+    """
+    from itertools import takewhile, izip
+    toLower = lambda xs: map(lambda x: x.lower(), xs)
+    allSame = lambda xs: len(set(toLower(xs))) == 1
+    return u''.join([i[0] for i in takewhile(allSame, izip(*args))])
@@ -22,7 +22,6 @@ import re
 import urllib
 import urllib2

-from .. import BeautifulSoup as bs
 from ..htmlcleanup import stripHTML
 from .. import exceptions as exceptions

@@ -41,29 +40,57 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
        
        # get storyId from url--url validation guarantees query correct
        m = re.match(self.getSiteURLPattern(),url)
+        urltitle='urltitle'
        if m:
-            self.story.setMetadata('storyId',m.group('id'))
+            if m.group('id1'):
+                self.story.setMetadata('storyId',m.group('id1'))
+                urltitle=m.group('urltitle1')
+            elif m.group('id2'):
+                self.story.setMetadata('storyId',m.group('id2'))
+                urltitle=m.group('urltitle2')
+            elif m.group('id3'):
+                self.story.setMetadata('storyId',m.group('id3'))
+            else:
+                raise InvalidStoryURL(url,
+                                      self.getSiteDomain(),
+                                      self.getSiteExampleURLs())
            
            # normalized story URL.
-            self._setURL('http://' + self.getSiteDomain() + '/fanfic/view_st.php/'+self.story.getMetadata('storyId'))
+            self._setURL('http://' + self.getSiteDomain() + '/fanfic/s/'+urltitle+'/'+self.story.getMetadata('storyId'))
        else:
            raise exceptions.InvalidStoryURL(url,
                                             self.getSiteDomain(),
                                             self.getSiteExampleURLs())
            
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%B %d, %Y %H:%M"
+            
    @staticmethod
    def getSiteDomain():
        return 'www.mediaminer.org'

    @classmethod
    def getSiteExampleURLs(cls):
-        return "http://"+cls.getSiteDomain()+"/fanfic/view_st.php/123456 http://"+cls.getSiteDomain()+"/fanfic/view_ch.php/1234123/123444#fic_c"
+        return "http://"+cls.getSiteDomain()+"/fanfic/s/story-title/123456 http://"+cls.getSiteDomain()+"/fanfic/c/story-title/chapter-title/123456/987612"

    def getSiteURLPattern(self):
-        ##  http://www.mediaminer.org/fanfic/view_st.php/76882
-        ##  http://www.mediaminer.org/fanfic/view_ch.php/167618/594087#fic_c
-        return re.escape("http://"+self.getSiteDomain())+\
-            "/fanfic/view_(st|ch)\.php/"+r"(?P<id>\d+)(/\d+(#fic_c)?)?$"
+        ## old urls
+        ## http://www.mediaminer.org/fanfic/view_st.php/76882
+        ## new urls
+        ## http://www.mediaminer.org/fanfic/s/ghosts-from-the-past/72
+        ## http://www.mediaminer.org/fanfic/c/ghosts-from-the-past/chapter-2/72/174
+        ## http://www.mediaminer.org/fanfic/s/robtech-final-missions/61553
+        ## http://www.mediaminer.org/fanfic/c/robtech-final-missions/robotech-final-missions-oneshot/61553/189830
+        return re.escape("http://"+self.getSiteDomain())+r"/fanfic/"+\
+            r"((s/(?P<urltitle1>[^/]+)/(?P<id1>\d+))|"+\
+            r"((c/(?P<urltitle2>[^/]+)/[^/]+/(?P<id2>\d+))/\d+)|"+\
+            r"(view_st\.php/(?P<id3>\d+)))"
+            
+    # Override stripURLParameters so the id parameter won't get stripped
+    @classmethod
+    def stripURLParameters(cls, url):
+        return url

    def extractChapterUrlsAndMetadata(self):

@@ -71,23 +98,26 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
        logger.debug("URL: "+url)

        try:
-            data = self._fetchUrl(url)
+            data = self._fetchUrl(url) # w/o trailing / gets 'chapter list' page even for one-shots.
        except urllib2.HTTPError, e:
            if e.code == 404:
+                logger.error("404 on %s"%url)
                raise exceptions.StoryDoesNotExist(self.url)
            else:
                raise e

        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
+        soup = self.make_soup(data)

        # [ A - All Readers ], strip '[' ']'
        ## Above title because we remove the smtxt font to get title.
-        smtxt = soup.find("font",{"class":"smtxt"})
+        smtxt = soup.find("h3",{"id":"post-rating"})
        if not smtxt:
+            logger.error("can't find rating")
            raise exceptions.StoryDoesNotExist(self.url)
-        rating = smtxt.string[1:-1]
-        self.story.setMetadata('rating',rating)
+        else:
+            rating = smtxt.string[1:-1]
+            self.story.setMetadata('rating',rating)

        # Find authorid and URL from... author url.
        a = soup.find('a', href=re.compile(r"/fanfic/src.php/u/\d+"))
@@ -103,41 +133,24 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
        ## <td class="ffh">The Kraut, The Bartender, and The Drunkard: Chapter 1</b> <font class="smtxt">[ P - Pre-Teen ]</font></td>
        ## <td class="ffh">Betrayal and Justice: A Cold Heart</b> <font size="-1">( Chapter 1 )</font> <font class="smtxt">[ A - All Readers ]</font></td>
        ## <td class="ffh">Question and Answer: Question and Answer</b> <font size="-1">( One-Shot )</font> <font class="smtxt">[ A - All Readers ]</font></td>
-        title = soup.find('td',{'class':'ffh'})
-        for font in title.findAll('font'):
-            font.extract() # removes 'font' tags from inside the td.        
-        if title.has_key('colspan'):
-            titlet = stripHTML(title)
-        else:
-            ## No colspan, it's part chapter title--even if it's a one-shot.
-            titlet = ':'.join(stripHTML(title).split(':')[:-1]) # strip trailing 'Chapter X' or chapter title
-        self.story.setMetadata('title',titlet)
-        ## The story title is difficult to reliably parse from the
-        ## story pages.  Getting it from the author page is, but costs
-        ## another fetch.
-        # authsoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
-        # titlea = authsoup.find('a',{'href':'/fanfic/view_st.php/'+self.story.getMetadata('storyId')})
-        # self.story.setMetadata('title',titlea.text)
+        # title = soup.find('td',{'class':'ffh'})
+        # for font in title.findAll('font'):
+        #     font.extract() # removes 'font' tags from inside the td.        
+        # if title.has_attr('colspan'):
+        #     titlet = stripHTML(title)
+        # else:
+        #     ## No colspan, it's part chapter title--even if it's a one-shot.
+        #     titlet = ':'.join(stripHTML(title).split(':')[:-1]) # strip trailing 'Chapter X' or chapter title
+        self.story.setMetadata('title',stripHTML(soup.find('h1',{'id':'post-title'})))

        # save date from first for later.
        firstdate=None
        
-        # Find the chapters
-        select = soup.find('select',{'name':'cid'})
-        if not select:
-            self.chapterUrls.append(( self.story.getMetadata('title'),self.url))
-        else:
-            for option in select.findAll("option"):
-                chapter = stripHTML(option.string)
-                ## chapter can be: Chapter 7 [Jan 23, 2011]
-                ##             or: Vigilant Moonlight ( Chapter 1 ) [Jan 30, 2004]
-                ##        or even: Prologue ( Prologue ) [Jul 31, 2010]
-                m = re.match(r'^(.*?) (\( .*? \) )?\[(.*?)\]$',chapter)
-                chapter = m.group(1)
-                # save date from first for later.
-                if not firstdate:
-                    firstdate = m.group(3)
-                self.chapterUrls.append((chapter,'http://'+self.host+'/fanfic/view_ch.php/'+self.story.getMetadata('storyId')+'/'+option['value']))
+        # Find the chapters - one-shot now have chapter list, too.
+        chap_p = soup.find('p',{'style':'margin-left:10px;'})
+        for (atag,aurl,name) in [ (x,x['href'],stripHTML(x)) for x in chap_p.find_all('a') ]:
+            self.chapterUrls.append((name,'http://'+self.host+aurl))
+            
        self.story.setMetadata('numChapters',len(self.chapterUrls))

        # category
@@ -146,27 +159,20 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
            self.story.addToList('category',a.string)
        
        # genre
-        # <a href="/fanfic/src.php/a/567">Ranma 1/2</a>
+        # <a href="/fanfic/src.php/g/567">Ranma 1/2</a>
        for a in soup.findAll('a',href=re.compile(r"^/fanfic/src.php/g/")):
            self.story.addToList('genre',a.string)

-        # if firstdate, then the block below will only have last updated.
-        if firstdate:
-            self.story.setMetadata('datePublished', makeDate(firstdate, "%b %d, %Y"))
-        # Everything else is in <tr bgcolor="#EEEED4">
-
-        metastr = stripHTML(soup.find("tr",{"bgcolor":"#EEEED4"})).replace('\n',' ').replace('\r',' ').replace('\t',' ')
-        # Latest Revision: August 03, 2010
-        m = re.match(r".*?(?:Latest Revision|Uploaded On): ([a-zA-Z]+ \d\d, \d\d\d\d)",metastr)
+        metastr = stripHTML(soup.find("div",{"class":"post-meta"}))
+        
+        # Latest Revision: February 07, 2015 15:21 PST
+        m = re.match(r".*?(?:Latest Revision|Uploaded On): ([a-zA-Z]+ \d\d, \d\d\d\d \d\d:\d\d)",metastr)
        if m:
-            self.story.setMetadata('dateUpdated', makeDate(m.group(1), "%B %d, %Y"))
-            if not firstdate:
-                self.story.setMetadata('datePublished',
-                                       self.story.getMetadataRaw('dateUpdated'))
-                
-        else:
-            self.story.setMetadata('dateUpdated',
-                                   self.story.getMetadataRaw('datePublished'))
+            self.story.setMetadata('dateUpdated', makeDate(m.group(1), self.dateformat))
+            # site doesn't give date published on index page.
+            # set to updated, change in chapters below.
+            # self.story.setMetadata('datePublished',
+            #                        self.story.getMetadataRaw('dateUpdated'))

        # Words: 123456
        m = re.match(r".*?\| Words: (\d+) \|",metastr)
@@ -192,44 +198,54 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):

        logger.debug('Getting chapter text from: %s' % url)

-        data=self._fetchUrl(url)
-        soup = bs.BeautifulStoneSoup(data,
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        data = self._fetchUrl(url)
+        soup = self.make_soup(data)

-        anchor = soup.find('a',{'name':'fic_c'})
+        headerstr = stripHTML(soup.find('div',{'class':'post-meta clearfix '}))
+        # print("data:%s"%data)
+        #header.extract()

-        if None == anchor:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-
-        ## find divs with align=left, those are paragraphs in newer stories.
-        divlist = anchor.findAllNext('div',{'align':'left'})
-        if divlist:
-            for div in divlist:
-                div.name='p' # convert to <p> mediaminer uses div with
-                             # a margin for paragraphs.
-                anchor.append(div) # cheat!  stuff all the content
-                                   # divs into anchor just as a
-                                   # holder.
-                del div['style']
-                del div['align']
-            anchor.name='div'
-            return self.utf8FromSoup(url,anchor)
-        
-        else:
-            logger.debug('Using kludgey text find for older mediaminer story.')
-            ## Some older mediaminer stories are unparsable with BeautifulSoup.
-            ## Really nasty formatting.  Sooo... Cheat!  Parse it ourselves a bit first.
-            ## Story stuff falls between:
-            data = "<div id='HERE'>" + data[data.find('<a name="fic_c">'):] +"</div>"
-            soup = bs.BeautifulStoneSoup(data,
-                                         selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-            for tag in soup.findAll('td',{'class':'ffh'}) + \
-                    soup.findAll('div',{'class':'acl'}) + \
-                    soup.findAll('div',{'class':'footer smtxt'}) + \
-                    soup.findAll('table',{'class':'tbbrdr'}):
-                tag.extract() # remove tag from soup.
+        m = re.match(r".*?Uploaded On: ([a-zA-Z]+ \d\d, \d\d\d\d \d\d:\d\d)",headerstr)
+        if m:
+            date = makeDate(m.group(1), self.dateformat)
+            if not self.story.getMetadataRaw('datePublished') or date < self.story.getMetadataRaw('datePublished'):
+                self.story.setMetadata('datePublished', date)
                
-            return self.utf8FromSoup(url,soup)
+        chapter = soup.find('div',{'id':'fanfic-text'})
+        
+        return self.utf8FromSoup(url,chapter)
+    
+        # chapter=self.make_soup('<div class="story"></div>').find('div')
+
+        # if None == header:
+        #     raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+
+        # ## find divs with align=left, those are paragraphs in newer stories.
+        # divlist = header.findAllNext('div',{'align':'left'})
+        # if divlist:
+        #     for div in divlist:
+        #         div.name='p' # convert to <p> mediaminer uses div with
+        #                      # a margin for paragraphs.
+        #         chapter.append(div)
+        #         del div['style']
+        #         del div['align']
+        #     return self.utf8FromSoup(url,chapter)
+        
+        # else:
+        #     logger.debug('Using kludgey text find for older mediaminer story.')
+        #     ## Some older mediaminer stories are unparsable with BeautifulSoup.
+        #     ## Really nasty formatting.  Sooo... Cheat!  Parse it ourselves a bit first.
+        #     ## Story stuff falls between:
+        #     data = "<div id='HERE'>" + data[data.find('<div class="adWrap">'):data.find('<div class="addthis_sharing_toolbox">')] +"</div>"
+        #     soup = self.make_soup(data)
+        #     for tag in soup.findAll('td',{'class':'ffh'}) + \
+        #             soup.findAll('div',{'class':'acl'}) + \
+        #             soup.findAll('div',{'class':'adWrap'}) + \
+        #             soup.findAll('div',{'class':'footer smtxt'}) + \
+        #             soup.findAll('table',{'class':'tbbrdr'}):
+        #         tag.extract() # remove tag from soup.
+                
+        #     return self.utf8FromSoup(url,soup)
        

 def getClass():
@@ -16,222 +16,24 @@
 #

 # Software: eFiction
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return NHAMagicalWorldsUsAdapter
+from base_efiction_adapter import BaseEfictionAdapter

 # Class name has to be unique.  Our convention is camel case the
 # sitename with Adapter at the end.  www is skipped.
-class NHAMagicalWorldsUsAdapter(BaseSiteAdapter):
+class NHAMagicalWorldsUsAdapter(BaseEfictionAdapter):

-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        
-		
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','nha')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = " %d/%m/%y"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
+    @staticmethod
    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
        return 'nha.magical-worlds.us'

    @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-        url = self.url
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-            
-        m = re.search(r"'viewstory.php\?sid=\d+((?:&amp;ageconsent=ok)?&amp;warning=\d+)'",data)
-        if m != None:
-            if self.is_adult or self.getConfig("is_adult"):
-                # We tried the default and still got a warning, so
-                # let's pull the warning number from the 'continue'
-                # link and reload data.
-                addurl = m.group(1)
-                # correct stupid &amp; error in url.
-                addurl = addurl.replace("&amp;","&")
-                url = self.url+'&index=1'+addurl
-                logger.debug("URL 2nd try: "+url)
-
-                try:
-                    data = self._fetchUrl(url)
-                except urllib2.HTTPError, e:
-                    if e.code == 404:
-                        raise exceptions.StoryDoesNotExist(self.url)
-                    else:
-                        raise e    
-            else:
-                raise exceptions.AdultCheckRequired(self.url)
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-            
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
-		
-        try:
-            # in case link points somewhere other than the first chapter
-            a = soup.findAll('option')[1]['value']
-            self.story.setMetadata('storyId',a.split('=',)[1])
-            url = 'http://'+self.host+'/'+a
-            soup = bs.BeautifulSoup(self._fetchUrl(url))
-        except:
-            pass
-		
-        for info in asoup.findAll('table', {'width' : '100%', 'bordercolor' : re.compile(r'#')}):
-            a = info.find('a')
-            if 'viewstory.php?sid='+self.story.getMetadata('storyId') == a['href'] or \
-                    ('viewstory.php?sid='+self.story.getMetadata('storyId')+'&') in a['href']:
-                self.story.setMetadata('title',stripHTML(a))
-                break
-		
-
-        # Find the chapters:
-        chapters=soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+'&chapter=\d+$'))
-        if len(chapters) == 0:
-            self.chapterUrls.append((self.story.getMetadata('title'),url))
-        else:
-            for chapter in chapters:
-                # just in case there's tags, like <i> in chapter titles.
-                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d):
-            try:
-                return d.name
-            except:
-                return ""
-				
-        cats = info.findAll('a',href=re.compile('categories.php'))
-        for cat in cats:
-            self.story.addToList('category',cat.string)
-					
-        a = info.find('a', href=re.compile(r'viewuser.php'))
-        val = a.nextSibling
-        svalue = ""
-        while not defaultGetattr(val) == 'br':
-            val = val.nextSibling
-        val = val.nextSibling
-        while not defaultGetattr(val) == 'br':
-            svalue += unicode(val)
-            val = val.nextSibling
-        self.setDescription(url,svalue)
+    def getSiteAbbrev(self):
+        return 'nha'
        
-        #does not provide convenient way to get word count
-        labels = info.findAll('i')
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = stripHTML(labelspan)
-
-            if 'Rating' in label:
-                self.story.setMetadata('rating', value.split(' -')[0])
-
-            if 'Genres' in label:
-                genres = value.string.split(', ')
-                for genre in genres:
-                    if 'None' not in genre:
-                        self.story.addToList('genre',genre.split(' -')[0])
-            
-            if 'Characters' in label:
-                chars = value.string.split(', ')
-                for char in chars:
-                    if 'None' not in char:
-                        self.story.addToList('characters',char.split(' -')[0])
-
-            if 'Warnings' in label:
-                warnings = value.string.split(', ')
-                for warning in warnings:
-                    if 'None' not in warning:
-                        self.story.addToList('warnings',warning.split(' -')[0])
-
-            if 'Completed' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(value.split(' -')[0], self.dateformat))
-            
-            if 'Updated' in label:
-                # there's a stray [ at the end.
-                #value = value[0:-1]
-                self.story.setMetadata('dateUpdated', makeDate(value.split(' -')[0], self.dateformat))
-
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-		
-        data = self._fetchUrl(url)
-
-        soup = bs.BeautifulSoup(data, selfClosingTags=('br','hr','span','center')) # some chapters seem to be hanging up on those tags, so it is safer to close them
-        
-        story = soup.find('div', {"id" : "story"})
-
-        if None == story:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    @classmethod
+    def getDateFormat(self):
+        return "%d/%m/%y" 
+                           
+def getClass():
+    return NHAMagicalWorldsUsAdapter
    
-        return self.utf8FromSoup(url,story)
@@ -0,0 +1,38 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2015 Fanficdownloader team, 2015 FanFicFare team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Software: eFiction
+import re
+from base_efiction_adapter import BaseEfictionAdapter
+
+class NineLivesDarkSolaceAdapter(BaseEfictionAdapter):
+
+    @staticmethod
+    def getSiteDomain():
+        return 'ninelives.dark-solace.org'
+
+    @classmethod
+    def getSiteAbbrev(self):
+        return '9lvs'
+        
+    @classmethod
+    def getDateFormat(self):
+        return "%B %d, %Y" 
+                           
+def getClass():
+    return NineLivesDarkSolaceAdapter
+    
@@ -171,7 +171,8 @@ class PonyFictionArchiveNetAdapter(BaseSiteAdapter):
            self.story.addToList('warnings',warning.string)
        
        status = soup.find('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
-        self.story.setMetadata('status',status.string)
+        if status: # apparently this site can have stories with neither In-Progress or Complete.
+            self.story.setMetadata('status',status.string)
        
        section = soup.findAll('span', {'class' : 'General'})[1]            
        
@@ -258,6 +258,7 @@ class PortkeyOrgAdapter(BaseSiteAdapter): # XXX
        data = self._fetchUrl(url)

        data = data.replace("HTML>","div>")
+        data = data.replace("html>","div>")
        
        soup = bs.BeautifulSoup(data)

@@ -0,0 +1,48 @@
+#  -*- coding: utf-8 -*-
+
+# Copyright 2015 FanFicFare team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import re
+
+from base_xenforoforum_adapter import BaseXenForoForumAdapter
+
+def getClass():
+    return QuestionablequestingComAdapter
+
+class QuestionablequestingComAdapter(BaseXenForoForumAdapter):
+
+    def __init__(self, config, url):
+        BaseXenForoForumAdapter.__init__(self, config, url)
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','qq')
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'questionablequesting.com'
+
+    @classmethod
+    def getAcceptDomains(cls):
+        return ["forum."+cls.getSiteDomain(),cls.getSiteDomain()]
+
+    @classmethod
+    def getConfigSections(cls):
+        "Only needs to be overriden if has additional ini sections."
+        return ['base_xenforoforum','forum.'+cls.getSiteDomain(),cls.getSiteDomain()]
+    
+    def getSiteURLPattern(self):
+        return r'https?://(forum.)?'+re.escape(self.getSiteDomain())+r'/(?P<tp>threads|posts)/(.+\.)?(?P<id>\d+)/?'
+        
@@ -0,0 +1,108 @@
+import re
+import urlparse
+import urllib2
+import datetime
+
+from .. import exceptions
+from base_adapter import BaseSiteAdapter
+
+SITE_DOMAIN = 'quotev.com'
+STORY_URL_TEMPLATE = 'http://www.quotev.com/story/%s'
+
+
+def getClass():
+    return QuotevComAdapter
+
+
+def get_url_path_segments(url):
+    return tuple(filter(None, url.split('/')[3:]))
+
+
+class QuotevComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        story_id = get_url_path_segments(url)[1]
+        self._setURL(STORY_URL_TEMPLATE % story_id)
+        self.story.setMetadata('storyId', story_id)
+        self.story.setMetadata('siteabbrev', SITE_DOMAIN)
+
+    @staticmethod
+    def getSiteDomain():
+        return SITE_DOMAIN
+
+    @classmethod
+    def getSiteExampleURLs(cls):
+        return STORY_URL_TEMPLATE % '1234'
+
+    def getSiteURLPattern(self):
+        pattern = re.escape(STORY_URL_TEMPLATE.rsplit('%', 1)[0]) + r'(.+?)($|&|/)'
+        pattern = pattern.replace(r'http\:', r'https?\:')
+        pattern = pattern.replace(r'https?\:\/\/www\.', r'https?\:\/\/(www\.)?')
+        return pattern
+
+    def use_pagecache(self):
+        return True
+
+    def extractChapterUrlsAndMetadata(self):
+        try:
+            data = self._fetchUrl(self.url)
+        except urllib2.HTTPError:
+            raise exceptions.FailedToDownload(self.url)
+
+        soup = self.make_soup(data)
+
+        element = soup.find('div', {'class': 'result_head'})
+        if not element:
+            raise exceptions.StoryDoesNotExist(self.url)
+
+        self.story.setMetadata('title', element.find('span', recursive=False).get_text())
+
+        element = soup.find('div', {'class': 'desc_creator'})('a')[1]
+        self.story.setMetadata('author', element.get_text())
+        self.story.setMetadata('authorId', get_url_path_segments(element['href'])[0])
+        self.story.setMetadata('authorUrl', urlparse.urljoin(self.url, element['href']))
+        self.setDescription(self.url, soup.find('div', id='qdesct'))
+        self.setCoverImage(self.url, urlparse.urljoin(self.url, soup.find('img', {'class': 'logo'})['src']))
+
+        for a in soup.find('div', {'class': 'tag'})('a'):
+            if a['href'] == '#':
+                continue
+
+            self.story.addToList('category', a.get_text())
+
+        elements = soup('span', {'class': 'q_time'})
+        self.story.setMetadata('datePublished', datetime.datetime.fromtimestamp(float(elements[0]['ts'])))
+        self.story.setMetadata('dateUpdated', datetime.datetime.fromtimestamp(float(elements[1]['ts'])))
+
+        for a in soup.find('div', id='rselect')('a'):
+            self.chapterUrls.append((a.get_text(), urlparse.urljoin(self.url, a['href'])))
+
+        self.story.setMetadata('numChapters', len(self.chapterUrls))
+
+        element = soup.find('div', {'class': 't'})('div', recursive=False)[1].div
+        data = filter(None, (x.strip() for x in element('span')[1].next_sibling.split(u'\xb7')))
+
+        if 'completed' in data:
+            self.story.setMetadata('status', 'Completed')
+            data.remove('completed')
+        else:
+            self.story.setMetadata('status', 'In-Progress')
+
+        for datum in data:
+            value, key = datum.split()
+            self.story.setMetadata(key, value.replace(',', '').replace('.', ''))
+
+        self.story.setMetadata('favorites', soup.find('div', id='favqn').get_text())
+        self.story.setMetadata('comments', soup.find('a', id='comment_btn').span.get_text())
+
+    def getChapterText(self, url):
+        data = self._fetchUrl(url)
+        soup = self.make_soup(data)
+
+        element = soup.find('div', id='restxt')
+        for a in element('a'):
+            a.unwrap()
+
+        return self.utf8FromSoup(url, element)
@@ -106,7 +106,9 @@ class SpikeluverComAdapter(BaseSiteAdapter):

        listbox_tag = soup.find('div', {'class': 'listbox'})
        for span_tag in listbox_tag('span'):
-            key = span_tag.string.strip(' :')
+            key = span_tag.string
+            if key:
+                key = key.strip(' :')
            try:
                value = stripHTML(span_tag.nextSibling)
            # This can happen with some fancy markup in the summary. Just
@@ -135,8 +137,10 @@ class SpikeluverComAdapter(BaseSiteAdapter):
                        contents.append(sibling)

                # Remove the preceding break line tag and other crud
-                contents.pop()
-                contents.pop()
+                if contents:
+                    contents.pop()
+                if contents:
+                    contents.pop()
                self.story.setMetadata('description', ''.join(contents))

            elif key == 'Rated':
@@ -21,7 +21,7 @@ logger = logging.getLogger(__name__)
 import re
 import urllib2

-from .. import BeautifulSoup as bs
+#from .. import BeautifulSoup as bs
 from ..htmlcleanup import stripHTML
 from .. import exceptions as exceptions

@@ -40,22 +40,22 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
        self.password = ""
        self.is_adult=False
-        
+
        # get storyId from url
        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2].split(':')[0])
        if 'storyInfo' in self.story.getMetadata('storyId'):
            self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        
+
        # normalized story URL.
        self._setURL('http://' + self.getSiteDomain() + '/s/'+self.story.getMetadata('storyId'))
-        
+
        # Each adapter needs to have a unique site abbreviation.
        self.story.setMetadata('siteabbrev','strol')

        # The date format will vary from site to site.
        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
        self.dateformat = "%Y-%m-%d"
-            
+
    @staticmethod # must be @staticmethod, don't remove it.
    def getSiteDomain():
        # The site domain.  Does have www here, if it uses it.
@@ -78,7 +78,7 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
                or "Access to unlinked chapters requires" in data:
            self.needToLogin = True
        return self.needToLogin
-        
+
    def performLogin(self, url):
        params = {}

@@ -91,13 +91,13 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
        params['rememberMe'] = '1'
        params['page'] = 'http://'+self.getSiteDomain()+'/'
        params['submit'] = 'Login'
-    
+
        loginUrl = 'http://' + self.getSiteDomain() + '/login.php'
        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                              params['theusername']))
-    
+
        d = self._fetchUrl(loginUrl, params,usecache=False)
-    
+
        if "My Account" not in d : #Member Account
            logger.info("Failed to login to URL %s as %s" % (loginUrl,
                                                              params['theusername']))
@@ -114,7 +114,7 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
        return True

    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
+    def doExtractChapterUrlsAndMetadata(self, get_cover=True):

        # index=1 makes sure we see the story chapter index.  Some
        # sites skip that for one-chapter stories.
@@ -137,22 +137,22 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
            # need to log in for this one.
            self.performLogin(url)
            data = self._fetchUrl(url+":i",usecache=False)
-        
+
        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
        elif "Error! The story you're trying to access is being filtered by your choice of contents filtering." in data:
            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Error! The story you're trying to access is being filtered by your choice of contents filtering.")
-            
+
        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
+        soup = self.make_soup(data)
        #print data

        # Now go hunting for all the meta data and the chapter list.
-        
+
        ## Title
        a = soup.find('h1')
        self.story.setMetadata('title',stripHTML(a))
-        
+
        notice = soup.find('div', {'class' : 'notice'})
        if notice:
            self.story.setMetadata('notice',unicode(notice))
@@ -178,7 +178,7 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
        page=0
        i=0
        while i == 0:
-            asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getList('authorUrl')[0]+"/"+str(page)))
+            asoup = self.make_soup(self._fetchUrl(self.story.getList('authorUrl')[0]+"/"+str(page)))

            a = asoup.findAll('td', {'class' : 'lc2'})
            for lc2 in a:
@@ -196,8 +196,8 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):

        score = lc2.findNext('th', {'class' : 'num'}).text
        if score != '-':
-            self.story.setMetadata('score', score)        
-        
+            self.story.setMetadata('score', score)
+
        lc4 = lc2.findNext('td', {'class' : 'lc4'})
        desc = lc4.contents[0]

@@ -212,7 +212,7 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
                self.story.setMetadata('seriesUrl',seriesUrl)
                series_name = stripHTML(a)
                logger.debug("Series name= %s" % series_name)
-                series_soup = bs.BeautifulSoup(self._fetchUrl(seriesUrl))
+                series_soup = self.make_soup(self._fetchUrl(seriesUrl))
                if series_soup:
                    logger.debug("Retrieving Series - looking for name")
                    series_name = series_soup.find('span', {'id' : 'ptitle'}).text.partition(' — ')[0]
@@ -221,7 +221,7 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
                desc = lc4.contents[2]
                # Check if series is in a universe
                universe_url = self.story.getList('authorUrl')[0]  + "&type=uni"
-                universes_soup = bs.BeautifulSoup(self._fetchUrl(universe_url) )
+                universes_soup = self.make_soup(self._fetchUrl(universe_url) )
                logger.debug("Universe url='{0}'".format(universe_url))
                if universes_soup:
                    universes = universes_soup.findAll('div', {'class' : 'ser-box'})
@@ -255,11 +255,11 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
                universe_name = stripHTML(a)
                universeUrl = 'http://'+self.host+a['href']
                logger.debug("Retrieving Universe - about to get page - universeUrl='{0}".format(universeUrl))
-                universe_soup = bs.BeautifulSoup(self._fetchUrl(universeUrl))
+                universe_soup = self.make_soup(self._fetchUrl(universeUrl))
                logger.debug("Retrieving Universe - have page")
                if universe_soup:
                    logger.debug("Retrieving Universe - looking for name")
-                    universe_name = universe_soup.find('h1', {'id' : 'ptitle'}).text.partition(' &mdash;')[0]
+                    universe_name = universe_soup.find('h1', {'id' : 'ptitle'}).text.partition('—')[0]
                    logger.debug("Universes name: '{0}'".format(universe_name))

                self.story.setMetadata('universeUrl',universeUrl)
@@ -274,7 +274,7 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
            pass

        self.setDescription('http://'+self.host+'/s/'+self.story.getMetadata('storyId'),desc)
-            
+
        for b in lc4.findAll('b'):
            #logger.debug('Getting metadata: "%s"' % b)
            label = b.text
@@ -284,24 +284,38 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
            else:
                value = b.nextSibling
            #logger.debug('label: "%s", value: "%s"' % (label, value))
-            
+
            if 'Sex' in label:
                self.story.setMetadata('rating', value)
-                
+
            if 'Tags' in label or 'Codes' in label:
                for code in re.split(r'\s*,\s*', value.strip()):
                     self.story.addToList('sitetags',code)
-                    
+
            if 'Posted' in label:
                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-                
+
            if 'Concluded' in label:
                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-                
+
            if 'Updated' in label:
                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-#                
+
+        # Some books have a cover in the index page.
+        # Samples are:
+        #     http://storiesonline.net/s/11999
+        #     http://storiesonline.net/s/10823
+        if get_cover:
+            logger.debug("Looking for the cover image...")
+            cover_url = ""
+            img = soup.find('img')
+            if img:
+                cover_url=img['src']
+            logger.debug("cover_url: %s"%cover_url)
+            if cover_url:
+                self.setCoverImage(url,cover_url)
+
        status = lc4.find('span', {'class' : 'ab'})
        if  status != None:
            self.story.setMetadata('status', 'In-Progress')
@@ -312,111 +326,146 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
        else:
            self.story.setMetadata('status', 'Completed')

-            
    # grab the text for an individual chapter.
    def getChapterText(self, url):

        logger.debug('Getting chapter text from: %s' % url)

-        soup = bs.BeautifulSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.find('div', {'id' : 'story'})
-        
+        soup = self.make_soup(self._fetchUrl(url))
+
+        chaptertag = soup.find('div', {'id' : 'story'})
+        if not chaptertag:
+            logger.debug("div id=story not found, try article")
+            chaptertag = soup.find('article', {'id' : 'story'})
+
        # some big chapters are split over several pages
-        pager = div.find('span', {'class' : 'pager'})
+        pager = chaptertag.find('span', {'class' : 'pager'})
+
+        self.cleanPage(chaptertag)
+
        if pager != None:
-            a = pager.previousSibling
-            while a != None:
-                logger.debug("before pager: {0}".format(a))
-                b = a.previousSibling
-                a.extract()
-                a = b

            urls=pager.findAll('a')
            urls=urls[:len(urls)-1]
+            logger.debug("pager urls:%s"%urls)
            pager.extract()
-            div.contents = div.contents[2:]
-#            logger.debug(div)
-                        
+            chaptertag.contents = chaptertag.contents[2:]
+
            for ur in urls:
-                soup = bs.BeautifulSoup(self._fetchUrl("http://"+self.getSiteDomain()+ur['href']),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-                div1 = soup.find('div', {'id' : 'story'})
-                
-                # Find the "Continues" marker on the current page and remove everything after that. 
-                continues = div.find('span', {'class' : 'conTag'})
-                if continues != None:
-                    while continues != None:
-#                        logger.debug("removing end: {0}".format(continues))
-                        b = continues.nextSibling
-                        continues.extract()
-                        continues = b
+                soup = self.make_soup(self._fetchUrl("http://"+self.getSiteDomain()+ur['href']))

-                # Find the "Continued" marker and delete everything before that
-                continued = div1.find('span', {'class' : 'conTag'})
-                if continued != None:
-                    a = continued.previousSibling
-                    while a != None:
-#                        logger.debug("before conTag: {0}".format(a))
-                        b = a.previousSibling
-                        a.extract()
-                        a = b
-                # Remove the pager from the end if this is the last page
-                endPager = div1.find('span', {'class' : 'pager'})
-                if endPager != None:
-                    b = endPager.nextSibling
-                    while endPager != None:
-                        logger.debug("removing end: {0}".format(endPager))
-                        b = endPager.nextSibling
-                        endPager.extract()
-                        endPager = b
-                    div1.contents = div1.contents[:len(div1) - 2]
-#                logger.debug("after removing pager: {0}".format(div1))
-                for tag in div1.contents[2:]:
-                    div.append(tag)
+                pagetag = soup.find('div', {'id' : 'story'})
+                if not pagetag:
+                    logger.debug("div id=story not found, try article")
+                    pagetag = soup.find('article', {'id' : 'story'})

-        # If it is a chapter, there are dates at the start for when it was posted or modified. These plus 
-        # everything before them can be discarded. 
-        postedDates = div.findAll('div', {'class' : 'date'})
+                self.cleanPage(pagetag)
+
+                for tag in pagetag.contents[2:]:
+                    chaptertag.append(tag)
+
+
+        if None == chaptertag:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+
+        return self.utf8FromSoup(url,chaptertag)
+
+    def cleanPage(self,pagetag):
+        "Consolidate 'page' clean up code so it can be called."
+
+        # some big chapters are split over several pages
+        # remove FIRST pager and everything before it.
+        tag = pagetag.find('span', {'class' : 'pager'})
+        while tag != None:
+            # logger.debug("remove before pager: {0}".format(tag))
+            prev = tag.previousSibling
+            tag.extract()
+            tag = prev
+
+        # Find the "Continues" marker on the current page and
+        # remove everything after that.  This is actually
+        # effecting the *previous* 'page'.  EXCEPT!--they are
+        # putting a 'conTag' at the *top* now, too.  So this
+        # was nuking every page but the first and last.  Now
+        # only if 'Continues'
+        for contag in pagetag.findAll('span', {'class' : 'conTag'}):
+            # remove everything after continues...
+            if 'Continuation' in contag.text:
+                tag = contag
+                while tag != None:
+                    # logger.debug("remove before Continuation: {0}".format(tag))
+                    prev = tag.previousSibling
+                    tag.extract()
+                    tag = prev
+            elif 'Continues' in contag.text:
+                tag = contag
+                while tag != None:
+                    # logger.debug("remove after Continues: {0}".format(tag))
+                    nxt = tag.nextSibling
+                    tag.extract()
+                    tag = nxt
+
+        # some big chapters are split over several pages
+        # remove LAST pager and everything before it.
+        # Only needed on last page.
+        tag = pagetag.find('span', {'class' : 'pager'})
+        while tag != None:
+            # logger.debug("remove after pager: {0}".format(tag))
+            nxt = tag.nextSibling
+            tag.extract()
+            tag = nxt
+
+        # If it is a chapter, there are dates at the start for when it was posted or modified. These plus
+        # everything before them can be discarded.
+        postedDates = pagetag.findAll('div', {'class' : 'date'})
        if postedDates:
            a = postedDates[0].previousSibling
            while a != None:
-#                logger.debug("before dates: {0}".format(a))
+                # logger.debug("before dates: {0}".format(a))
                b = a.previousSibling
                a.extract()
                a = b
-            for a in div.findAll('div', {'class' : 'date'}):
+            for a in pagetag.findAll('div', {'class' : 'date'}):
                a.extract()

        # For single chapter stories, there is a copyright statement. Remove this and everything
        # before it.
-        copy = div.find('h4', {'class': 'copy'})
+        copy = pagetag.find('h4', {'class': 'copy'})
        while copy != None:
-#            logger.debug("before copyright: {0}".format(copy))
+            # logger.debug("before copyright: {0}".format(copy))
            b = copy.previousSibling
            copy.extract()
            copy = b

        # For a story or the last chapter, remove voting form and the in library box
-        a = div.find('div', {'id' : 'vote-form'})
+        a = pagetag.find('div', {'id' : 'vote-form'})
        if a != None:
            a.extract()
-        a = div.find('div', {'id' : 'b-man-div'})
+        a = pagetag.find('div', {'id' : 'top-header'})
+        if a != None:
+            a.extract()
+        a = pagetag.find('div', {'id' : 'b-man-div'})
        if a != None:
            a.extract()

-        # Kill the "The End" header and everything after it.
-        a = div.find(['h2', 'h3'], {'class' : 'end'})
-        logger.debug("Chapter end= '{0}'".format(a))
+        # Kill the vote form and everything after it.
+        a = pagetag.find('div', {'class' : 'vform'})
+        # logger.debug("Chapter end= '{0}'".format(a))
        while a != None:
            b = a.nextSibling
            a.extract()
            a=b

+        # Kill the vote form and everything after it.
+        a = pagetag.find('h3', {'class' : 'end'})
+        # logger.debug("Chapter end= '{0}'".format(a))
+        while a != None:
+            b = a.nextSibling
+            a.extract()
+            a=b
+
+        foot = pagetag.find('footer')
+        if foot != None:
+            foot.extract()
+

-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)
@@ -81,6 +81,12 @@ class TestSiteAdapter(BaseSiteAdapter):
            
            return

+        if idnum >= 700 and idnum <= 710:
+            self._setURL('http://test1.com?sid=%s'%(idnum+100))
+            self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+            idstr = self.story.getMetadata('storyId')
+            idnum = int(idstr)
+
        if idstr == '665' and not (self.is_adult or self.getConfig("is_adult")):
            logger.warn("self.is_adult:%s"%self.is_adult)
            raise exceptions.AdultCheckRequired(self.url)
@@ -104,6 +104,8 @@ class BaseSiteAdapter(Configurable):
        self.chapterFirst = None
        self.chapterLast = None
        self.oldchapters = None
+        self.oldchaptersmap = None
+        self.oldchaptersdata = None
        self.oldimgs = None
        self.oldcover = None # (data of existing cover html, data of existing cover image)
        self.calibrebookmark = None
@@ -140,28 +142,6 @@ class BaseSiteAdapter(Configurable):
        '''
        self.get_cookiejar().load(filename, ignore_discard=True, ignore_expires=True)
        
-    # def save_cookiejar(self,filename):
-    #     '''
-    #     Assumed to be a FileCookieJar if self.cookiejar set.
-    #     Takes file *name*.
-    #     '''
-    #     self.get_cookiejar().save(filename, ignore_discard=True, ignore_expires=True)
-
-    # def save_pagecache(self,filename):
-    #     '''
-    #     Writes pickle of pagecache to file *name*
-    #     '''
-    #     with open(filename, 'wb') as f:
-    #         pickle.dump(self.get_pagecache(),
-    #                     f,protocol=pickle.HIGHEST_PROTOCOL)
-        
-    # def load_pagecache(self,filename):
-    #     '''
-    #     Reads pickle of pagecache from file *name*
-    #     '''
-    #     with open(filename, 'rb') as f:
-    #         self.set_pagecache(pickle.load(f))
-
    def get_pagecache(self):
        return self.pagecache
    
@@ -185,9 +165,9 @@ class BaseSiteAdapter(Configurable):
        else:
            return None

-    def _set_to_pagecache(self,cachekey,data):
+    def _set_to_pagecache(self,cachekey,data,redirectedurl):
        if self.use_pagecache():
-            self.get_pagecache()[cachekey] = data
+            self.get_pagecache()[cachekey] = (data,redirectedurl)

    def use_pagecache(self):
        '''
@@ -257,7 +237,8 @@ class BaseSiteAdapter(Configurable):
        cachekey=self._get_cachekey(url, parameters, headers)
        if usecache and self._has_cachekey(cachekey):
            logger.debug("#####################################\npagecache HIT: %s"%cachekey)
-            return self._get_from_pagecache(cachekey)
+            data,redirecturl = self._get_from_pagecache(cachekey)
+            return data
        
        logger.debug("#####################################\npagecache MISS: %s"%cachekey)
        self.do_sleep(extrasleep)
@@ -272,13 +253,23 @@ class BaseSiteAdapter(Configurable):
                         data=urllib.urlencode(parameters),
                         headers=headers)
        data = self._decode(self.opener.open(req,None,float(self.getConfig('connect_timeout',30.0))).read())
-        self._set_to_pagecache(cachekey,data)
+        self._set_to_pagecache(cachekey,data,url)
        return data

    def _fetchUrlRaw(self, url,
                     parameters=None,
                     extrasleep=None,
                     usecache=True):
+        
+        return self._fetchUrlRawOpened(url,
+                                       parameters,
+                                       extrasleep,
+                                       usecache)[0]
+        
+    def _fetchUrlRawOpened(self, url,
+                           parameters=None,
+                           extrasleep=None,
+                           usecache=True):
        '''
        When should cache be cleared or not used? logins...
        
@@ -289,16 +280,25 @@ class BaseSiteAdapter(Configurable):
        cachekey=self._get_cachekey(url, parameters)
        if usecache and self._has_cachekey(cachekey):
            logger.debug("#####################################\npagecache HIT: %s"%cachekey)
-            return self._get_from_pagecache(cachekey)
+            data,redirecturl = self._get_from_pagecache(cachekey)
+            class FakeOpened:
+                def __init__(self,data,url):
+                    self.data=data
+                    self.url=url
+                def geturl(self): return self.url
+                def read(self): return self.data
+            return (data,FakeOpened(data,redirecturl))
        
        logger.debug("#####################################\npagecache MISS: %s"%cachekey)
        self.do_sleep(extrasleep)
        if parameters != None:
-            data = self.opener.open(url.replace(' ','%20'),urllib.urlencode(parameters),float(self.getConfig('connect_timeout',30.0))).read()
+            opened = self.opener.open(url.replace(' ','%20'),urllib.urlencode(parameters),float(self.getConfig('connect_timeout',30.0)))
        else:
-            data = self.opener.open(url.replace(' ','%20'),None,float(self.getConfig('connect_timeout',30.0))).read()
-        self._set_to_pagecache(cachekey,data)
-        return data
+            opened = self.opener.open(url.replace(' ','%20'),None,float(self.getConfig('connect_timeout',30.0)))
+        data = opened.read()
+        self._set_to_pagecache(cachekey,data,opened.url)
+        
+        return (data,opened)

    def set_sleep(self,val):
        logger.debug("\n===========\n set sleep time %s\n==========="%val)
@@ -312,20 +312,30 @@ class BaseSiteAdapter(Configurable):
        elif self.getConfig('slow_down_sleep_time'):
            time.sleep(float(self.getConfig('slow_down_sleep_time')))
        
-    # parameters is a dict()
    def _fetchUrl(self, url,
                  parameters=None,
                  usecache=True,
                  extrasleep=None):
+        return self._fetchUrlOpened(url,
+                                    parameters,
+                                    usecache,
+                                    extrasleep)[0]
+
+    # parameters is a dict()
+    def _fetchUrlOpened(self, url,
+                        parameters=None,
+                        usecache=True,
+                        extrasleep=None):

        excpt=None
        for sleeptime in [0, 0.5, 4, 9]:
            time.sleep(sleeptime)	
            try:
-                return self._decode(self._fetchUrlRaw(url,
+                (data,opened)=self._fetchUrlRawOpened(url,
                                                      parameters=parameters,
                                                      usecache=usecache,
-                                                      extrasleep=extrasleep))
+                                                      extrasleep=extrasleep)
+                return (self._decode(data),opened)
            except u2.HTTPError, he:
                excpt=he
                if he.code == 404:
@@ -345,6 +355,7 @@ class BaseSiteAdapter(Configurable):
            self.chapterFirst=int(first)-1
        if last:
            self.chapterLast=int(last)-1
+        self.story.set_chapters_range(first,last)
    
    # Does the download the first time it's called.
    def getStory(self):
@@ -352,21 +363,40 @@ class BaseSiteAdapter(Configurable):
            self.getStoryMetadataOnly(get_cover=True)

            for index, (title,url) in enumerate(self.chapterUrls):
+                newchap = False
                if (self.chapterFirst!=None and index < self.chapterFirst) or \
                        (self.chapterLast!=None and index > self.chapterLast):
                    self.story.addChapter(url,
                                          removeEntities(title),
                                          None)
                else:
-                    if self.oldchapters and index < len(self.oldchapters):
+                    data = None
+                    if self.oldchaptersmap:
+                        if url in self.oldchaptersmap:
+                            data = self.utf8FromSoup(None,
+                                                     self.oldchaptersmap[url],
+                                                     partial(cachedfetch,self._fetchUrlRaw,self.oldimgs))
+                    elif self.oldchapters and index < len(self.oldchapters):
                        data = self.utf8FromSoup(None,
                                                 self.oldchapters[index],
                                                 partial(cachedfetch,self._fetchUrlRaw,self.oldimgs))
-                    else:
+
+                    # if already marked new -- ie, origtitle and title don't match
+                    # logger.debug("self.oldchaptersdata[url]:%s"%(self.oldchaptersdata[url]))
+                    newchap = (self.oldchaptersdata is not None and
+                               url in self.oldchaptersdata and (
+                            self.oldchaptersdata[url]['chapterorigtitle'] !=
+                            self.oldchaptersdata[url]['chaptertitle']) )
+                    
+                    if not data:
                        data = self.getChapterText(url)
+                        # if had to fetch and has existing chapters
+                        newchap = bool(self.oldchapters or self.oldchaptersmap)
+                        
                    self.story.addChapter(url,
                                          removeEntities(title),
-                                          removeEntities(data))
+                                          removeEntities(data),
+                                          newchap)
            self.storyDone = True
            
            # include image, but no cover from story, add default_cover_image cover.
@@ -379,10 +409,12 @@ class BaseSiteAdapter(Configurable):
                                                               self.getConfig('allow_unsafe_filename')),
                                     self._fetchUrlRaw,
                                     cover=True)
+                self.story.setMetadata('cover_image','default')

            # no new cover, set old cover, if there is one.
            if not self.story.cover and self.oldcover:
                self.story.oldcover = self.oldcover
+                self.story.setMetadata('cover_image','old')
                
            # cheesy way to carry calibre bookmark file forward across update.
            if self.calibrebookmark:
@@ -397,11 +429,24 @@ class BaseSiteAdapter(Configurable):
            self.doExtractChapterUrlsAndMetadata(get_cover=get_cover)
            
            if not self.story.getMetadataRaw('dateUpdated'):
-                self.story.setMetadata('dateUpdated',self.story.getMetadataRaw('datePublished'))
+                if self.story.getMetadataRaw('datePublished'):
+                    self.story.setMetadata('dateUpdated',self.story.getMetadataRaw('datePublished'))
+                else:
+                    self.story.setMetadata('dateUpdated',self.story.getMetadataRaw('dateCreated'))                

            self.metadataDone = True
        return self.story

+    def setStoryMetadata(self,metahtml):
+        if metahtml:
+            self.story.load_html_metadata(metahtml)
+            self.metadataDone = True
+            if not self.story.getMetadataRaw('dateUpdated'):
+                if self.story.getMetadataRaw('datePublished'):
+                    self.story.setMetadata('dateUpdated',self.story.getMetadataRaw('datePublished'))
+                else:
+                    self.story.setMetadata('dateUpdated',self.story.getMetadataRaw('dateCreated'))                
+    
    def hookForUpdates(self,chaptercount):
        "Usually not needed."
        return chaptercount
@@ -418,6 +463,11 @@ class BaseSiteAdapter(Configurable):
        "Only needs to be overriden if != site domain."
        return cls.getSiteDomain()
    
+    @classmethod
+    def getConfigSections(cls):
+        "Only needs to be overriden if has additional ini sections."
+        return [cls.getConfigSection()]
+    
    @classmethod
    def stripURLParameters(cls,url):
        "Only needs to be overriden if URL contains more than one parameter"
@@ -467,6 +517,13 @@ class BaseSiteAdapter(Configurable):

    def setDescription(self,url,svalue):
        #print("\n\nsvalue:\n%s\n"%svalue)
+        strval = u"%s"%svalue # works for either soup or string
+        if self.hasConfig('description_limit'):
+            limit = int(self.getConfig('description_limit'))
+            if limit and len(strval) > limit:
+                svalue = strval[:limit]
+
+        #print(u"[[[[[\n\n%s\n\n]]]]]]]]"%svalue) # works for either soup or string
        if self.getConfig('keep_summary_html'):
            if isinstance(svalue,basestring):
                # bs4/html5lib add html, header and body tags, which
@@ -479,8 +536,10 @@ class BaseSiteAdapter(Configurable):

    def setCoverImage(self,storyurl,imgurl):
        if self.getConfig('include_images'):
-            self.story.addImgUrl(storyurl,imgurl,self._fetchUrlRaw,cover=True,
-                                 coverexclusion=self.getConfig('cover_exclusion_regexp'))
+            return self.story.addImgUrl(storyurl,imgurl,self._fetchUrlRaw,cover=True,
+                                        coverexclusion=self.getConfig('cover_exclusion_regexp'))
+        else:
+            return (None,None)

    # bs3 & bs4 are different here.
    # will move to a bs3 vs bs4 block if there's lots of changes.
@@ -541,7 +600,8 @@ class BaseSiteAdapter(Configurable):
                    if t.name not in ('p') and t.string != None and len(t.string.strip()) == 0 :
                        t.extract()
        except AttributeError, ae:
-            logger.error("Error parsing HTML, probably poor input HTML. %s"%ae)
+            if "%s"%ae != "'NoneType' object has no attribute 'next_element'":
+                logger.error("Error parsing HTML, probably poor input HTML. %s"%ae)
        
        retval = unicode(soup)

@@ -70,6 +70,11 @@ class BaseEfictionAdapter(BaseSiteAdapter):
        self.triedAcceptWarnings = False
        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.

+    @classmethod
+    def getConfigSections(cls):
+        "Only needs to be overriden if has additional ini sections."
+        return ['base_efiction',cls.getConfigSection()]
+    
    @classmethod
    def getAcceptDomains(cls):
        return [cls.getSiteDomain(),'www.' + cls.getSiteDomain()]
@@ -89,7 +94,6 @@ class BaseEfictionAdapter(BaseSiteAdapter):
        """
        return ["Windows-1252", "utf8"]

-
    @classmethod
    def getPathToArchive(cls):
        """
@@ -144,7 +148,7 @@ class BaseEfictionAdapter(BaseSiteAdapter):
        """
        Get the URL to a user page on this site.
        """
-        return "%s?sid=%s" % (self.getUrlForPhp(self.getViewUserPhpName()), userId)
+        return "%s?uid=%s" % (self.getUrlForPhp(self.getViewUserPhpName()), userId)

    @classmethod
    def getLoginUrl(self):
@@ -265,7 +269,6 @@ class BaseEfictionAdapter(BaseSiteAdapter):
                else:
                    super(NameOfMyAdapter, self).handleMetadata(key, value)
        """
-        # logger.debug("metadata: '%s' == '%s'" % (key, value))
        if value == 'None':
            return
        elif key == 'Summary':
@@ -288,7 +291,7 @@ class BaseEfictionAdapter(BaseSiteAdapter):
                self.story.addToList('challenge', val)
        elif key == 'Chapters':
            self.story.setMetadata('numChapters', int(value))
-        elif key == 'Rating':
+        elif key == 'Rating' or key == 'Rated':
            self.story.setMetadata('rating', value)
        elif key == 'Word count':
            self.story.setMetadata('numWords', value)
@@ -0,0 +1,244 @@
+#  -*- coding: utf-8 -*-
+
+# Copyright 2015 FanFicFare team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+logger = logging.getLogger(__name__)
+
+class BaseXenForoForumAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["utf8",
+                       "Windows-1252"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+							   
+							   
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])        
+        
+        # get storyId from url--url validation guarantees query correct
+        m = re.match(self.getSiteURLPattern(),url)
+        if m:
+            self.story.setMetadata('storyId',m.group('id'))
+
+            # normalized story URL.
+            self._setURL(self.getURLPrefix() + '/'+m.group('tp')+'/'+self.story.getMetadata('storyId')+'/')
+        else:
+            raise exceptions.InvalidStoryURL(url,
+                                             self.getSiteDomain(),
+                                             self.getSiteExampleURLs())
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','fsb')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%b %d, %Y at %I:%M %p"
+            
+    @classmethod
+    def getConfigSections(cls):
+        "Only needs to be overriden if has additional ini sections."
+        return ['base_xenforoforum',cls.getConfigSection()]
+    
+    @classmethod
+    def getURLPrefix(cls):
+        # The site domain.  Does have www here, if it uses it.
+        return 'https://' + cls.getSiteDomain() 
+
+    @classmethod
+    def getSiteExampleURLs(cls):
+        return cls.getURLPrefix()+"/threads/some-story-name.123456/"
+
+    def getSiteURLPattern(self):
+        return r"https?://"+re.escape(self.getSiteDomain())+r"/(?P<tp>threads|posts)/(.+\.)?(?P<id>\d+)/?"
+        
+    def use_pagecache(self):
+        '''
+        adapters that will work with the page cache need to implement
+        this and change it to True.
+        '''
+        return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        useurl = self.url
+        logger.info("url: "+useurl)
+
+        try:
+            (data,opened) = self._fetchUrlOpened(useurl)
+            useurl = opened.geturl()
+            logger.info("use useurl: "+useurl)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = self.make_soup(data)
+
+        a = soup.find('h3',{'class':'userText'}).find('a')
+        self.story.addToList('authorId',a['href'].split('/')[1])
+        self.story.addToList('authorUrl',self.getURLPrefix()+'/'+a['href'])
+        self.story.addToList('author',a.text)
+
+        h1 = soup.find('div',{'class':'titleBar'}).h1
+        self.story.setMetadata('title',stripHTML(h1))
+        
+        if '#' in useurl:
+            anchorid = useurl.split('#')[1]
+            soup = soup.find('li',id=anchorid)
+        else:
+            # try threadmarks if no '#' in , require at least 2.
+            threadmarksa = soup.find('a',{'class':'threadmarksTrigger'})
+            if threadmarksa:
+                soupmarks = self.make_soup(self._fetchUrl(self.getURLPrefix()+'/'+threadmarksa['href']))
+                markas = soupmarks.find('ol',{'class':'overlayScroll'}).find_all('a')
+                if len(markas) > 1:
+                    for (atag,url,name) in [ (x,x['href'],stripHTML(x)) for x in markas ]:
+                        date = self.make_date(atag.find_next_sibling('div',{'class':'extra'}))
+                        if not self.story.getMetadataRaw('datePublished') or date < self.story.getMetadataRaw('datePublished'):
+                            self.story.setMetadata('datePublished', date)
+                        if not self.story.getMetadataRaw('dateUpdated') or date > self.story.getMetadataRaw('dateUpdated'):
+                            self.story.setMetadata('dateUpdated', date)
+                            
+                        self.chapterUrls.append((name,self.getURLPrefix()+'/'+url))
+
+                    ## only use tags if threadmarks for chapters.
+                    ##  a bit arbitrary, but likely.
+                    for tag in soup.findAll('a',{'class':'tag'}):
+                        self.story.addToList('forumtags',stripHTML(tag))
+
+            soup = soup.find('li',{'class':'message'}) # limit first post for date stuff below. ('#' posts above)
+                
+        # Now go hunting for the 'chapter list'.
+        bq = soup.find('blockquote') # assume first posting contains TOC urls.
+        
+        bq.name='div'
+
+        for iframe in bq.find_all('iframe'):
+            iframe.extract() # calibre book reader & editor don't like iframes to youtube.
+
+        for qdiv in bq.find_all('div',{'class':'quoteExpand'}):
+            qdiv.extract() # Remove <div class="quoteExpand">click to expand</div>
+            
+        self.setDescription(useurl,bq)
+
+        # otherwise, use first post links--include first post since
+        # that's often also the first chapter.
+        if not self.chapterUrls:
+            self.chapterUrls.append(("First Post",useurl))
+            for (url,name) in [ (x['href'],stripHTML(x)) for x in bq.find_all('a') ]:
+                logger.debug("found chapurl:%s"%url)
+                if not url.startswith('http'):
+                    url = self.getURLPrefix()+'/'+url
+    
+                if ( url.startswith(self.getURLPrefix()) or
+                     url.startswith('http://'+self.getSiteDomain()) or
+                     url.startswith('https://'+self.getSiteDomain()) ) and ('/posts/' in url or '/threads/' in url):
+                    # brute force way to deal with SB's http->https change when hardcoded http urls.
+                    url = url.replace('http://'+self.getSiteDomain(),self.getURLPrefix())
+                    logger.debug("used chapurl:%s"%(url))
+                    self.chapterUrls.append((name,url))
+                    if url == useurl and 'First Post' == self.chapterUrls[0][0]:
+                        # remove "First Post" if included in list.
+                        logger.debug("delete dup 'First Post' chapter: %s %s"%self.chapterUrls[0])
+                        del self.chapterUrls[0]
+                        
+            # Didn't use threadmarks, so take created/updated dates
+            # from the 'first' posting created and updated.
+            date = self.make_date(soup.find('a',{'class':'datePermalink'}))
+            if date:
+                self.story.setMetadata('datePublished', date)
+                self.story.setMetadata('dateUpdated', date) # updated overwritten below if found.
+        
+            date = self.make_date(soup.find('div',{'class':'editDate'}))
+            if date:
+                self.story.setMetadata('dateUpdated', date) 
+            
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+    def make_date(self,parenttag): # forums use a BS thing where dates
+                                  # can appear different if recent.
+        datestr=None
+        try:
+            datetag = parenttag.find('span',{'class':'DateTime'})
+            if datetag:
+                datestr = datetag['title']
+            else:
+                datetag = parenttag.find('abbr',{'class':'DateTime'})
+                if datetag:
+                    datestr="%s at %s"%(datetag['data-datestring'],datetag['data-timestring'])
+            # Apr 24, 2015 at 4:39 AM
+            # May 1, 2015 at 5:47 AM
+            datestr = re.sub(r' (\d[^\d])',r' 0\1',datestr) # add leading 0 for single digit day & hours.
+            return makeDate(datestr, self.dateformat)
+        except:
+            logger.debug('No date found in %s'%parenttag)
+            return None
+        
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+        logger.debug('Getting chapter text from: %s' % url)
+
+        ## there's some history of stories with links to the wrong
+        ## page.  This changes page#post URLs to perma-link URLs.
+        ## Which will be redirected back to page#posts, but the
+        ## *correct* ones.
+        # http://forums.sufficientvelocity.com/threads/harry-potter-and-the-not-fatal-at-all-cultural-exchange-program.330/page-4#post-39915
+        # https://forums.sufficientvelocity.com/posts/39915/
+        if '#post-' in url:
+            url = self.getURLPrefix()+'/posts/'+url.split('#post-')[1]+'/'
+        
+        origurl = url
+        (data,opened) = self._fetchUrlOpened(url)
+        url = opened.geturl()
+        if '#' in origurl and '#' not in url:
+            url = url + origurl[origurl.index('#'):]
+        logger.debug("chapter URL redirected to: %s"%url)
+
+        soup = self.make_soup(data)
+
+        if '#' in url:
+            anchorid = url.split('#')[1]
+            soup = soup.find('li',id=anchorid)
+                
+        bq = soup.find('blockquote')
+
+        bq.name='div'
+
+        for iframe in bq.find_all('iframe'):
+            iframe.extract() # calibre book reader & editor don't like iframes to youtube.
+
+        for qdiv in bq.find_all('div',{'class':'quoteExpand'}):
+            qdiv.extract() # Remove <div class="quoteExpand">click to expand</div>
+
+        return self.utf8FromSoup(url,bq)
@@ -16,7 +16,7 @@
 #

 from optparse import OptionParser
-from os.path import expanduser, isfile, join, dirname
+from os.path import expanduser, join, dirname
 from os import access, R_OK
 from subprocess import call
 import ConfigParser
@@ -41,12 +41,14 @@ try:
    # running under calibre
    from calibre_plugins.fanfictiondownloader_plugin.fanficfare import adapters, writers, exceptions
    from calibre_plugins.fanfictiondownloader_plugin.fanficfare.configurable import Configuration
-    from calibre_plugins.fanfictiondownloader_plugin.fanficfare.epubutils import get_dcsource_chaptercount, get_update_data
+    from calibre_plugins.fanfictiondownloader_plugin.fanficfare.epubutils import (
+        get_dcsource_chaptercount, get_update_data, reset_orig_chapters_epub)
    from calibre_plugins.fanfictiondownloader_plugin.fanficfare.geturls import get_urls_from_page
 except ImportError:
    from fanficfare import adapters, writers, exceptions
    from fanficfare.configurable import Configuration
-    from fanficfare.epubutils import get_dcsource_chaptercount, get_update_data
+    from fanficfare.epubutils import (
+        get_dcsource_chaptercount, get_update_data, reset_orig_chapters_epub)
    from fanficfare.geturls import get_urls_from_page


@@ -87,6 +89,9 @@ def main(argv=None, parser=None, passed_defaultsini=None, passed_personalini=Non
    parser.add_option('-u', '--update-epub',
                      action='store_true', dest='update',
                      help='Update an existing epub with new chapters, give epub filename instead of storyurl.', )
+    parser.add_option('--unnew',
+                      action='store_true', dest='unnew',
+                      help='Remove (new) chapter marks left by mark_new_chapters setting.', )
    parser.add_option('--update-cover',
                      action='store_true', dest='updatecover',
                      help='Update cover in an existing epub, otherwise existing cover (if any) is used on update.  Only valid with --update-epub.', )
@@ -129,6 +134,9 @@ def main(argv=None, parser=None, passed_defaultsini=None, passed_personalini=Non
    if options.update and options.format != 'epub':
        parser.error('-u/--update-epub only works with epub')

+    if options.unnew and options.format != 'epub':
+        parser.error('--unnew only works with epub')
+
    # for passing in a file list
    if options.infile:
        urls=[]
@@ -144,14 +152,20 @@ def main(argv=None, parser=None, passed_defaultsini=None, passed_personalini=Non
    else:
        urls = args

-    for url in urls:
-        try:
-            do_download(url,
-                        options,
-                        passed_defaultsini,
-                        passed_personalini)
-        except Exception, e:
-            print "URL(%s) Failed: Exception (%s). Run URL individually for more detail."%(url,e)
+    if len(urls) > 1:
+        for url in urls:
+            try:
+                do_download(url,
+                            options,
+                            passed_defaultsini,
+                            passed_personalini)
+            except Exception, e:
+                print "URL(%s) Failed: Exception (%s). Run URL individually for more detail."%(url,e)
+    else:
+        do_download(urls[0],
+                    options,
+                    passed_defaultsini,
+                    passed_personalini)

 # make rest a function and loop on it.
 def do_download(arg,
@@ -162,6 +176,12 @@ def do_download(arg,
    # Attempt to update an existing epub.
    chaptercount = None
    output_filename = None
+
+    if options.unnew:
+        # remove mark_new_chapters marks
+        reset_orig_chapters_epub(arg,arg)
+        return
+    
    if options.update:
        try:
            url, chaptercount = get_dcsource_chaptercount(arg)
@@ -178,7 +198,7 @@ def do_download(arg,
        url = arg
        
    try:
-        configuration = Configuration(adapters.getConfigSectionFor(url), options.format)
+        configuration = Configuration(adapters.getConfigSectionsFor(url), options.format)
    except exceptions.UnknownSite, e:
        if options.list or options.normalize:
            # list for page doesn't have to be a supported site.
@@ -194,25 +214,18 @@ def do_download(arg,
    if passed_defaultsini:
        configuration.readfp(passed_defaultsini)

-    if isfile(join(dirname(__file__), 'defaults.ini')):
-        conflist.append(join(dirname(__file__), 'defaults.ini'))
-        
-    if isfile(join(homepath, 'defaults.ini')):
-        conflist.append(join(homepath, 'defaults.ini'))
-    if isfile(join(homepath2, 'defaults.ini')):
-        conflist.append(join(homepath2, 'defaults.ini'))
-    if isfile('defaults.ini'):
-        conflist.append('defaults.ini')
+    # don't need to check existance for our selves.
+    conflist.append(join(dirname(__file__), 'defaults.ini'))
+    conflist.append(join(homepath, 'defaults.ini'))
+    conflist.append(join(homepath2, 'defaults.ini'))
+    conflist.append('defaults.ini')

    if passed_personalini:
        configuration.readfp(passed_personalini)

-    if isfile(join(homepath, 'personal.ini')):
-        conflist.append(join(homepath, 'personal.ini'))
-    if isfile(join(homepath2, 'personal.ini')):
-        conflist.append(join(homepath2, 'personal.ini'))
-    if isfile('personal.ini'):
-        conflist.append('personal.ini')
+    conflist.append(join(homepath, 'personal.ini'))
+    conflist.append(join(homepath2, 'personal.ini'))
+    conflist.append('personal.ini')

    if options.configfile:
        conflist.extend(options.configfile)
@@ -312,7 +325,15 @@ def do_download(arg,
                # update now handled by pre-populating the old
                # images and chapters in the adapter rather than
                # merging epubs.
-                url, chaptercount, adapter.oldchapters, adapter.oldimgs, adapter.oldcover, adapter.calibrebookmark, adapter.logfile = get_update_data(output_filename)
+                (url,
+                 chaptercount,
+                 adapter.oldchapters,
+                 adapter.oldimgs,
+                 adapter.oldcover,
+                 adapter.calibrebookmark,
+                 adapter.logfile,
+                 adapter.oldchaptersmap,
+                 adapter.oldchaptersdata) = (get_update_data(output_filename))[0:9]

                print 'Do update - epub(%d) vs url(%d)' % (chaptercount, urlchaptercount)

@@ -17,6 +17,7 @@

 import ConfigParser, re
 import exceptions
+import codecs
 from ConfigParser import DEFAULTSECT, MissingSectionHeaderError, ParsingError

 # All of the writers(epub,html,txt) and adapters(ffnet,twlt,etc)
@@ -41,15 +42,48 @@ def re_compile(regex,line):
        return re.compile(regex)
    except Exception, e: 
        raise exceptions.RegularExpresssionFailed(e,regex,line)
-    
+
+# fall back labels.
+titleLabels = {
+    'category':'Category',
+    'genre':'Genre',
+    'language':'Language',
+    'status':'Status',
+    'series':'Series',
+    'characters':'Characters',
+    'ships':'Relationships',
+    'datePublished':'Published',
+    'dateUpdated':'Updated',
+    'dateCreated':'Packaged',
+    'rating':'Rating',
+    'warnings':'Warnings',
+    'numChapters':'Chapters',
+    'numWords':'Words',
+    'site':'Site',
+    'storyId':'Story ID',
+    'authorId':'Author ID',
+    'extratags':'Extra Tags',
+    'title':'Title',
+    'storyUrl':'Story URL',
+    'description':'Summary',
+    'author':'Author',
+    'authorUrl':'Author URL',
+    'formatname':'File Format',
+    'formatext':'File Extension',
+    'siteabbrev':'Site Abbrev',
+    'version':'Downloader Version'
+    }
+
 formatsections = ['html','txt','epub','mobi']
 othersections = ['defaults','overrides']

 def get_valid_sections():
-    sites = adapters.getConfigSections()
+    sites = adapters.getConfigSections() 
    sitesections = list(othersections)
    for section in sites:
        sitesections.append(section)
+        # also allows [www.base_efiction] and [www.base_forum]. Not
+        # likely to matter.
        if section.startswith('www.'):
            # add w/o www if has www
            sitesections.append(section[4:])
@@ -99,6 +133,7 @@ def get_valid_set_options():
               'replace_hr':(None,None,boollist),
               'sort_ships':(None,None,boollist),
               'strip_chapter_numbers':(None,None,boollist),
+               'mark_new_chapters':(None,None,boollist),
               'titlepage_use_table':(None,None,boollist),
               
               'use_ssl_unverified_context':(None,None,boollist),
@@ -111,22 +146,18 @@ def get_valid_set_options():
               
               'fix_fimf_blockquotes':(['fimfiction.net'],None,boollist),
               'fail_on_password':(['fimfiction.net'],None,boollist),
-               'do_update_hook':(['fimfiction.net'],None,boollist),
+               'do_update_hook':(['fimfiction.net',
+                                  'archiveofourown.org'],None,boollist),

               'force_login':(['phoenixsong.net'],None,boollist),
               'non_breaking_spaces':(['fictionmania.tv'],None,boollist),
               'universe_as_series':(['storiesonline.net'],None,boollist),
               'strip_text_links':(['bloodshedverse.com'],None,boollist),

-               # eFiction Base
-               'bulk_load':(['fannation.shades-of-moonlight.com',
-                             'fhsarchive.com',
-                             'lotrfanfiction.com',
-                             'themaplebookshelf.com',
-                             'devianthearts.com',
-                             'thehookupzone.net',
-                             'libraryofmoria.com',
-                             ],None,boollist),
+               # eFiction Base adapters allow bulk_load
+               # kept forgetting to add them, so now it's automatic.
+               'bulk_load':(adapters.get_bulk_load_sites(),
+                            None,boollist),
               
               'include_logpage':(None,['epub'],boollist+['smart']),
               
@@ -136,6 +167,7 @@ def get_valid_set_options():
               'grayscale_images':(None,['epub','html'],boollist),
               'no_image_processing':(None,['epub','html'],boollist),
               }
+
    return dict(valdict)

 def get_valid_scalar_entries():
@@ -163,6 +195,7 @@ def get_valid_scalar_entries():
                 'seriesHTML',
                 'langcode',
                 'output_css',
+                 'cover_image',
                 ])

 def get_valid_entries():
@@ -173,6 +206,7 @@ def get_valid_keywords():
    return list(['(in|ex)clude_metadata_(pre|post)',
                 'add_chapter_numbers',
                 'add_genre_when_multi_category',
+                 'adult_ratings',
                 'allow_unsafe_filename',
                 'always_overwrite',
                 'anthology_tags',
@@ -181,8 +215,13 @@ def get_valid_keywords():
                 'bulk_load',
                 'chapter_end',
                 'chapter_start',
-                 'chapter_title_add_pattern',
                 'chapter_title_strip_pattern',
+                 'chapter_title_def_pattern',
+                 'chapter_title_add_pattern',
+                 'chapter_title_new_pattern',
+                 'chapter_title_addnew_pattern',
+                 'title_chapter_range_pattern',
+                 'mark_new_chapters',
                 'check_next_chapter',
                 'skip_author_cover',
                 'collect_series',
@@ -195,8 +234,10 @@ def get_valid_keywords():
                 'datePublished_format',
                 'dateUpdated_format',
                 'default_cover_image',
+                 'description_limit',
                 'do_update_hook',
                 'exclude_notes',
+                 'exclude_editor_signature',
                 'extra_logpage_entries',
                 'extra_subject_tags',
                 'extra_titlepage_entries',
@@ -248,6 +289,7 @@ def get_valid_keywords():
                 'output_filename_safepattern',
                 'password',
                 'post_process_cmd',
+                 'rating_titles',
                 'remove_transparency',
                 'replace_br_with_p',
                 'replace_hr',
@@ -302,13 +344,20 @@ def make_generate_cover_settings(param):

 class Configuration(ConfigParser.SafeConfigParser):

-    def __init__(self, site, fileform):
+    def __init__(self, sections, fileform):
+        site = sections[-1] # first section is site DN.
        ConfigParser.SafeConfigParser.__init__(self)

        self.linenos=dict() # key by section or section,key -> lineno
        
-        self.sectionslist = ['defaults']
+        ## [injected] section has even less priority than [defaults]
+        self.sectionslist = ['defaults','injected']

+        ## add other sections (not including site DN) after defaults,
+        ## but before site-specific.
+        for section in sections[:-1]:
+            self.addConfigSection(section)
+        
        if site.startswith("www."):
            sitewith = site
            sitewithout = site.replace("www.","")
@@ -318,8 +367,13 @@ class Configuration(ConfigParser.SafeConfigParser):
        
        self.addConfigSection(sitewith)
        self.addConfigSection(sitewithout)
+        
        if fileform:
            self.addConfigSection(fileform)
+            ## add other sections:fileform (not including site DN)
+            ## after fileform, but before site-specific:fileform.
+            for section in sections[:-1]:
+                self.addConfigSection(section+":"+fileform)
            self.addConfigSection(sitewith+":"+fileform)
            self.addConfigSection(sitewithout+":"+fileform)
        self.addConfigSection("overrides")
@@ -407,6 +461,30 @@ class Configuration(ConfigParser.SafeConfigParser):
        else:
            return self.linenos.get(section,None)
    
+    ## Copied from Python 2.7 library so as to make read utf8.
+    def read(self, filenames):
+        """Read and parse a filename or a list of filenames.
+        Files that cannot be opened are silently ignored; this is
+        designed so that you can specify a list of potential
+        configuration file locations (e.g. current directory, user's
+        home directory, systemwide directory), and all existing
+        configuration files in the list will be read.  A single
+        filename may also be given.
+        Return list of successfully read files.
+        """
+        if isinstance(filenames, basestring):
+            filenames = [filenames]
+        read_ok = []
+        for filename in filenames:
+            try:
+                fp = codecs.open(filename,encoding='utf-8')
+            except IOError:
+                continue
+            self._read(fp, filename)
+            fp.close()
+            read_ok.append(filename)
+        return read_ok
+    
    ## Copied from Python 2.7 library so as to make it save linenos too.
    #
    # Regular expressions for parsing section headers and options.
@@ -609,3 +687,13 @@ class Configurable(object):

    def get_config_list(self, sections, key):
        return self.configuration.get_config_list(sections,key)
+
+    def get_label(self, entry):
+        if self.hasConfig(entry+"_label"):
+            label=self.getConfig(entry+"_label")
+        elif entry in titleLabels:
+            label=titleLabels[entry]
+        else:
+            label=entry.title()
+        return label
+
@@ -148,8 +148,10 @@ zip_filename: ${title}-${siteabbrev}_${storyId}${formatext}.zip
 ## zip_filename.
 allow_unsafe_filename: false

-## The regex pattern of 'unsafe' filename chars for above.
-#output_filename_safepattern:[^a-zA-Z0-9_\. \[\]\(\)&'-]+
+## The regex pattern of 'unsafe' filename chars for above.  First
+## character . OR any one or more characters that are NOT a letter,
+## number, or one of _. []()&'-
+output_filename_safepattern:(^\.|/\.|[^a-zA-Z0-9_\. \[\]\(\)&'-]+)

 ## entries to make epub subjects and calibre tags
 ## lastupdate creates two tags: "Last Update Year/Month: %Y/%m" and "Last Update: %Y/%m/%d"
@@ -268,6 +270,11 @@ keep_summary_html:true
 ## kept.  Some sites(AO3) use <a title="Translation">Foreign language</a>
 #keep_title_attr: false

+## If a chapter range was given, use this pattern for the book title.
+## replace_metadata and include/exclude will be applied *after* this.
+## Set to empty value to disable.
+title_chapter_range_pattern:${title} (Ch ${first}-${last})
+
 ## Don't like the numbers at the start of chapter titles on some
 ## sites?  You can use strip_chapter_numbers to strip them off.  Just
 ## want to make them all look the same?  Strip them off, then add them
@@ -300,12 +307,31 @@ chapter_title_strip_pattern:^[0-9]+[\.: -]+
 ## etc
 #chapter_title_strip_pattern:^([0-9]+[\.: -]+)?(Chapter *[0-9]+[\.:, -]*)?

-## Uses a python template substitution.  The ${index} is the 'chapter'
-## number and ${title} is the chapter title, after applying
-## chapter_title_strip_pattern.  Those are the only variables available.
-## "The Beginning" => "1. The Beginning" 
+## If true, when updating an epub that already has old chapters, new
+## chapters will be marked in the TOC and chapter header by using
+## chapter_title_new_pattern and chapter_title_addnew_pattern to set the chapter.
+mark_new_chapters:false
+
+## chapter title patterns use python template substitution.  The
+## ${index} is the 'chapter' number and ${title} is the chapter title,
+## after applying chapter_title_strip_pattern.  Those are the only
+## variables available.
+
+## The basic pattern used when not using add_chapter_numbers or
+## mark_new_chapters
+chapter_title_def_pattern:${title}
+
+## Pattern used with add_chapter_numbers, but not mark_new_chapters
 chapter_title_add_pattern:${index}. ${title}

+## Pattern used with mark_new_chapters, but not add_chapter_numbers
+## (new) is just text and can be changed.
+chapter_title_new_pattern:(new) ${title}
+
+## Pattern used with add_chapter_numbers and mark_new_chapters
+## (new) is just text and can be changed.
+chapter_title_addnew_pattern:${index}. (new) ${title}
+
 ## Reorder ships so b/a and c/b/a become a/b and a/b/c. Only separates
 ## on '/', so use replace_metadata to change separator first if
 ## needed. Something like: ships=>[ ]*(/|&amp;|&)[ ]*=>/ You can use
@@ -327,6 +353,77 @@ sort_ships:false
 ## User-agent
 user_agent:FFF/2.X

+## Added for [base_xenforoforum], but can be used with other sites,
+## too.  Limit the 'description' to the first X *characters*
+## collected.  Character count includes HTML tags, so it can be
+## non-intuitive.
+#description_limit:1000
+
+[base_efiction]
+## At the time of writing, eFiction Base adapters allow downloading
+## the whole story in bulk using the 'Print' feature. If 'bulk_load'
+## is set to 'true', both metadata and chapters can be loaded in one
+## step
+bulk_load:true
+
+[base_xenforoforum]
+## Currently only forums.spacebattles.com and forums.sufficientvelocity.com
+
+cover_exclusion_regexp:/clear.png
+
+## I saw lots of chapters name simply '1.1' etc during testing.
+strip_chapter_numbers:false
+
+## Copy title to tagsfromtitle for parsing tags.
+add_to_extra_valid_entries:,tagsfromtitle,forumtags
+
+## '.NOREPL' tells the system to *not* apply title's
+## in/exclude/replace_metadata -- Only works on include_in_ lines.
+include_in_tagsfromtitle:title.NOREPL
+
+tagsfromtitle_label:Tags from Title
+forumtags_label:Tags from Forum
+
+## might want to do this, maybe not.  Will often include category, but
+## also often include non-category stuff.
+# include_in_category:tagsfromtitle 
+
+add_to_include_metadata_pre:
+# only keep tagsfromtitle with ( or [ in.
+ tagsfromtitle=~[\[\(]
+ 
+add_to_replace_metadata:
+# remove anything outside () or []
+ tagsfromtitle=>^.*?([\(\[]([^\]\)]+)[\)\]]).*?$=>\1
+# remove () []
+ tagsfromtitle=>[\(\)\[\]]=>
+# change (spaces)slash(or semicolon)(spaces) to comma
+ tagsfromtitle=> *[/;] *=>,
+ tagsfromtitle=> [xX] =>,
+ 
+# remove [] or () blocks and leading/trailing spaces/dashes/colons
+ title=>[-: ]*[\(\[]([^\]\)]+)[\)\]][-: ]*=>
+# remove 'Thread' and the next word, usually "Thread 2", "Thread
+# four", "Thread iv", etc
+ title,tagsfromtitle=>[-: ]*[Tt]hread [^ ]+[-: ]*=>
+
+add_to_extra_titlepage_entries:,tagsfromtitle,forumtags
+
+## '.SPLIT' tells the system to split by ','
+add_to_include_subject_tags:,tagsfromtitle.SPLIT
+## Or to also include forumtags add this to personal.ini:
+#add_to_include_subject_tags:,forumtags
+
+## base_xenforoforum reads Published and Updated datetimes from
+## Threadmarks if used, or from the posted & updated times of the
+## 'first' post if no threadmarks.
+datePublished_format:%%Y-%%m-%%d %%H:%%M:%%S
+dateUpdated_format:%%Y-%%m-%%d %%H:%%M:%%S
+
+## Only take the first X characters of the 'first' post to use as
+## the description.
+description_limit:500
+
 ## Each output format has a section that overrides [defaults]
 [html]

@@ -582,6 +679,8 @@ extratags: FanFiction,Testing,HTML
 ## doesn't like that.  If do_update_hook is uncommented and set true,
 ## the adapter will discard all existing chapters from the newest one
 ## on when updating to enforce accurate chapters.
+## Starting July 2015, FFF stores chapter URLs in the chapter files.
+## Stories downloaded after that shouldn't need this setting anymore.
 #do_update_hook:false

 ## AO3 adapter defines a few extra metadata entries.
@@ -597,6 +696,18 @@ hits_label:Hits
 collections_label:Collections
 bookmarks_label:Bookmarks

+## AO3 doesn't have anything it calls 'genre'.  The adapter used to be
+## hardcoded to include the site specific metadata freeformtags &
+## ao3categories in the standard metadata field genre.  By making it
+## configurable, users can change it.
+include_in_genre: freeformtags, ao3categories
+
+## AO3 uses the word 'category' differently than most sites.  The
+## adapter used to be hardcoded to include the site specific metadata
+## fandom in the standard metadata field category.  By making it
+## configurable, users can change it.
+include_in_category:fandoms
+
 ## freeformtags was previously typo'ed as freefromtags.  This way,
 ## freefromtags will still work for people who've used it.
 include_in_freefromtags:freeformtags
@@ -608,7 +719,7 @@ include_in_freefromtags:freeformtags
 #extra_subject_tags:fandoms,freeformtags,ao3categories

 ## AO3 chapters can include several different types of notes.  We've
-## traditional included them all in the chapter text, but this allows
+## traditionally included them all in the chapter text, but this allows
 ## you to customize which you include.  Copy this parameter to your
 ## personal.ini and list the ones you don't want.
 #exclude_notes:authorheadnotes,chaptersummary,chapterheadnotes,chapterfootnotes,authorfootnotes
@@ -758,11 +869,6 @@ extraships:Spike/Buffy
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true

-## Virtually all eFiction-based sites allow downloading the whole story in
-## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
-## metadata and chapters can be loaded in one step
-bulk_load:true
-
 [dramione.org]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -925,11 +1031,6 @@ extraships:Harry Potter/Hermione Granger
 #username:YourName
 #password:yourpassword

-## Virtually all eFiction-based sites allow downloading the whole story in
-## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
-## metadata and chapters can be loaded in one step
-bulk_load:true
-
 extra_valid_entries: readings,romance
 extra_titlepage_entries: readings,romance
 readings_label: Readings
@@ -1019,6 +1120,12 @@ extra_valid_entries:size
 # don't show twitter icon.
 cover_exclusion_regexp:/res/css/bir.png

+[forums.spacebattles.com]
+## see [base_xenforoforum]
+
+[forums.sufficientvelocity.com]
+## see [base_xenforoforum]
+
 [grangerenchanted.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -1065,6 +1172,18 @@ extracategories:In Death
 ## cover image.  This lets you exclude them.
 cover_exclusion_regexp:/public/style_emoticons/.*

+[it-could-happen.net]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+## RPF == http://en.wikipedia.org/wiki/Real_person_fiction
+## Site dedicated to these categories/characters/ships
+extracategories:Glee RPF
+extracharacters:Darren Criss, Chris Colfer
+extraships:Darren Criss/Chris Colfer
+
 [ksarchive.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:Star Trek
@@ -1084,11 +1203,6 @@ eroticatags_label:Erotica Tags
 extra_titlepage_entries: eroticatags

 [lotrfanfiction.com]
-## Virtually all eFiction-based sites allow downloading the whole story in
-## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
-## metadata and chapters can be loaded in one step
-bulk_load:true
-
 extra_valid_entries: readings
 readings_label: Readings

@@ -1142,6 +1256,12 @@ extracategories:NCIS
 extracategories:Buffy: The Vampire Slayer
 extracharacters:Willow

+[ninelives.dark-solace.org]
+## Site dedicated to these categories/characters/ships
+extracategories:The Walking Dead
+extracharacters:Carol,Daryl
+extraships:Carol/Daryl
+
 [nocturnal-light.net]
 ## Extra metadata that this adapter knows about.  See [dramione.org]
 ## for examples of how to use them.
@@ -1338,11 +1458,6 @@ extracategories:Transgender
 ## confirm they are adult for adult content.
 #is_adult:true

-## Virtually all eFiction-based sites allow downloading the whole story in
-## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
-## metadata and chapters can be loaded in one step
-bulk_load:true
-
 [thehexfiles.net]
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter
@@ -1362,11 +1477,6 @@ extraships:Harry Potter/Draco Malfoy
 ## personal.ini, not defaults.ini.
 #is_adult:true

-## Virtually all eFiction-based sites allow downloading the whole story in
-## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
-## metadata and chapters can be loaded in one step
-bulk_load:true
-
 ## Site dedicated to these categories/characters/ships
 extracategories:Criminal Minds

@@ -1376,11 +1486,6 @@ extracategories:Criminal Minds
 ## personal.ini, not defaults.ini.
 #is_adult:true

-## Virtually all eFiction-based sites allow downloading the whole story in
-## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
-## metadata and chapters can be loaded in one step
-bulk_load:true
-
 extra_valid_entries: readings,challenge
 extra_titlepage_entries: readings,challenge
 challenge_label: Challenge
@@ -1449,6 +1554,8 @@ extra_titlepage_entries:readings,awards
 awards_label:Awards
 readings_label:Readings

+## some sites include images that we don't ever want becoming the
+## cover image.  This lets you exclude them.
 cover_exclusion_regexp:art/.*Awards.jpg

 [voracity2.e-fic.com]
@@ -1639,6 +1746,10 @@ dateUpdated_format:%%Y-%%m-%%d %%H:%%M:%%S
 ## to on, but can be switched off if it is found to cause problems.
 fix_fimf_blockquotes:true

+## some sites include images that we don't ever want becoming the
+## cover image.  This lets you exclude them.
+cover_exclusion_regexp:/images/emoticons/
+
 ## Site dedicated to these categories/characters/ships
 extracategories:My Little Pony: Friendship is Magic

@@ -1722,12 +1833,38 @@ extraships:InuYasha/Kagome
 ## Site dedicated to these categories/characters/ships
 extracategories:Lord of the Rings

-## Virtually all eFiction-based sites allow downloading the whole story in
-## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
-## metadata and chapters can be loaded in one step
-bulk_load:true
+[www.masseffect2.in]
+## Site dedicated to this fandom.
+extracategories: Mass Effect
+
+## Ucoz.com engine, upon which MassEffect2.in is based, imposes an unspecified limit on request frequency.
+## Reports vary from `5 requests per second' to `2 requests per second for more than 10 per minute'.
+## With default settings, a several-hours IP ban may follow, so set it higher.
+slow_down_sleep_time: 2
+
+## Whether to exclude editor signature from the bottom of chapter text.
+exclude_editor_signature: false
+
+## Stories on the site almost never have cover image, and for the stories which do,
+## this may be adjusted in `personal.ini' before downloading.
+never_make_cover: true
+
+## Titles for ratings identified by 1- or 2-letter codes from `ERATING system'
+## (`система Р.Е.Й.Т.И.Н.Г.').  MassEffect2.in and some other sites adopted it,
+## but changed titles and update them occasionally.
+rating_titles: R=RESTRICTED (16+), E=EXEMPT (18+), I=ART HOUSE, T=To every, A=IN=Иной мир, Nn=Новый мир, G=О\, Господи!
+adult_ratings: E,R

 [www.mediaminer.org]
+dateUpdated_format:%%Y-%%m-%%d %%H:%%M:%%S
+## Note that mediaminer doesn't give datePublished on the story's
+## index page--it's collected from the earliest uploaded chapter.  So
+## it's not available when only fetching metadata.
+datePublished_format:%%Y-%%m-%%d %%H:%%M:%%S
+
+## some sites include images that we don't ever want becoming the
+## cover image.  This lets you exclude them.
+cover_exclusion_regexp:/img/rss.png

 [www.midnightwhispers.ca]
 ## Some sites do not require a login, but do require the user to
@@ -1795,6 +1932,13 @@ extracategories:Psych
 ## Site dedicated to these categories/characters/ships
 extracategories:Queer as Folk

+[quotev.com]
+extra_valid_entries:pages,readers,reads,favorites
+pages_label:Pages
+readers_label:Readers
+reads_label:Reads
+favorites_label:Favorites
+
 [www.restrictedsection.org]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -1,18 +1,17 @@
-#!/usr/bin/env python
-# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
-from __future__ import (unicode_literals, division, absolute_import,
-                        print_function)
+# -*- coding: utf-8 -*-

 __license__   = 'GPL v3'
-__copyright__ = '2014, Jim Miller'
+__copyright__ = '2015, Jim Miller'
 __docformat__ = 'restructuredtext en'

 import logging
 logger = logging.getLogger(__name__)

 import re, os, traceback
-from zipfile import ZipFile
+from collections import defaultdict
+from zipfile import ZipFile, ZIP_STORED, ZIP_DEFLATED
 from xml.dom.minidom import parseString
+from StringIO import StringIO

 import bs4 as bs

@@ -42,7 +41,7 @@ def get_update_data(inputio,

    ## Save the path to the .opf file--hrefs inside it are relative to it.
    relpath = get_path_part(rootfilename)
-            
+
    oldcover = None
    calibrebookmark = None
    logfile = None
@@ -90,7 +89,9 @@ def get_update_data(inputio,

    filecount = 0
    soups = [] # list of xhmtl blocks
+    urlsoups = {} # map of xhtml blocks by url
    images = {} # dict() longdesc->data
+    datamaps = defaultdict(dict) # map of data maps by url
    if getfilecount:
        # spin through the manifest--only place there are item tags.
        for item in contentdom.getElementsByTagName("item"):
@@ -124,31 +125,57 @@ def get_update_data(inputio,
                                logger.warn("Image %s not found!\n(originally:%s)"%(newsrc,longdesc))
                                logger.warn("Exception: %s"%(unicode(e)))
                                traceback.print_exc()
-                        soup = soup.find('body')
+                        bodysoup = soup.find('body')
                        # ffdl epubs have chapter title h3
-                        h3 = soup.find('h3')
+                        h3 = bodysoup.find('h3')
                        if h3:
                            h3.extract()
                        # TtH epubs have chapter title h2
-                        h2 = soup.find('h2')
+                        h2 = bodysoup.find('h2')
                        if h2:
                            h2.extract()

-                        for skip in soup.findAll(attrs={'class':'skip_on_ffdl_update'}):
+                        for skip in bodysoup.findAll(attrs={'class':'skip_on_ffdl_update'}):
                            skip.extract()
-                            
-                        soups.append(soup)
-                        
+
+                        ## <meta name="chapterurl" content="${url}"></meta>
+                        #print("look for meta chapurl")
+                        currenturl = None
+                        chapurl = soup.find('meta',{'name':'chapterurl'})
+                        if chapurl:
+                            if chapurl['content'] not in urlsoups: # keep first found if more than one.
+                            #print("Found chapurl['content']:%s"%chapurl['content'])
+                                currenturl = chapurl['content']
+                                urlsoups[chapurl['content']] = bodysoup
+                        else:
+                            # for older pre-meta.  Only temp.
+                            chapa = bodysoup.find('a',{'class':'chapterurl'})
+                            if chapa and chapa['href'] not in urlsoups: # keep first found if more than one.
+                                urlsoups[chapa['href']] = bodysoup
+                                currenturl = chapa['href']
+                                chapa.extract()
+
+                        chapterorigtitle = soup.find('meta',{'name':'chapterorigtitle'})
+                        if chapterorigtitle:
+                            datamaps[currenturl]['chapterorigtitle'] = chapterorigtitle['content']
+
+                        chaptertitle = soup.find('meta',{'name':'chaptertitle'})
+                        if chaptertitle:
+                            datamaps[currenturl]['chaptertitle'] = chaptertitle['content']
+
+                        soups.append(bodysoup)
+
                    filecount+=1

    try:
        calibrebookmark = epub.read("META-INF/calibre_bookmarks.txt")
    except:
        pass
-                    
+
    #for k in images.keys():
        #print("\tlongdesc:%s\n\tData len:%s\n"%(k,len(images[k])))
-    return (source,filecount,soups,images,oldcover,calibrebookmark,logfile)
+    # print("datamaps:%s"%datamaps)
+    return (source,filecount,soups,images,oldcover,calibrebookmark,logfile,urlsoups,datamaps)

 def get_path_part(n):
    relpath = os.path.dirname(n)
@@ -172,7 +199,7 @@ def get_story_url_from_html(inputio,_is_good_url=None):

    ## Save the path to the .opf file--hrefs inside it are relative to it.
    relpath = get_path_part(rootfilename)
-            
+
    # spin through the manifest--only place there are item tags.
    for item in contentdom.getElementsByTagName("item"):
        # First, count the 'chapter' files.  FFF uses file0000.xhtml,
@@ -192,3 +219,129 @@ def get_story_url_from_html(inputio,_is_good_url=None):
                if _is_good_url == None or _is_good_url(ahref):
                    return ahref
    return None
+
+def reset_orig_chapters_epub(inputio,outfile):
+    inputepub = ZipFile(inputio, 'r') # works equally well with a path or a blob
+
+    ## build zip in memory in case updating in place(CLI).
+    zipio = StringIO()
+
+    ## Write mimetype file, must be first and uncompressed.
+    ## Older versions of python(2.4/5) don't allow you to specify
+    ## compression by individual file.
+    ## Overwrite if existing output file.
+    outputepub = ZipFile(zipio, 'w', compression=ZIP_STORED)
+    outputepub.debug = 3
+    outputepub.writestr("mimetype", "application/epub+zip")
+    outputepub.close()
+
+    ## Re-open file for content.
+    outputepub = ZipFile(zipio, "a", compression=ZIP_DEFLATED)
+    outputepub.debug = 3
+
+    changed = False
+
+    unmerge_tocncxdoms = {}
+    ## spin through file contents, saving any unmerge toc.ncx files.
+    for zf in inputepub.namelist():
+        ## logger.debug("zf:%s"%zf)
+        if zf.endswith('/toc.ncx'):
+            ## logger.debug("toc.ncx zf:%s"%zf)
+            unmerge_tocncxdoms[zf] = parseString(inputepub.read(zf))
+
+    tocncxdom = parseString(inputepub.read('toc.ncx'))
+    ## spin through file contents.
+    for zf in inputepub.namelist():
+        if zf not in ['mimetype','toc.ncx'] and not zf.endswith('/toc.ncx'):
+            entrychanged = False
+            data = inputepub.read(zf)
+            # if isinstance(data,unicode):
+            #     logger.debug("\n\n\ndata is unicode\n\n\n")
+            if re.match(r'.*/file\d+\.xhtml',zf):
+                #logger.debug("zf:%s"%zf)
+                data = data.decode('utf-8')
+                soup = bs.BeautifulSoup(data,"html5lib")
+
+                chapterorigtitle = None
+                tag = soup.find('meta',{'name':'chapterorigtitle'})
+                if tag:
+                    chapterorigtitle = tag['content']
+
+                # toctitle is separate for add_chapter_numbers:toconly users.
+                chaptertoctitle = None
+                tag = soup.find('meta',{'name':'chaptertoctitle'})
+                if tag:
+                    chaptertoctitle = tag['content']
+                elif chapterorigtitle:
+                    chaptertoctitle = chapterorigtitle
+
+                chaptertitle = None
+                tag = soup.find('meta',{'name':'chaptertitle'})
+                if tag:
+                    chaptertitle = tag['content']
+
+                if chaptertitle and chapterorigtitle and chapterorigtitle != chaptertitle:
+                    origdata = data
+                    # print("\n%s\n%s\n"%(chapterorigtitle,chaptertitle))
+                    data = data.replace(u'<meta name="chaptertitle" content="'+chaptertitle+u'"></meta>',
+                                        u'<meta name="chaptertitle" content="'+chapterorigtitle+u'"></meta>')
+                    data = data.replace(u'<title>'+chaptertitle+u'</title>',u'<title>'+chapterorigtitle+u'</title>')
+                    data = data.replace(u'<h3>'+chaptertitle+u'</h3>',u'<h3>'+chapterorigtitle+u'</h3>')
+
+                    entrychanged = ( origdata != data )
+                    changed = changed or entrychanged
+
+                    if entrychanged:
+                        _replace_tocncx(tocncxdom,zf,chaptertoctitle)
+                        ## Also look for and update individual
+                        ## book toc.ncx files for anthology in case
+                        ## it's unmerged.
+                        zf_toc = zf[:zf.rfind('/OEBPS/')]+'/toc.ncx'
+                        mergedprefix_len = len(zf[:zf.rfind('/OEBPS/')])+1
+
+                        if zf_toc in unmerge_tocncxdoms:
+                            _replace_tocncx(unmerge_tocncxdoms[zf_toc],zf[mergedprefix_len:],chaptertoctitle)
+
+                outputepub.writestr(zf,data.encode('utf-8'))
+            else:
+                # possibly binary data, thus no .encode().
+                outputepub.writestr(zf,data)
+
+    for tocnm, tocdom in unmerge_tocncxdoms.items():
+        outputepub.writestr(tocnm,tocdom.toxml(encoding='utf-8'))
+
+    outputepub.writestr('toc.ncx',tocncxdom.toxml(encoding='utf-8'))
+    outputepub.close()
+    # declares all the files created by Windows.  otherwise, when
+    # it runs in appengine, windows unzips the files as 000 perms.
+    for zf in outputepub.filelist:
+        zf.create_system = 0
+
+    # only *actually* write if changed.
+    if changed:
+        if isinstance(outfile,basestring):
+            with open(outfile,"wb") as outputio:
+                outputio.write(zipio.getvalue())
+        else:
+            outfile.write(zipio.getvalue())
+
+    inputepub.close()
+    zipio.close()
+
+    return changed
+
+
+def _replace_tocncx(tocncxdom,zf,chaptertoctitle):
+    ## go after the TOC entry, too.
+    # <navPoint id="file0005" playOrder="6">
+    #   <navLabel>
+    #     <text>5. (new) Chapter 4</text>
+    #   </navLabel>
+    #   <content src="OEBPS/file0005.xhtml"/>
+    # </navPoint>
+    for contenttag in tocncxdom.getElementsByTagName("content"):
+        if contenttag.getAttribute('src') == zf:
+            texttag = contenttag.parentNode.getElementsByTagName('navLabel')[0].getElementsByTagName('text')[0]
+            texttag.childNodes[0].replaceWholeText(chaptertoctitle)
+            #logger.debug("text label:%s"%texttag.toxml())
+            continue
@@ -24,6 +24,13 @@ class FailedToDownload(Exception):
    def __str__(self):
        return self.error

+class RejectImage(Exception):
+    def __init__(self,error):
+        self.error=error
+
+    def __str__(self):
+        return self.error
+
 class InvalidStoryURL(Exception):
    def __init__(self,url,domain,example):
        self.url=url
@@ -15,12 +15,15 @@
 # limitations under the License.
 #

-import re
-import urlparse
-import urllib2 as u2
-
-import imaplib
+import collections
 import email
+import imaplib
+import re
+import urllib2 as u2
+import urlparse
+
+import logging
+logger = logging.getLogger(__name__)

 from BeautifulSoup import BeautifulSoup 
 from gziphttp import GZipProcessor
@@ -74,28 +77,25 @@ def get_urls_from_page(url,configuration=None,normalize=False):
    return get_urls_from_html(data,url,configuration,normalize,restrictsearch)

 def get_urls_from_html(data,url=None,configuration=None,normalize=False,restrictsearch=None):
+    urls = collections.OrderedDict()

-    normalized = [] # normalized url
-    retlist = [] # orig urls.
-    
    if not configuration:
        configuration = Configuration("test1.com","EPUB")

    soup = BeautifulSoup(data)
    if restrictsearch:
        soup = soup.find(*restrictsearch)
-        #print("restrict search:%s"%soup)
+        #logger.debug("restrict search:%s"%soup)
    
    for a in soup.findAll('a'):
        if a.has_key('href'):
-            #print("a['href']:%s"%a['href'])
+            #logger.debug("a['href']:%s"%a['href'])
            href = form_url(url,a['href'])
-            #print("1 urlhref:%s"%href)
+            #logger.debug("1 urlhref:%s"%href)
            # this (should) catch normal story links, some javascript
            # 'are you old enough' links, and 'Report This' links.
-            # The 'normalized' set prevents duplicates.
            if 'story.php' in a['href']:
-                #print("trying:%s"%a['href'])
+                #logger.debug("trying:%s"%a['href'])
                m = re.search(r"(?P<sid>(view)?story\.php\?(sid|psid|no|story|stid)=\d+)",a['href'])
                if m != None:
                    href = form_url(a['href'] if '//' in a['href'] else url,
@@ -103,34 +103,32 @@ def get_urls_from_html(data,url=None,configuration=None,normalize=False,restrict
                    
            try:
                href = href.replace('&index=1','')
-                #print("2 urlhref:%s"%href)
+                #logger.debug("2 urlhref:%s"%href)
                adapter = adapters.getAdapter(configuration,href)
-                #print("found adapter")
-                if adapter.story.getMetadata('storyUrl') not in normalized:
-                    normalized.append(adapter.story.getMetadata('storyUrl'))
-                    retlist.append(href)
+                #logger.debug("found adapter")
+                if adapter.story.getMetadata('storyUrl') not in urls:
+                    urls[adapter.story.getMetadata('storyUrl')] = [href]
+                else:
+                    urls[adapter.story.getMetadata('storyUrl')].append(href)
            except Exception, e:
-                #print e
+                #logger.debug e
                pass

-    if normalize:
-        return normalized
-    else:
-        return retlist
+    # Simply return the longest URL with the assumption that it contains the
+    # most user readable metadata, if not normalized
+    return urls.keys() if normalize else [max(value, key=len) for key, value in urls.items()]
+

 def get_urls_from_text(data,configuration=None,normalize=False):
-
-    normalized = [] # normalized url
-    retlist = [] # orig urls.
+    urls = collections.OrderedDict()
    data=unicode(data)
-    
+
    if not configuration:
        configuration = Configuration("test1.com","EPUB")
    
    for href in re.findall('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+', data):
        # this (should) catch normal story links, some javascript
        # 'are you old enough' links, and 'Report This' links.
-        # The 'normalized' set prevents duplicates.
        if 'story.php' in href:
            m = re.search(r"(?P<sid>(view)?story\.php\?(sid|psid|no|story|stid)=\d+)",href)
            if m != None:
@@ -138,16 +136,17 @@ def get_urls_from_text(data,configuration=None,normalize=False):
        try:
            href = href.replace('&index=1','')
            adapter = adapters.getAdapter(configuration,href)
-            if adapter.story.getMetadata('storyUrl') not in normalized:
-                normalized.append(adapter.story.getMetadata('storyUrl'))
-                retlist.append(href)
+            if adapter.story.getMetadata('storyUrl') not in urls:
+                urls[adapter.story.getMetadata('storyUrl')] = [href]
+            else:
+                urls[adapter.story.getMetadata('storyUrl')].append(href)
        except:
            pass

-    if normalize:
-        return normalized
-    else:
-        return retlist
+    # Simply return the longest URL with the assumption that it contains the
+    # most user readable metadata, if not normalized
+    return urls.keys() if normalize else [max(value, key=len) for key, value in urls.items()]
+

 def form_url(parenturl,url):
     url = url.strip() # ran across an image with a space in the
@@ -177,7 +176,8 @@ def form_url(parenturl,url):
     return returl
       
 def get_urls_from_imap(srv,user,passwd,folder,markread=True):
-    
+
+    logger.debug("get_urls_from_imap srv:(%s)"%srv)
    mail = imaplib.IMAP4_SSL(srv)
    mail.login(user, passwd)
    mail.list()
@@ -186,8 +186,8 @@ def get_urls_from_imap(srv,user,passwd,folder,markread=True):
    
    result, data = mail.uid('search', None, "UNSEEN")
    
-    #print("result:%s"%result)
-    #print("data:%s"%data)
+    #logger.debug("result:%s"%result)
+    #logger.debug("data:%s"%data)
    urls=set()
    
    #latest_email_uid = data[0].split()[-1]
@@ -195,8 +195,8 @@ def get_urls_from_imap(srv,user,passwd,folder,markread=True):

        result, data = mail.uid('fetch', email_uid, '(BODY.PEEK[])') #RFC822
    
-        #print("result:%s"%result)
-        #print("data:%s"%data)
+        #logger.debug("result:%s"%result)
+        #logger.debug("data:%s"%data)
    
        raw_email = data[0][1]
        
@@ -205,28 +205,28 @@ def get_urls_from_imap(srv,user,passwd,folder,markread=True):
    
        email_message = email.message_from_string(raw_email)
     
-        #print "To:%s"%email_message['To']
-        #print "From:%s"%email_message['From']
-        #print "Subject:%s"%email_message['Subject']
+        #logger.debug "To:%s"%email_message['To']
+        #logger.debug "From:%s"%email_message['From']
+        #logger.debug "Subject:%s"%email_message['Subject']
    
-    #    print("payload:%s"%email_message.get_payload())
+    #    logger.debug("payload:%s"%email_message.get_payload())

        urllist=[]
        for part in email_message.walk():
            try:
-            #print("part mime:%s"%part.get_content_type())
+            #logger.debug("part mime:%s"%part.get_content_type())
                if part.get_content_type() == 'text/plain':
                    urllist.extend(get_urls_from_text(part.get_payload(decode=True)))
                if part.get_content_type() == 'text/html':
                    urllist.extend(get_urls_from_html(part.get_payload(decode=True)))
            except Exception as e:
-                print("Failed to read email content: %s"%e)
-        #print "urls:%s"%get_urls_from_text(get_first_text_block(email_message))
+                logger.error("Failed to read email content: %s"%e)
+        #logger.debug "urls:%s"%get_urls_from_text(get_first_text_block(email_message))

        if urllist and markread:
            #obj.store(data[0].replace(' ',','),'+FLAGS','\Seen')
            r,d = mail.uid('store',email_uid,'+FLAGS','(\\SEEN)')
-            #print("seen result:%s->%s"%(email_uid,r))
+            #logger.debug("seen result:%s->%s"%(email_uid,r))
                
        [ urls.add(x) for x in urllist ]
    
@@ -81,10 +81,10 @@ def removeEntities(text):
    
    try:
        t = text.decode('utf-8')
-    except UnicodeEncodeError, e:
+    except (UnicodeEncodeError,UnicodeDecodeError), e:
        try:
            t = text.encode ('ascii', 'xmlcharrefreplace') 
-        except UnicodeEncodeError, e:
+        except (UnicodeEncodeError,UnicodeDecodeError), e:
            t = text
    text = t 
    # replace numeric versions of [&<>] with named versions,
@@ -16,18 +16,25 @@
 #

 import os, re
+from collections import namedtuple
 import urlparse
 import string
+import json
+import datetime
 from math import floor
 from functools import partial
 import logging
 logger = logging.getLogger(__name__)
 import urlparse as up

+import bs4
+
 import exceptions
 from htmlcleanup import conditionalRemoveEntities, removeAllEntities
 from configurable import Configurable, re_compile

+Chapter = namedtuple('Chapter', 'url title html origtitle toctitle new')
+
 SPACE_REPLACE=u'\s'
 SPLIT_META=u'\,'

@@ -44,6 +51,8 @@ imagetypes = {

 try:
    from calibre.utils.magick import Image
+    from StringIO import StringIO
+    from gif import GifInfo, CHECK_IS_ANIMATED
    convtype = {'jpg':'JPG', 'png':'PNG'}

    def convert_image(url,data,sizes,grayscale,
@@ -55,6 +64,10 @@ try:
        owidth, oheight = img.size
        nwidth, nheight = sizes
        scaled, nwidth, nheight = fit_image(owidth, oheight, nwidth, nheight)
+
+        if normalize_format_name(img.format)=="gif" and GifInfo(StringIO(data),CHECK_IS_ANIMATED).frameCount > 1:
+            raise exceptions.RejectImage("Animated gifs come out purely--not going to use it.")
+
        if scaled:
            img.size = (nwidth, nheight)
            export = True
@@ -224,7 +237,7 @@ langs = {
    "Devanagari":"hi",

    ## These are from/for AO3:
-    
+
    u'العربية':'ar',
    u'беларуская':'be',
    u'Български език':'bg',
@@ -325,7 +338,7 @@ class InExMatch:
        else:
            retval = self.match == value
            #print(">>>>>>>>>>>>>%s==%s r: %s,%s=%s"%(self.match,value,self.negate,retval, self.negate != retval))
-            
+
        return self.negate != retval

    def __str__(self):
@@ -338,7 +351,7 @@ class InExMatch:
        else:
            s='='
        return u'InExMatch(%s %s%s %s)'%(self.keys,f,s,self.match)
-    
+
 ## metakey[,metakey]=~pattern
 ## metakey[,metakey]==string
 ## *for* part lines.  Effect only when trailing conditional key=~regexp matches
@@ -358,7 +371,7 @@ def set_in_ex_clude(setting):
            match = InExMatch(line)
            dest.append([match,condmatch])
    return dest
-              
+
 ## Two or three part lines.  Two part effect everything.
 ## Three part effect only those key(s) lists.
 ## pattern=>replacement
@@ -402,7 +415,9 @@ class Story(Configurable):
        except:
            self.metadata = {'version':'4.4'}
        self.in_ex_cludes = {}
-        self.chapters = [] # chapters will be tuples of (title,html)
+        self.chapters = [] # chapters will be namedtuple of Chapter(url,title,html,etc)
+        self.chapter_first = None
+        self.chapter_last = None
        self.imgurls = []
        self.imgtuples = []

@@ -431,9 +446,13 @@ class Story(Configurable):
                iel = []
                self.in_ex_cludes[ie] = set_in_ex_clude(ies)

-    def join_list(self, key, vallist):
-        return self.getConfig("join_string_"+key,u", ").replace(SPACE_REPLACE,' ').join(map(unicode, vallist))
+    def set_chapters_range(self,first=None,last=None):
+        self.chapter_first=first
+        self.chapter_last=last
                
+    def join_list(self, key, vallist):
+        return self.getConfig("join_string_"+key,u", ").replace(SPACE_REPLACE,' ').join(map(unicode, [ x for x in vallist if x is not None ]))
+
    def setMetadata(self, key, value, condremoveentities=True):

        # keep as list type, but set as only value.
@@ -445,20 +464,20 @@ class Story(Configurable):
                self.metadata[key]=conditionalRemoveEntities(value)
            else:
                self.metadata[key]=value
-                
+
        if key == "language":
            try:
                # getMetadata not just self.metadata[] to do replace_metadata.
                self.setMetadata('langcode',langs[self.getMetadata(key)])
            except:
                self.setMetadata('langcode','en')
-                
+
        if key == 'dateUpdated' and value:
            # Last Update tags for Bill.
-            self.addToList('lastupdate',value.strftime("Last Update Year/Month: %Y/%m"))
+            self.addToList('lastupdate',value.strftime("Last Update Year/Month: %Y/%m"),clear=True)
            self.addToList('lastupdate',value.strftime("Last Update: %Y/%m/%d"))

-        
+
    def do_in_ex_clude(self,which,value,key):
        if value and which in self.in_ex_cludes:
            include = 'include' in which
@@ -487,7 +506,7 @@ class Story(Configurable):
            if include and keyfound and not found:
                value = None
        return value
-        
+

    def doReplacements(self,value,key,return_list=False,seen_list=[]):
        value = self.do_in_ex_clude('include_metadata_pre',value,key)
@@ -526,7 +545,7 @@ class Story(Configurable):
                        # print("replacement,value:%s,%s->%s"%(replacement,value,regexp.sub(replacement,value)))
                        value = regexp.sub(replacement,value)
                        retlist = [value]
-                    
+
        for val in retlist:
            retlist = map(partial(self.do_in_ex_clude,'include_metadata_post',key=key),retlist)
            retlist = map(partial(self.do_in_ex_clude,'exclude_metadata_post',key=key),retlist)
@@ -538,6 +557,69 @@ class Story(Configurable):
        else:
            return self.join_list(key,retlist)

+    # for saving an html-ified copy of metadata.
+    def dump_html_metadata(self):
+        lines=[]
+        for k,v in sorted(self.metadata.iteritems()):
+            classes=['metadata']
+            if isinstance(v, (datetime.date, datetime.datetime, datetime.time)):
+                classes.append("datetime")
+                val = v.isoformat()
+            elif isinstance(v,list):
+                classes.append("list")
+                if '' in v:
+                    v.remove('')
+                if None in v:
+                    v.remove(None)
+                #logger.debug("k:%s v:%s"%(k,v))
+                # force ints/floats to strings.
+                val = "<ul>\n<li>%s</li>\n</ul>" % "</li>\n<li>".join([ "%s"%x for x in v ])
+            elif isinstance(v, (int)):
+                classes.append("int")
+                val = v
+            else:
+                val = v
+
+            # don't include items passed in for calibre cols, etc.
+            if not k.startswith('calibre_') and k not in ['output_css']:
+                lines.append("<p><span class='label'>%s</span>: <div class='%s' id='%s'>%s</div><p>\n"%(
+                        self.get_label(k),
+                        " ".join(classes),
+                        k,val))
+        return "\n".join(lines)
+
+    # for loading an html-ified copy of metadata.
+    def load_html_metadata(self,data):
+        soup = bs4.BeautifulSoup(data,'html5lib')
+        for tag in soup.find_all('div','metadata'):
+            val = None
+            if 'datetime' in tag['class']:
+                v = tag.string
+                try:
+                    val = datetime.datetime.strptime(v, '%Y-%m-%dT%H:%M:%S.%f')
+                except ValueError:
+                    try:
+                        val = datetime.datetime.strptime(v, '%Y-%m-%dT%H:%M:%S')
+                    except ValueError:
+                        try:
+                            val = datetime.datetime.strptime(v, '%Y-%m-%d')
+                        except ValueError:
+                            pass
+            elif 'list' in tag['class']:
+                val = []
+                for i in tag.find_all('li'):
+                    val.append(i.string)
+            elif 'int' in tag['class']:
+                val = int(tag.string)
+            else:
+                val = unicode("\n".join([ unicode(c) for c in tag.contents ]))
+                
+            #logger.debug("key(%s)=val(%s)"%(tag['id'],val))
+            if val:
+                self.metadata[tag['id']]=val
+                
+        # self.metadata = json.loads(s, object_hook=datetime_decoder)
+        
    def getMetadataRaw(self,key):
        if self.isValidMetaEntry(key) and self.metadata.has_key(key):
            return self.metadata[key]
@@ -568,6 +650,14 @@ class Story(Configurable):
                if key in ("datePublished","dateUpdated"):
                    value = value.strftime(self.getConfig(key+"_format","%Y-%m-%d"))

+                if key == "title" and (self.chapter_first or self.chapter_last) and self.getConfig("title_chapter_range_pattern"):
+                    first = self.chapter_first or "1"
+                    last = self.chapter_last or self.getMetadata("numChapters")
+                    templ = string.Template(self.getConfig("title_chapter_range_pattern"))
+                    value = templ.substitute({'title':value,
+                                              'first':commaGroups(first),
+                                              'last':commaGroups(last)})
+
            if doreplacements:
                value=self.doReplacements(value,key)
            if removeallentities and value != None:
@@ -610,7 +700,7 @@ class Story(Configurable):
                                                    self.getMetadata('author', removeallentities, doreplacements)))

        self.extendList("extratags",self.getConfigList("extratags"))
-            
+
        if self.getMetadataRaw('seriesUrl'):
            self.setMetadata('seriesHTML',linkhtml%('series',self.getMetadata('seriesUrl', removeallentities, doreplacements),
                                                    self.getMetadata('series', removeallentities, doreplacements)))
@@ -665,9 +755,6 @@ class Story(Configurable):
        if not value in self.metadata[listname]:
            self.metadata[listname].append(value)

-        if listname == 'category' and self.getConfig('add_genre_when_multi_category') and len(self.metadata[listname]) > 1:
-            self.addToList('genre',self.getConfig('add_genre_when_multi_category'))
-
    def isList(self,listname):
        'Everything set with an include_in_* is considered a list.'
        return self.isListType(listname) or \
@@ -687,8 +774,12 @@ class Story(Configurable):
        # includelist prevents infinite recursion of include_in_'s
        if self.hasConfig("include_in_"+listname) and listname not in includelist:
            for k in self.getConfigList("include_in_"+listname):
+                ldorepl = doreplacements
+                if k.endswith('.NOREPL'):
+                    k = k[:-len('.NOREPL')]
+                    ldorepl = False
                retlist.extend(self.getList(k,removeallentities=False,
-                                            doreplacements=doreplacements,includelist=includelist+[listname]))
+                                            doreplacements=ldorepl,includelist=includelist+[listname]))
        else:

            if not self.isList(listname):
@@ -696,6 +787,8 @@ class Story(Configurable):
                                            doreplacements=doreplacements)]
            else:
                retlist = self.getMetadataRaw(listname)
+                if retlist is None:
+                    retlist = []

        if retlist:
            if doreplacements:
@@ -703,12 +796,19 @@ class Story(Configurable):
                for val in retlist:
                    newretlist.extend(self.doReplacements(val,listname,return_list=True))
                retlist = newretlist
-                
+
            if removeallentities:
                retlist = map(removeAllEntities,retlist)
-                
+
            retlist = filter( lambda x : x!=None and x!='' ,retlist)

+        if listname == 'genre' and self.getConfig('add_genre_when_multi_category') and len(self.getList('category',
+                                                                                                        removeallentities=False,
+                                                                                                        # to avoid inf loops if genre/cat substs
+                                                                                                        doreplacements=False
+                                                                                                        )) > 1:
+            retlist.append(self.getConfig('add_genre_when_multi_category'))
+
        # reorder ships so b/a and c/b/a become a/b and a/b/c.  Only on '/',
        # use replace_metadata to change separator first if needed.
        # ships=>[ ]*(/|&amp;|&)[ ]*=>/
@@ -734,7 +834,16 @@ class Story(Configurable):

        # metadata all go into dc:subject tags, but only if they are configured.
        for (name,value) in self.getAllMetadata(removeallentities=removeallentities,keeplists=True).iteritems():
-            if name in tags_list:
+            if name+'.SPLIT' in tags_list:
+                flist=[]
+                if isinstance(value,list):
+                    for tag in value:
+                        flist.extend(tag.split(','))
+                else:
+                    flist.extend(value)
+                for tag in flist:
+                    subjectset.add(tag)
+            elif name in tags_list:
                if isinstance(value,list):
                    for tag in value:
                        subjectset.add(tag)
@@ -748,24 +857,63 @@ class Story(Configurable):

        return list(subjectset | set(self.getConfigList("extratags")))

-    def addChapter(self, url, title, html):
+    def addChapter(self, url, title, html, newchap=False):
+        # logger.debug("addChapter(%s,%s)"%(url,newchap))
        if self.getConfig('strip_chapter_numbers') and \
                self.getConfig('chapter_title_strip_pattern'):
            title = re.sub(self.getConfig('chapter_title_strip_pattern'),"",title)
-        self.chapters.append( (url,title,html) )
+        self.chapters.append( Chapter(url,title,html,title,title,newchap) )

    def getChapters(self,fortoc=False):
-        "Chapters will be tuples of (title,html)"
+        "Chapters will be Chapter namedtuples"
        retval = []
-        ## only add numbers if more than one chapter.
-        if len(self.chapters) > 1 and \
-                (self.getConfig('add_chapter_numbers') == "true" \
-                     or (self.getConfig('add_chapter_numbers') == "toconly" and fortoc)) \
-                     and self.getConfig('chapter_title_add_pattern'):
-            for index, (url,title,html) in enumerate(self.chapters):
-                retval.append( (url,
-                                string.Template(self.getConfig('chapter_title_add_pattern')).substitute({'index':index+1,'title':title}),
-                                html) )
+
+        ## only add numbers if more than one chapter.  Ditto (new) marks.
+        if len(self.chapters) > 1:
+            addnums = ( self.getConfig('add_chapter_numbers') == "true" 
+                        or (self.getConfig('add_chapter_numbers') == "toconly" and fortoc) )
+            
+            marknew = self.getConfig('mark_new_chapters')=='true'
+
+            defpattern = self.getConfig('chapter_title_def_pattern','${title}') # default val in case of missing defaults.ini
+            if addnums and marknew:
+                pattern = self.getConfig('chapter_title_add_pattern')
+                newpattern = self.getConfig('chapter_title_addnew_pattern')
+            elif addnums:
+                pattern = self.getConfig('chapter_title_add_pattern')
+                newpattern = pattern
+            elif marknew:
+                pattern = defpattern
+                newpattern = self.getConfig('chapter_title_new_pattern')
+            else:
+                pattern = defpattern
+                newpattern = pattern
+
+            if self.getConfig('add_chapter_numbers') in ["true","toconly"]:
+                tocpattern = self.getConfig('chapter_title_add_pattern')
+            else:
+                tocpattern = defpattern
+
+            # logger.debug("Patterns: (%s)(%s)"%(pattern,newpattern))
+            templ = string.Template(pattern)
+            newtempl = string.Template(newpattern)
+            toctempl = string.Template(tocpattern)            
+            
+            for index, chap in enumerate(self.chapters):
+                if chap.new:
+                    usetempl = newtempl
+                else:
+                    usetempl = templ
+                # logger.debug("chap.url, chap.new: (%s)(%s)"%(chap.url,chap.new))
+                retval.append( Chapter(chap.url,
+                                       # 'new'
+                                       usetempl.substitute({'index':index+1,'title':chap.title}),
+                                       chap.html,
+                                       # 'orig'
+                                       templ.substitute({'index':index+1,'title':chap.title}),
+                                       # 'toc'
+                                       toctempl.substitute({'index':index+1,'title':chap.title}),
+                                       chap.new) )
        else:
            retval = self.chapters

@@ -779,9 +927,12 @@ class Story(Configurable):

        if not allowunsafefilename:
            values={}
-            pattern = re_compile(self.getConfig("output_filename_safepattern",r"[^a-zA-Z0-9_\. \[\]\(\)&'-]+"),"output_filename_safepattern")
+            pattern = re_compile(self.getConfig("output_filename_safepattern",r"(^\.|/\.|[^a-zA-Z0-9_\. \[\]\(\)&'-]+)"),"output_filename_safepattern")
            for k in origvalues.keys():
-                values[k]=re.sub(pattern,'_', removeAllEntities(self.getMetadata(k)))
+                if k == 'formatext': # don't do file extension--we set it anyway.
+                    values[k]=self.getMetadata(k)
+                else:
+                    values[k]=re.sub(pattern,'_', removeAllEntities(self.getMetadata(k)))

        return string.Template(template).substitute(values).encode('utf8')

@@ -832,6 +983,7 @@ class Story(Configurable):
                #print("\n===========\nparsedUrl.path:%s\ntoppath:%s\nimgurl:%s\n\n"%(parsedUrl.path,toppath,imgurl))

        # apply coverexclusion to explicit covers, too.  Primarily for ffnet imageu.
+        #print("[[[[[\n\n %s %s \n\n]]]]]]]"%(imgurl,coverexclusion))
        if cover and coverexclusion and re.search(coverexclusion,imgurl):
            return (None,None)

@@ -874,6 +1026,7 @@ class Story(Configurable):
                self.imgurls.insert(0,imgurl)
                newsrc = "images/cover.%s"%ext
                self.cover=newsrc
+                self.setMetadata('cover_image','specific')
                self.imgtuples.insert(0,{'newsrc':newsrc,'mime':mime,'data':data})
            else:
                self.imgurls.append(imgurl)
@@ -888,6 +1041,7 @@ class Story(Configurable):
                        not (coverexclusion and re.search(coverexclusion,imgurl)):
                    newsrc = "images/cover.%s"%ext
                    self.cover=newsrc
+                    self.setMetadata('cover_image','first')
                    self.imgtuples.append({'newsrc':newsrc,'mime':mime,'data':data})
                    self.imgurls.append(imgurl)

@@ -921,4 +1075,3 @@ def commaGroups(s):
        groups.append(s[-3:])
        s = s[:-3]
    return s + ','.join(reversed(groups))
-
@@ -45,36 +45,6 @@ class BaseStoryWriter(Configurable):
        self.adapter = adapter
        self.story = adapter.getStoryMetadataOnly() # only cache the metadata initially.
        
-        # fall back labels.
-        self.titleLabels = {
-            'category':'Category',
-            'genre':'Genre',
-            'language':'Language',
-            'status':'Status',
-            'series':'Series',
-            'characters':'Characters',
-            'ships':'Relationships',
-            'datePublished':'Published',
-            'dateUpdated':'Updated',
-            'dateCreated':'Packaged',
-            'rating':'Rating',
-            'warnings':'Warnings',
-            'numChapters':'Chapters',
-            'numWords':'Words',
-            'site':'Site',
-            'storyId':'Story ID',
-            'authorId':'Author ID',
-            'extratags':'Extra Tags',
-            'title':'Title',
-            'storyUrl':'Story URL',
-            'description':'Summary',
-            'author':'Author',
-            'authorUrl':'Author URL',
-            'formatname':'File Format',
-            'formatext':'File Extension',
-            'siteabbrev':'Site Abbrev',
-            'version':'Downloader Version'
-            }
        self.story.setMetadata('formatname',self.getFormatName())
        self.story.setMetadata('formatext',self.getFormatExt())

@@ -135,15 +105,16 @@ class BaseStoryWriter(Configurable):
                            TEMPLATE=WIDE_ENTRY
                        else:
                            TEMPLATE=ENTRY
-                            
-                        if self.hasConfig(entry+"_label"):
-                            label=self.getConfig(entry+"_label")
-                        elif entry in self.titleLabels:
-                            logger.debug("Using fallback label for %s_label"%entry)
-                            label=self.titleLabels[entry]
-                        else:
-                            label="%s"%entry.title()
-                            logger.debug("No known label for %s, fallback to '%s'"%(entry,label))
+
+                        label=self.get_label(entry)
+                        # if self.hasConfig(entry+"_label"):
+                        #     label=self.getConfig(entry+"_label")
+                        # elif entry in self.titleLabels:
+                        #     logger.debug("Using fallback label for %s_label"%entry)
+                        #     label=self.titleLabels[entry]
+                        # else:
+                        #     label="%s"%entry.title()
+                        #     logger.debug("No known label for %s, fallback to '%s'"%(entry,label))

                        # If the label for the title entry is empty, use the
                        # 'no title' option if there is one.
@@ -177,12 +148,12 @@ class BaseStoryWriter(Configurable):
            
            self._write(out,START.substitute(self.story.getAllMetadata()))

-            for index, (url,title,html) in enumerate(self.story.getChapters(fortoc=True)):
-                if html:
-                    self._write(out,ENTRY.substitute({'chapter':title,
+            for index, chap in enumerate(self.story.getChapters(fortoc=True)):
+                if chap.html:
+                    self._write(out,ENTRY.substitute({'chapter':chap.title,
                                                      'number':index+1,
                                                      'index':"%04d"%(index+1),
-                                                      'url':url}))
+                                                      'url':chap.url}))

            self._write(out,END.substitute(self.story.getAllMetadata()))

@@ -28,7 +28,7 @@ import re
 from xml.dom.minidom import parse, parseString, getDOMImplementation

 from base_writer import *
-from ..htmlcleanup import stripHTML
+from ..htmlcleanup import stripHTML,removeEntities

 logger = logging.getLogger(__name__)

@@ -133,6 +133,10 @@ ${value}<br />
 <head>
 <title>${chapter}</title>
 <link href="stylesheet.css" type="text/css" charset="UTF-8" rel="stylesheet"/>
+<meta name="chapterurl" content="${url}"></meta>
+<meta name="chapterorigtitle" content="${origchapter}"></meta>
+<meta name="chaptertoctitle" content="${tocchapter}"></meta>
+<meta name="chaptertitle" content="${chapter}"></meta>
 </head>
 <body>
 <h3>${chapter}</h3>
@@ -258,14 +262,15 @@ div { margin: 0pt; padding: 0pt; }
            if self.isValidMetaEntry(entry):
                val = self.story.getMetadata(entry)
                if val and ( entry not in oldvalues or val != oldvalues[entry] ):
-                    if self.hasConfig(entry+"_label"):
-                        label=self.getConfig(entry+"_label")
-                    elif entry in self.titleLabels:
-                        logger.debug("Using fallback label for %s_label"%entry)
-                        label=self.titleLabels[entry]
-                    else:
-                        label="%s"%entry.title()
-                        logger.debug("No known label for %s, fallback to '%s'"%(entry,label))
+                    label=self.get_label(entry)
+                    # if self.hasConfig(entry+"_label"):
+                    #     label=self.getConfig(entry+"_label")
+                    # elif entry in self.titleLabels:
+                    #     logger.debug("Using fallback label for %s_label"%entry)
+                    #     label=self.titleLabels[entry]
+                    # else:
+                    #     label="%s"%entry.title()
+                    #     logger.debug("No known label for %s, fallback to '%s'"%(entry,label))

                    retval = retval + ENTRY.substitute({'id':entry,
                                                        'label':label,
@@ -501,13 +506,13 @@ div { margin: 0pt; padding: 0pt; }
            items.append(("log_page","OEBPS/log_page.xhtml","application/xhtml+xml","Update Log"))
            itemrefs.append("log_page")
            
-        for index, (url,title,html) in enumerate(self.story.getChapters(fortoc=True)):
-            if html:
+        for index, chap in enumerate(self.story.getChapters(fortoc=True)):
+            if chap.html:
                i=index+1
                items.append(("file%04d"%i,
                              "OEBPS/file%04d.xhtml"%i,
                              "application/xhtml+xml",
-                              title))
+                              chap.title))
                itemrefs.append("file%04d"%i)

        manifest = contentdom.createElement("manifest")
@@ -649,19 +654,21 @@ div { margin: 0pt; padding: 0pt; }
        else:
            CHAPTER_END = self.EPUB_CHAPTER_END
        
-        for index, (url,title,html) in enumerate(self.story.getChapters()):
-            if html:
-                logger.debug('Writing chapter text for: %s' % title)
-                vals={'url':url, 'chapter':title, 'index':"%04d"%(index+1), 'number':index+1}
-                fullhtml = CHAPTER_START.substitute(vals) + html + CHAPTER_END.substitute(vals)
+        for index, chap in enumerate(self.story.getChapters()): # (url,title,html)
+            if chap.html:
+                logger.debug('Writing chapter text for: %s' % chap.title)
+                vals={'url':removeEntities(chap.url),
+                      'chapter':chap.title,
+                      'origchapter':chap.origtitle,
+                      'tocchapter':chap.toctitle,
+                      'index':"%04d"%(index+1),
+                      'number':index+1}
+                fullhtml = CHAPTER_START.substitute(vals) + \
+                    chap.html + CHAPTER_END.substitute(vals)
                # ffnet(& maybe others) gives the whole chapter text
                # as one line.  This causes problems for nook(at
                # least) when the chapter size starts getting big
                # (200k+)
-                #fullhtml = fullhtml.replace('</p>','</p>\n').replace('<br />','<br />\n')
-                # The replaces above added tons of extra newlines
-                # during *each* epub update.  The regexp version adds
-                # only one and removes any extra.
                fullhtml = re.sub(r'(</p>|<br ?/>)\n*',r'\1\n',fullhtml)

                outputepub.writestr("OEBPS/file%04d.xhtml"%(index+1),fullhtml.encode('utf-8'))
@@ -128,12 +128,12 @@ ${output_css}
        else:
            CHAPTER_END = self.HTML_CHAPTER_END
        
-        for index, (url,title,html) in enumerate(self.story.getChapters()):
-            if html:
-                logging.debug('Writing chapter text for: %s' % title)
-                vals={'url':url, 'chapter':title, 'index':"%04d"%(index+1), 'number':index+1}
+        for index, chap in enumerate(self.story.getChapters()):
+            if chap.html:
+                logging.debug('Writing chapter text for: %s' % chap.title)
+                vals={'url':chap.url, 'chapter':chap.title, 'index':"%04d"%(index+1), 'number':index+1}
                self._write(out,CHAPTER_START.substitute(vals))
-                self._write(out,html)
+                self._write(out,chap.html)
                self._write(out,CHAPTER_END.substitute(vals))

        self._write(out,FILE_END.substitute(self.story.getAllMetadata()))
@@ -161,11 +161,11 @@ ${value}<br />
        else:
            CHAPTER_END = self.MOBI_CHAPTER_END
        
-        for index, (url,title,html) in enumerate(self.story.getChapters()):
-            if html:
-                logger.debug('Writing chapter text for: %s' % title)
-                vals={'url':url, 'chapter':title, 'index':"%04d"%(index+1), 'number':index+1}
-                fullhtml = CHAPTER_START.substitute(vals) + html + CHAPTER_END.substitute(vals)
+        for index, chap in enumerate(self.story.getChapters()):
+            if chap.html:
+                logger.debug('Writing chapter text for: %s' % chap.title)
+                vals={'url':chap.url, 'chapter':chap.title, 'index':"%04d"%(index+1), 'number':index+1}
+                fullhtml = CHAPTER_START.substitute(vals) + chap.html + CHAPTER_END.substitute(vals)
                # ffnet(& maybe others) gives the whole chapter text
                # as one line.  This causes problems for nook(at
                # least) when the chapter size starts getting big
@@ -154,12 +154,12 @@ End file.
        else:
            CHAPTER_END = self.TEXT_CHAPTER_END
        
-        for index, (url, title,html) in enumerate(self.story.getChapters()):
-            if html:
-                logging.debug('Writing chapter text for: %s' % title)
-                vals={'url':url, 'chapter':title, 'index':"%04d"%(index+1), 'number':index+1}
+        for index, chap in enumerate(self.story.getChapters()):
+            if chap.html:
+                logging.debug('Writing chapter text for: %s' % chap.title)
+                vals={'url':chap.url, 'chapter':chap.title, 'index':"%04d"%(index+1), 'number':index+1}
                self._write(out,self.lineends(self.wraplines(removeAllEntities(CHAPTER_START.substitute(vals)))))
-                self._write(out,self.lineends(html2text(html,wrap_width=self.wrap_width)))
+                self._write(out,self.lineends(html2text(chap.html,wrap_width=self.wrap_width)))
                self._write(out,self.lineends(self.wraplines(removeAllEntities(CHAPTER_END.substitute(vals)))))

        self._write(out,self.lineends(self.wraplines(FILE_END.substitute(self.story.getAllMetadata()))))
@@ -0,0 +1,283 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+A pure Python GIF metadata extractor.
+Supports adjustable detail to fine-tune performance.
+
+Example code and full epydoc docstrings included.
+
+Uses:
+ - Identifying whether a GIF is static or animated.
+ - Extracting the dimensions, pixel aspect ratio, number of frames, loop count,
+    global palette or palette size, and background color.
+ - Extracting comments and other plaintext.
+ - Testing for various structural errors.
+
+TODO:
+ - Provide basic support for XMP Metadata extraction
+   - http://en.wikipedia.org/wiki/Extensible_Metadata_Platform#Location_in_file_types
+   - http://www.matthewflickinger.com/lab/whatsinagif/bits_and_bytes.asp
+   - Generate test GIF with http://code.google.com/p/python-xmp-toolkit/
+
+Changelog:
+ - 0.2.2: Audited the code and made some corrections.
+ - 0.2.1: 40% speed improvement (went from 15 to 9 seconds for 1000 images)
+ - 0.2.0: Feature-complete
+ - 0.1.0: Initial release
+"""
+
+__appname__ = "gif.py"
+__author__  = "Stephan Sokolow (deitarion/SSokolow)"
+__version__ = "0.2.2"
+__license__ = "PSF License 2.4 or higher (The Python License)"
+
+#{ Check Types (enum, numerical ordering is significant)
+CHECK_IS_GIF_FILE   = 0  #: Just check for a valid GIF header.
+CHECK_IS_ANIMATED   = 1  #: Check whether the file has more than one frame.
+CHECK_COUNT_FRAMES  = 2  #: Count the number of frames in the file.
+CHECK_PARSE_PALETTE = 3  #: Parse the palette and resolve the background color.
+CHECK_READ_COMMENTS = 4  #: Load comments (can sometimes be large) into the L{GifInfo} object.
+CHECK_READ_ALL_TEXT = 5  #: Also load the contents of Plain Text extension blocks.
+
+CHECK_ALL = CHECK_READ_ALL_TEXT #: alias to allow for future modifications
+
+#{  Warning Codes (bitfield)
+WARN_NONE = 0        #: No warnings
+WARN_BAD_IMG = 1     #: Corruption (of the [sub]block size field(s)) or truncation detected in an image block
+WARN_BAD_EXT = 2     #: Corruption (of the [sub]block size field(s)) or truncation detected in an extension block
+WARN_BAD_SIZE = 4    #: An image block specifies dimensions exceeding the global canvas size
+WARN_BAD_BGCOLOR = 8 #: The background color index specified is greater than the palette size
+WARN_EOF  = 16       #: File is missing it's trailer. (Corrupt elsewhere, truncated, or breaking spec by using EOF as the terminator.)
+WARN_TRUNC  = 32     #: File is definitively either truncated or corrupt. (An EOF was encountered part-way through a structure.)
+WARN_LOOP_POS = 64   #: Netscape Application Extension block (animation-control) was present but not first in the file.
+#}
+
+import struct
+
+#{ Structures used by GifInfo
+gifHeaderStruct = struct.Struct('<xxxxxxHHBBB')  #: File header
+gifImageStruct = struct.Struct('<HHHHB')         #: Image block header
+gifExtenStruct = struct.Struct('<BB')            #: Top-level extension block header
+gifNetscapeStruct = struct.Struct('<BBH')        #: NETSCAPE Loop-control sub-block
+gifColorTripleStruct = struct.Struct('<BBB')     #: RGB palette element
+gifPlaintextStruct = struct.Struct('<BHHHHBBBB') #: Plain Text block header
+#}
+
+class BadHeaderException(Exception):
+    """Raised when no valid GIF header is found"""
+
+class GifInfo(object):
+    """A class for loading and storing metadata from GIF files.
+
+    Accepts paths and file-like objects.
+
+    When using L{CHECK_ALL}, this can also be used to walk past a valid
+    GIF file in an un-delimited byte stream in order to identify the point at
+    which the following file starts. (It doesn't C{fh.seek(0)} or C{fh.close()} after
+    use)
+    """
+    warnFlags = WARN_NONE  #: A bit field of C{WARN_*} flags set by L{__init__}
+    checkLevel = CHECK_ALL #: Default C{CHECK_*} level used by L{__init__}
+
+    #{ Pre-defined "unset" values for GIF Metadata
+    path = None        #: The path to the file, if one was passed to L{__init__}
+    version = None     #: C{87a} or C{89a}
+    width = None
+    height = None
+    loopCount = None
+    pixelAspect = None
+    paletteSize = None #: Always calculated if a global palette is present
+    palette = None     #: The palette as a list of integer RGB tuples. Requires L{CHECK_PARSE_PALETTE}.
+    bgColor = None     #: Global background color as an RGB tuple. Requires L{CHECK_PARSE_PALETTE}.
+    comments = None    #: Text in Comment (0xFE) extension blocks as a list of strings. Requires L{CHECK_READ_COMMENTS}
+    otherText = None   #: Text in "Plain Text" (0x01) extension blocks as a list of strings. Requires L{CHECK_READ_ALL_TEXT}
+    frameCount = 0
+    #}
+
+    def __init__(self, fh, checkLevel=checkLevel):
+        """
+        @param fh: A path or file-like object for a GIF file.
+        @param checkLevel: A C{CHECK_*} constant.
+
+        @raises BadHeaderException: The given file lacks a valid GIF header.
+        @raises IOError: The underlying C{open()} system call failed.
+        """
+        self.checkLevel = checkLevel
+        if isinstance(fh, basestring):
+            self.path = fh
+            fh = open(fh, 'rb')
+
+        header = fh.read(gifHeaderStruct.size)
+        if len(header) < gifHeaderStruct.size:
+            raise BadHeaderException("File is too small to be a GIF")
+
+        self.version = header[3:6]
+        if header[0:3] != 'GIF' or self.version not in ['87a', '89a']:
+            raise BadHeaderException("File does not have a recognizable GIF header")
+        elif self.checkLevel <= CHECK_IS_GIF_FILE:
+            return
+
+        self.width, self.height, GCTF_Byte, bgColor, self.pixelAspect = gifHeaderStruct.unpack(header)
+
+        if self.pixelAspect:
+            self.pixelAspect = (self.pixelAspect + 15) / 64.0
+
+        rawPalette = self._getPalette(fh, GCTF_Byte)
+        self.paletteSize = int(len(rawPalette) / 3)
+        if self.checkLevel >= CHECK_PARSE_PALETTE:
+            self.palette = []
+            for pos in range(0, self.paletteSize):
+                self.palette.append(gifColorTripleStruct.unpack_from(rawPalette, pos * 3))
+
+        if self.paletteSize and bgColor > self.paletteSize:
+            self.warnFlags = self.warnFlags | WARN_BAD_BGCOLOR
+        elif self.palette:
+            self.bgColor = self.palette[bgColor]
+
+        # Iterate blocks
+        self.firstBlock = True
+        blocktype = self._read(fh, 1)
+        while not blocktype == chr(0x3B) and not self.warnFlags & WARN_EOF:
+            self._blockHandlers.get(blocktype, lambda x, y:'')(self, fh)
+            if self.checkLevel <= CHECK_IS_ANIMATED and self.frameCount > 1:
+                return
+
+            self.firstBlock = False
+            blocktype = self._read(fh, 1)
+
+        del self.firstBlock
+
+    def _handleImageBlock(self, fh):
+        """"""
+        self.frameCount += 1
+        try:
+            x, y, w, h, LCTF_Byte = gifImageStruct.unpack(self._read(fh, gifImageStruct.size))
+        except:
+            self.warnFlags = self.warnFlags | WARN_EOF | WARN_TRUNC
+            return
+
+        if x + w > self.width or y + h > self.height:
+            self.warnFlags = self.warnFlags | WARN_BAD_SIZE
+
+        self._getPalette(fh, LCTF_Byte) # Skip the local color table if present
+        fh.read(1)                      # Skip the LZW minimum code size.
+
+        # Skip content and test for the block terminator
+        if not self._skipSubBlocks(fh): # For example, if it's a zero-length string like EOF would return.
+            self.warnFlags = self.warnFlags | WARN_BAD_IMG
+
+    def _handleGenericExtensionBlock(self, fh):
+        """@todo: Rewrite this so extension block types have method handlers."""
+        try:
+            extType, blkSize = gifExtenStruct.unpack(self._read(fh, gifExtenStruct.size))
+        except:
+            self.warnFlags = self.warnFlags | WARN_EOF | WARN_TRUNC
+            return
+        startOffset = fh.tell()
+
+        if extType == 0x01 and self.checkLevel >= CHECK_READ_ALL_TEXT: # Plain Text Block
+            self._read(fh, gifPlaintextStruct.size)
+            self.otherText = self.otherText or []
+            blkSize = self._read(fh, 1)
+            while blkSize and blkSize != '\x00':
+                self.otherText.append(self._read(fh, ord(blkSize)))
+                blkSize = self._read(fh, 1)
+        elif extType == 0xFE and self.checkLevel >= CHECK_READ_COMMENTS: # Comment Block
+            self.comments = self.comments or []
+            blkSize = self._read(fh, 1)
+            while blkSize and blkSize != '\x00':
+                self.comments.append(self._read(fh, ord(blkSize)))
+                blkSize = self._read(fh, 1)
+        elif extType == 0xFF: # Application Block
+            if blkSize == 0x0B and self._read(fh, blkSize) == "NETSCAPE2.0":
+                try:
+                    a, b, self.loopCount = gifNetscapeStruct.unpack(self._read(fh, gifNetscapeStruct.size))
+                except:
+                    self.warnFlags = self.warnFlags | WARN_EOF | WARN_TRUNC
+                    return
+
+                if a != 3 and b != 1:
+                    self.warnFlags = self.warnFlags | WARN_BAD_EXT
+
+                if not self.firstBlock:
+                    self.warnFlags = self.warnFlags | WARN_LOOP_POS
+        else:
+            fh.seek( startOffset + blkSize ) # Skip the contents
+
+        # Test for the block terminator
+        if not self._skipSubBlocks(fh):
+            self.warnFlags = self.warnFlags | WARN_BAD_EXT
+
+    def _getPalette(self, handle, bitfield):
+        """Using the size value from C{bitfield},
+        load the palette at C{handle}'s current file pointer position."""
+        if bitfield & int("10000000", 2):
+            nBits = bitfield & int("00000111", 2)
+            tableSize = 3 * 2**( nBits + 1 )
+            return handle.read(tableSize)
+        else:
+            return ''
+
+    def _read(self, handle, size):
+        """Attempt to read the specified number of bytes. Set L{WARN_EOF} if
+        fewer are received."""
+        content = handle.read(size)
+        if len(content) < size:
+            self.warnFlags = self.warnFlags | WARN_EOF
+        return content
+
+    def _skipSubBlocks(self, handle):
+        """Skip sub-blocks beginning at the current file pointer position
+        using fseek."""
+        offset = handle.tell()
+        blkSize = handle.read(1)
+        while blkSize and blkSize != '\x00':
+            offset += ord(blkSize) + 1
+            handle.seek(offset)
+            blkSize = handle.read(1)
+        return blkSize
+
+    _blockHandlers = {
+            chr(0x2C) : _handleImageBlock,
+            chr(0x21) : _handleGenericExtensionBlock,
+        }
+
+def gif_is_animated(path):
+    """A simple convenience function for testing whether a GIF is animated.
+    @rtype: C{bool}
+    """
+    return GifInfo(file(path,'rb'), CHECK_IS_ANIMATED).frameCount > 1
+
+if __name__ == '__main__':
+    from optparse import OptionParser
+    parser = OptionParser(description=__doc__.split('\n\n')[0],
+            version="%%prog v%s" % __version__, usage="%prog <path> ...")
+
+    opts, args = parser.parse_args()
+
+    if args:
+        for fpath in args:
+            try:
+                info = GifInfo(fpath, CHECK_COUNT_FRAMES)
+                warnFlags = (
+                        (info.warnFlags & WARN_BAD_IMG     and 'I' or ' ') +
+                        (info.warnFlags & WARN_BAD_EXT     and 'X' or ' ') +
+                        (info.warnFlags & WARN_BAD_SIZE    and 'C' or ' ') +
+                        (info.warnFlags & WARN_BAD_BGCOLOR and 'B' or ' ') +
+                        (info.warnFlags & WARN_EOF         and 'E' or ' ') +
+                        (info.warnFlags & WARN_TRUNC       and 'T' or ' ') +
+                        (info.warnFlags & WARN_LOOP_POS    and 'L' or ' ')
+                    )
+                print "[%s](%3s Frames): %s" % (warnFlags, info.frameCount, info.path)
+            except BadHeaderException, err:
+                print "%s: %s" % (str(err), fpath)
+        print "\nWarning Flags:"
+        print " I = Image Chunk Corruption/Truncation"
+        print " X = Extension Chunk Corruption/Truncation"
+        print " C = Image Chunk Dimensions Exceed Global Canvas"
+        print " B = Bad Background Color (Index Exceeds Palette Size)"
+        print " E = Unexpected EOF Encountered (Missing Image Terminator)"
+        print " T = EOF Encountered Within A Block Header (Corrupt or Truncated File)"
+        print " L = Loop-control block misplaced within the file"
+        print
+        print "Note: A nearly-threefold speed-up can be had by using CHECK_IS_ANIMATED rather than CHECK_COUNT_FRAMES"
@@ -20,4 +20,4 @@ from .serializer import serialize

 __all__ = ["HTMLParser", "parse", "parseFragment", "getTreeBuilder",
           "getTreeWalker", "serialize"]
-__version__ = "0.999"
+__version__ = "0.99999"
@@ -1,292 +1,290 @@
 from __future__ import absolute_import, division, unicode_literals

 import string
-import gettext
-_ = gettext.gettext

 EOF = None

 E = {
    "null-character":
-        _("Null character in input stream, replaced with U+FFFD."),
+        "Null character in input stream, replaced with U+FFFD.",
    "invalid-codepoint":
-        _("Invalid codepoint in stream."),
+        "Invalid codepoint in stream.",
    "incorrectly-placed-solidus":
-        _("Solidus (/) incorrectly placed in tag."),
+        "Solidus (/) incorrectly placed in tag.",
    "incorrect-cr-newline-entity":
-        _("Incorrect CR newline entity, replaced with LF."),
+        "Incorrect CR newline entity, replaced with LF.",
    "illegal-windows-1252-entity":
-        _("Entity used with illegal number (windows-1252 reference)."),
+        "Entity used with illegal number (windows-1252 reference).",
    "cant-convert-numeric-entity":
-        _("Numeric entity couldn't be converted to character "
-          "(codepoint U+%(charAsInt)08x)."),
+        "Numeric entity couldn't be converted to character "
+        "(codepoint U+%(charAsInt)08x).",
    "illegal-codepoint-for-numeric-entity":
-        _("Numeric entity represents an illegal codepoint: "
-          "U+%(charAsInt)08x."),
+        "Numeric entity represents an illegal codepoint: "
+        "U+%(charAsInt)08x.",
    "numeric-entity-without-semicolon":
-        _("Numeric entity didn't end with ';'."),
+        "Numeric entity didn't end with ';'.",
    "expected-numeric-entity-but-got-eof":
-        _("Numeric entity expected. Got end of file instead."),
+        "Numeric entity expected. Got end of file instead.",
    "expected-numeric-entity":
-        _("Numeric entity expected but none found."),
+        "Numeric entity expected but none found.",
    "named-entity-without-semicolon":
-        _("Named entity didn't end with ';'."),
+        "Named entity didn't end with ';'.",
    "expected-named-entity":
-        _("Named entity expected. Got none."),
+        "Named entity expected. Got none.",
    "attributes-in-end-tag":
-        _("End tag contains unexpected attributes."),
+        "End tag contains unexpected attributes.",
    'self-closing-flag-on-end-tag':
-        _("End tag contains unexpected self-closing flag."),
+        "End tag contains unexpected self-closing flag.",
    "expected-tag-name-but-got-right-bracket":
-        _("Expected tag name. Got '>' instead."),
+        "Expected tag name. Got '>' instead.",
    "expected-tag-name-but-got-question-mark":
-        _("Expected tag name. Got '?' instead. (HTML doesn't "
-          "support processing instructions.)"),
+        "Expected tag name. Got '?' instead. (HTML doesn't "
+        "support processing instructions.)",
    "expected-tag-name":
-        _("Expected tag name. Got something else instead"),
+        "Expected tag name. Got something else instead",
    "expected-closing-tag-but-got-right-bracket":
-        _("Expected closing tag. Got '>' instead. Ignoring '</>'."),
+        "Expected closing tag. Got '>' instead. Ignoring '</>'.",
    "expected-closing-tag-but-got-eof":
-        _("Expected closing tag. Unexpected end of file."),
+        "Expected closing tag. Unexpected end of file.",
    "expected-closing-tag-but-got-char":
-        _("Expected closing tag. Unexpected character '%(data)s' found."),
+        "Expected closing tag. Unexpected character '%(data)s' found.",
    "eof-in-tag-name":
-        _("Unexpected end of file in the tag name."),
+        "Unexpected end of file in the tag name.",
    "expected-attribute-name-but-got-eof":
-        _("Unexpected end of file. Expected attribute name instead."),
+        "Unexpected end of file. Expected attribute name instead.",
    "eof-in-attribute-name":
-        _("Unexpected end of file in attribute name."),
+        "Unexpected end of file in attribute name.",
    "invalid-character-in-attribute-name":
-        _("Invalid character in attribute name"),
+        "Invalid character in attribute name",
    "duplicate-attribute":
-        _("Dropped duplicate attribute on tag."),
+        "Dropped duplicate attribute on tag.",
    "expected-end-of-tag-name-but-got-eof":
-        _("Unexpected end of file. Expected = or end of tag."),
+        "Unexpected end of file. Expected = or end of tag.",
    "expected-attribute-value-but-got-eof":
-        _("Unexpected end of file. Expected attribute value."),
+        "Unexpected end of file. Expected attribute value.",
    "expected-attribute-value-but-got-right-bracket":
-        _("Expected attribute value. Got '>' instead."),
+        "Expected attribute value. Got '>' instead.",
    'equals-in-unquoted-attribute-value':
-        _("Unexpected = in unquoted attribute"),
+        "Unexpected = in unquoted attribute",
    'unexpected-character-in-unquoted-attribute-value':
-        _("Unexpected character in unquoted attribute"),
+        "Unexpected character in unquoted attribute",
    "invalid-character-after-attribute-name":
-        _("Unexpected character after attribute name."),
+        "Unexpected character after attribute name.",
    "unexpected-character-after-attribute-value":
-        _("Unexpected character after attribute value."),
+        "Unexpected character after attribute value.",
    "eof-in-attribute-value-double-quote":
-        _("Unexpected end of file in attribute value (\")."),
+        "Unexpected end of file in attribute value (\").",
    "eof-in-attribute-value-single-quote":
-        _("Unexpected end of file in attribute value (')."),
+        "Unexpected end of file in attribute value (').",
    "eof-in-attribute-value-no-quotes":
-        _("Unexpected end of file in attribute value."),
+        "Unexpected end of file in attribute value.",
    "unexpected-EOF-after-solidus-in-tag":
-        _("Unexpected end of file in tag. Expected >"),
+        "Unexpected end of file in tag. Expected >",
    "unexpected-character-after-solidus-in-tag":
-        _("Unexpected character after / in tag. Expected >"),
+        "Unexpected character after / in tag. Expected >",
    "expected-dashes-or-doctype":
-        _("Expected '--' or 'DOCTYPE'. Not found."),
+        "Expected '--' or 'DOCTYPE'. Not found.",
    "unexpected-bang-after-double-dash-in-comment":
-        _("Unexpected ! after -- in comment"),
+        "Unexpected ! after -- in comment",
    "unexpected-space-after-double-dash-in-comment":
-        _("Unexpected space after -- in comment"),
+        "Unexpected space after -- in comment",
    "incorrect-comment":
-        _("Incorrect comment."),
+        "Incorrect comment.",
    "eof-in-comment":
-        _("Unexpected end of file in comment."),
+        "Unexpected end of file in comment.",
    "eof-in-comment-end-dash":
-        _("Unexpected end of file in comment (-)"),
+        "Unexpected end of file in comment (-)",
    "unexpected-dash-after-double-dash-in-comment":
-        _("Unexpected '-' after '--' found in comment."),
+        "Unexpected '-' after '--' found in comment.",
    "eof-in-comment-double-dash":
-        _("Unexpected end of file in comment (--)."),
+        "Unexpected end of file in comment (--).",
    "eof-in-comment-end-space-state":
-        _("Unexpected end of file in comment."),
+        "Unexpected end of file in comment.",
    "eof-in-comment-end-bang-state":
-        _("Unexpected end of file in comment."),
+        "Unexpected end of file in comment.",
    "unexpected-char-in-comment":
-        _("Unexpected character in comment found."),
+        "Unexpected character in comment found.",
    "need-space-after-doctype":
-        _("No space after literal string 'DOCTYPE'."),
+        "No space after literal string 'DOCTYPE'.",
    "expected-doctype-name-but-got-right-bracket":
-        _("Unexpected > character. Expected DOCTYPE name."),
+        "Unexpected > character. Expected DOCTYPE name.",
    "expected-doctype-name-but-got-eof":
-        _("Unexpected end of file. Expected DOCTYPE name."),
+        "Unexpected end of file. Expected DOCTYPE name.",
    "eof-in-doctype-name":
-        _("Unexpected end of file in DOCTYPE name."),
+        "Unexpected end of file in DOCTYPE name.",
    "eof-in-doctype":
-        _("Unexpected end of file in DOCTYPE."),
+        "Unexpected end of file in DOCTYPE.",
    "expected-space-or-right-bracket-in-doctype":
-        _("Expected space or '>'. Got '%(data)s'"),
+        "Expected space or '>'. Got '%(data)s'",
    "unexpected-end-of-doctype":
-        _("Unexpected end of DOCTYPE."),
+        "Unexpected end of DOCTYPE.",
    "unexpected-char-in-doctype":
-        _("Unexpected character in DOCTYPE."),
+        "Unexpected character in DOCTYPE.",
    "eof-in-innerhtml":
-        _("XXX innerHTML EOF"),
+        "XXX innerHTML EOF",
    "unexpected-doctype":
-        _("Unexpected DOCTYPE. Ignored."),
+        "Unexpected DOCTYPE. Ignored.",
    "non-html-root":
-        _("html needs to be the first start tag."),
+        "html needs to be the first start tag.",
    "expected-doctype-but-got-eof":
-        _("Unexpected End of file. Expected DOCTYPE."),
+        "Unexpected End of file. Expected DOCTYPE.",
    "unknown-doctype":
-        _("Erroneous DOCTYPE."),
+        "Erroneous DOCTYPE.",
    "expected-doctype-but-got-chars":
-        _("Unexpected non-space characters. Expected DOCTYPE."),
+        "Unexpected non-space characters. Expected DOCTYPE.",
    "expected-doctype-but-got-start-tag":
-        _("Unexpected start tag (%(name)s). Expected DOCTYPE."),
+        "Unexpected start tag (%(name)s). Expected DOCTYPE.",
    "expected-doctype-but-got-end-tag":
-        _("Unexpected end tag (%(name)s). Expected DOCTYPE."),
+        "Unexpected end tag (%(name)s). Expected DOCTYPE.",
    "end-tag-after-implied-root":
-        _("Unexpected end tag (%(name)s) after the (implied) root element."),
+        "Unexpected end tag (%(name)s) after the (implied) root element.",
    "expected-named-closing-tag-but-got-eof":
-        _("Unexpected end of file. Expected end tag (%(name)s)."),
+        "Unexpected end of file. Expected end tag (%(name)s).",
    "two-heads-are-not-better-than-one":
-        _("Unexpected start tag head in existing head. Ignored."),
+        "Unexpected start tag head in existing head. Ignored.",
    "unexpected-end-tag":
-        _("Unexpected end tag (%(name)s). Ignored."),
+        "Unexpected end tag (%(name)s). Ignored.",
    "unexpected-start-tag-out-of-my-head":
-        _("Unexpected start tag (%(name)s) that can be in head. Moved."),
+        "Unexpected start tag (%(name)s) that can be in head. Moved.",
    "unexpected-start-tag":
-        _("Unexpected start tag (%(name)s)."),
+        "Unexpected start tag (%(name)s).",
    "missing-end-tag":
-        _("Missing end tag (%(name)s)."),
+        "Missing end tag (%(name)s).",
    "missing-end-tags":
-        _("Missing end tags (%(name)s)."),
+        "Missing end tags (%(name)s).",
    "unexpected-start-tag-implies-end-tag":
-        _("Unexpected start tag (%(startName)s) "
-          "implies end tag (%(endName)s)."),
+        "Unexpected start tag (%(startName)s) "
+        "implies end tag (%(endName)s).",
    "unexpected-start-tag-treated-as":
-        _("Unexpected start tag (%(originalName)s). Treated as %(newName)s."),
+        "Unexpected start tag (%(originalName)s). Treated as %(newName)s.",
    "deprecated-tag":
-        _("Unexpected start tag %(name)s. Don't use it!"),
+        "Unexpected start tag %(name)s. Don't use it!",
    "unexpected-start-tag-ignored":
-        _("Unexpected start tag %(name)s. Ignored."),
+        "Unexpected start tag %(name)s. Ignored.",
    "expected-one-end-tag-but-got-another":
-        _("Unexpected end tag (%(gotName)s). "
-          "Missing end tag (%(expectedName)s)."),
+        "Unexpected end tag (%(gotName)s). "
+        "Missing end tag (%(expectedName)s).",
    "end-tag-too-early":
-        _("End tag (%(name)s) seen too early. Expected other end tag."),
+        "End tag (%(name)s) seen too early. Expected other end tag.",
    "end-tag-too-early-named":
-        _("Unexpected end tag (%(gotName)s). Expected end tag (%(expectedName)s)."),
+        "Unexpected end tag (%(gotName)s). Expected end tag (%(expectedName)s).",
    "end-tag-too-early-ignored":
-        _("End tag (%(name)s) seen too early. Ignored."),
+        "End tag (%(name)s) seen too early. Ignored.",
    "adoption-agency-1.1":
-        _("End tag (%(name)s) violates step 1, "
-          "paragraph 1 of the adoption agency algorithm."),
+        "End tag (%(name)s) violates step 1, "
+        "paragraph 1 of the adoption agency algorithm.",
    "adoption-agency-1.2":
-        _("End tag (%(name)s) violates step 1, "
-          "paragraph 2 of the adoption agency algorithm."),
+        "End tag (%(name)s) violates step 1, "
+        "paragraph 2 of the adoption agency algorithm.",
    "adoption-agency-1.3":
-        _("End tag (%(name)s) violates step 1, "
-          "paragraph 3 of the adoption agency algorithm."),
+        "End tag (%(name)s) violates step 1, "
+        "paragraph 3 of the adoption agency algorithm.",
    "adoption-agency-4.4":
-        _("End tag (%(name)s) violates step 4, "
-          "paragraph 4 of the adoption agency algorithm."),
+        "End tag (%(name)s) violates step 4, "
+        "paragraph 4 of the adoption agency algorithm.",
    "unexpected-end-tag-treated-as":
-        _("Unexpected end tag (%(originalName)s). Treated as %(newName)s."),
+        "Unexpected end tag (%(originalName)s). Treated as %(newName)s.",
    "no-end-tag":
-        _("This element (%(name)s) has no end tag."),
+        "This element (%(name)s) has no end tag.",
    "unexpected-implied-end-tag-in-table":
-        _("Unexpected implied end tag (%(name)s) in the table phase."),
+        "Unexpected implied end tag (%(name)s) in the table phase.",
    "unexpected-implied-end-tag-in-table-body":
-        _("Unexpected implied end tag (%(name)s) in the table body phase."),
+        "Unexpected implied end tag (%(name)s) in the table body phase.",
    "unexpected-char-implies-table-voodoo":
-        _("Unexpected non-space characters in "
-          "table context caused voodoo mode."),
+        "Unexpected non-space characters in "
+        "table context caused voodoo mode.",
    "unexpected-hidden-input-in-table":
-        _("Unexpected input with type hidden in table context."),
+        "Unexpected input with type hidden in table context.",
    "unexpected-form-in-table":
-        _("Unexpected form in table context."),
+        "Unexpected form in table context.",
    "unexpected-start-tag-implies-table-voodoo":
-        _("Unexpected start tag (%(name)s) in "
-          "table context caused voodoo mode."),
+        "Unexpected start tag (%(name)s) in "
+        "table context caused voodoo mode.",
    "unexpected-end-tag-implies-table-voodoo":
-        _("Unexpected end tag (%(name)s) in "
-          "table context caused voodoo mode."),
+        "Unexpected end tag (%(name)s) in "
+        "table context caused voodoo mode.",
    "unexpected-cell-in-table-body":
-        _("Unexpected table cell start tag (%(name)s) "
-          "in the table body phase."),
+        "Unexpected table cell start tag (%(name)s) "
+        "in the table body phase.",
    "unexpected-cell-end-tag":
-        _("Got table cell end tag (%(name)s) "
-          "while required end tags are missing."),
+        "Got table cell end tag (%(name)s) "
+        "while required end tags are missing.",
    "unexpected-end-tag-in-table-body":
-        _("Unexpected end tag (%(name)s) in the table body phase. Ignored."),
+        "Unexpected end tag (%(name)s) in the table body phase. Ignored.",
    "unexpected-implied-end-tag-in-table-row":
-        _("Unexpected implied end tag (%(name)s) in the table row phase."),
+        "Unexpected implied end tag (%(name)s) in the table row phase.",
    "unexpected-end-tag-in-table-row":
-        _("Unexpected end tag (%(name)s) in the table row phase. Ignored."),
+        "Unexpected end tag (%(name)s) in the table row phase. Ignored.",
    "unexpected-select-in-select":
-        _("Unexpected select start tag in the select phase "
-          "treated as select end tag."),
+        "Unexpected select start tag in the select phase "
+        "treated as select end tag.",
    "unexpected-input-in-select":
-        _("Unexpected input start tag in the select phase."),
+        "Unexpected input start tag in the select phase.",
    "unexpected-start-tag-in-select":
-        _("Unexpected start tag token (%(name)s in the select phase. "
-          "Ignored."),
+        "Unexpected start tag token (%(name)s in the select phase. "
+        "Ignored.",
    "unexpected-end-tag-in-select":
-        _("Unexpected end tag (%(name)s) in the select phase. Ignored."),
+        "Unexpected end tag (%(name)s) in the select phase. Ignored.",
    "unexpected-table-element-start-tag-in-select-in-table":
-        _("Unexpected table element start tag (%(name)s) in the select in table phase."),
+        "Unexpected table element start tag (%(name)s) in the select in table phase.",
    "unexpected-table-element-end-tag-in-select-in-table":
-        _("Unexpected table element end tag (%(name)s) in the select in table phase."),
+        "Unexpected table element end tag (%(name)s) in the select in table phase.",
    "unexpected-char-after-body":
-        _("Unexpected non-space characters in the after body phase."),
+        "Unexpected non-space characters in the after body phase.",
    "unexpected-start-tag-after-body":
-        _("Unexpected start tag token (%(name)s)"
-          " in the after body phase."),
+        "Unexpected start tag token (%(name)s)"
+        " in the after body phase.",
    "unexpected-end-tag-after-body":
-        _("Unexpected end tag token (%(name)s)"
-          " in the after body phase."),
+        "Unexpected end tag token (%(name)s)"
+        " in the after body phase.",
    "unexpected-char-in-frameset":
-        _("Unexpected characters in the frameset phase. Characters ignored."),
+        "Unexpected characters in the frameset phase. Characters ignored.",
    "unexpected-start-tag-in-frameset":
-        _("Unexpected start tag token (%(name)s)"
-          " in the frameset phase. Ignored."),
+        "Unexpected start tag token (%(name)s)"
+        " in the frameset phase. Ignored.",
    "unexpected-frameset-in-frameset-innerhtml":
-        _("Unexpected end tag token (frameset) "
-          "in the frameset phase (innerHTML)."),
+        "Unexpected end tag token (frameset) "
+        "in the frameset phase (innerHTML).",
    "unexpected-end-tag-in-frameset":
-        _("Unexpected end tag token (%(name)s)"
-          " in the frameset phase. Ignored."),
+        "Unexpected end tag token (%(name)s)"
+        " in the frameset phase. Ignored.",
    "unexpected-char-after-frameset":
-        _("Unexpected non-space characters in the "
-          "after frameset phase. Ignored."),
+        "Unexpected non-space characters in the "
+        "after frameset phase. Ignored.",
    "unexpected-start-tag-after-frameset":
-        _("Unexpected start tag (%(name)s)"
-          " in the after frameset phase. Ignored."),
+        "Unexpected start tag (%(name)s)"
+        " in the after frameset phase. Ignored.",
    "unexpected-end-tag-after-frameset":
-        _("Unexpected end tag (%(name)s)"
-          " in the after frameset phase. Ignored."),
+        "Unexpected end tag (%(name)s)"
+        " in the after frameset phase. Ignored.",
    "unexpected-end-tag-after-body-innerhtml":
-        _("Unexpected end tag after body(innerHtml)"),
+        "Unexpected end tag after body(innerHtml)",
    "expected-eof-but-got-char":
-        _("Unexpected non-space characters. Expected end of file."),
+        "Unexpected non-space characters. Expected end of file.",
    "expected-eof-but-got-start-tag":
-        _("Unexpected start tag (%(name)s)"
-          ". Expected end of file."),
+        "Unexpected start tag (%(name)s)"
+        ". Expected end of file.",
    "expected-eof-but-got-end-tag":
-        _("Unexpected end tag (%(name)s)"
-          ". Expected end of file."),
+        "Unexpected end tag (%(name)s)"
+        ". Expected end of file.",
    "eof-in-table":
-        _("Unexpected end of file. Expected table content."),
+        "Unexpected end of file. Expected table content.",
    "eof-in-select":
-        _("Unexpected end of file. Expected select content."),
+        "Unexpected end of file. Expected select content.",
    "eof-in-frameset":
-        _("Unexpected end of file. Expected frameset content."),
+        "Unexpected end of file. Expected frameset content.",
    "eof-in-script-in-script":
-        _("Unexpected end of file. Expected script content."),
+        "Unexpected end of file. Expected script content.",
    "eof-in-foreign-lands":
-        _("Unexpected end of file. Expected foreign content"),
+        "Unexpected end of file. Expected foreign content",
    "non-void-element-with-trailing-solidus":
-        _("Trailing solidus not allowed on element %(name)s"),
+        "Trailing solidus not allowed on element %(name)s",
    "unexpected-html-element-in-foreign-content":
-        _("Element %(name)s not allowed in a non-html context"),
+        "Element %(name)s not allowed in a non-html context",
    "unexpected-end-tag-before-html":
-        _("Unexpected end tag (%(name)s) before html."),
+        "Unexpected end tag (%(name)s) before html.",
    "XXX-undefined-error":
-        _("Undefined error (this sucks and should be fixed)"),
+        "Undefined error (this sucks and should be fixed)",
 }

 namespaces = {
@@ -298,7 +296,7 @@ namespaces = {
    "xmlns": "http://www.w3.org/2000/xmlns/"
 }

-scopingElements = frozenset((
+scopingElements = frozenset([
    (namespaces["html"], "applet"),
    (namespaces["html"], "caption"),
    (namespaces["html"], "html"),
@@ -316,9 +314,9 @@ scopingElements = frozenset((
    (namespaces["svg"], "foreignObject"),
    (namespaces["svg"], "desc"),
    (namespaces["svg"], "title"),
-))
+])

-formattingElements = frozenset((
+formattingElements = frozenset([
    (namespaces["html"], "a"),
    (namespaces["html"], "b"),
    (namespaces["html"], "big"),
@@ -333,9 +331,9 @@ formattingElements = frozenset((
    (namespaces["html"], "strong"),
    (namespaces["html"], "tt"),
    (namespaces["html"], "u")
-))
+])

-specialElements = frozenset((
+specialElements = frozenset([
    (namespaces["html"], "address"),
    (namespaces["html"], "applet"),
    (namespaces["html"], "area"),
@@ -416,22 +414,22 @@ specialElements = frozenset((
    (namespaces["html"], "wbr"),
    (namespaces["html"], "xmp"),
    (namespaces["svg"], "foreignObject")
-))
+])

-htmlIntegrationPointElements = frozenset((
+htmlIntegrationPointElements = frozenset([
    (namespaces["mathml"], "annotaion-xml"),
    (namespaces["svg"], "foreignObject"),
    (namespaces["svg"], "desc"),
    (namespaces["svg"], "title")
-))
+])

-mathmlTextIntegrationPointElements = frozenset((
+mathmlTextIntegrationPointElements = frozenset([
    (namespaces["mathml"], "mi"),
    (namespaces["mathml"], "mo"),
    (namespaces["mathml"], "mn"),
    (namespaces["mathml"], "ms"),
    (namespaces["mathml"], "mtext")
-))
+])

 adjustForeignAttributes = {
    "xlink:actuate": ("xlink", "actuate", namespaces["xlink"]),
@@ -451,21 +449,21 @@ adjustForeignAttributes = {
 unadjustForeignAttributes = dict([((ns, local), qname) for qname, (prefix, local, ns) in
                                  adjustForeignAttributes.items()])

-spaceCharacters = frozenset((
+spaceCharacters = frozenset([
    "\t",
    "\n",
    "\u000C",
    " ",
    "\r"
-))
+])

-tableInsertModeElements = frozenset((
+tableInsertModeElements = frozenset([
    "table",
    "tbody",
    "tfoot",
    "thead",
    "tr"
-))
+])

 asciiLowercase = frozenset(string.ascii_lowercase)
 asciiUppercase = frozenset(string.ascii_uppercase)
@@ -486,7 +484,7 @@ headingElements = (
    "h6"
 )

-voidElements = frozenset((
+voidElements = frozenset([
    "base",
    "command",
    "event-source",
@@ -502,11 +500,11 @@ voidElements = frozenset((
    "input",
    "source",
    "track"
-))
+])

-cdataElements = frozenset(('title', 'textarea'))
+cdataElements = frozenset(['title', 'textarea'])

-rcdataElements = frozenset((
+rcdataElements = frozenset([
    'style',
    'script',
    'xmp',
@@ -514,27 +512,27 @@ rcdataElements = frozenset((
    'noembed',
    'noframes',
    'noscript'
-))
+])

 booleanAttributes = {
-    "": frozenset(("irrelevant",)),
-    "style": frozenset(("scoped",)),
-    "img": frozenset(("ismap",)),
-    "audio": frozenset(("autoplay", "controls")),
-    "video": frozenset(("autoplay", "controls")),
-    "script": frozenset(("defer", "async")),
-    "details": frozenset(("open",)),
-    "datagrid": frozenset(("multiple", "disabled")),
-    "command": frozenset(("hidden", "disabled", "checked", "default")),
-    "hr": frozenset(("noshade")),
-    "menu": frozenset(("autosubmit",)),
-    "fieldset": frozenset(("disabled", "readonly")),
-    "option": frozenset(("disabled", "readonly", "selected")),
-    "optgroup": frozenset(("disabled", "readonly")),
-    "button": frozenset(("disabled", "autofocus")),
-    "input": frozenset(("disabled", "readonly", "required", "autofocus", "checked", "ismap")),
-    "select": frozenset(("disabled", "readonly", "autofocus", "multiple")),
-    "output": frozenset(("disabled", "readonly")),
+    "": frozenset(["irrelevant"]),
+    "style": frozenset(["scoped"]),
+    "img": frozenset(["ismap"]),
+    "audio": frozenset(["autoplay", "controls"]),
+    "video": frozenset(["autoplay", "controls"]),
+    "script": frozenset(["defer", "async"]),
+    "details": frozenset(["open"]),
+    "datagrid": frozenset(["multiple", "disabled"]),
+    "command": frozenset(["hidden", "disabled", "checked", "default"]),
+    "hr": frozenset(["noshade"]),
+    "menu": frozenset(["autosubmit"]),
+    "fieldset": frozenset(["disabled", "readonly"]),
+    "option": frozenset(["disabled", "readonly", "selected"]),
+    "optgroup": frozenset(["disabled", "readonly"]),
+    "button": frozenset(["disabled", "autofocus"]),
+    "input": frozenset(["disabled", "readonly", "required", "autofocus", "checked", "ismap"]),
+    "select": frozenset(["disabled", "readonly", "autofocus", "multiple"]),
+    "output": frozenset(["disabled", "readonly"]),
 }

 # entitiesWindows1252 has to be _ordered_ and needs to have an index. It
@@ -574,7 +572,7 @@ entitiesWindows1252 = (
    376     # 0x9F  0x0178  LATIN CAPITAL LETTER Y WITH DIAERESIS
 )

-xmlEntities = frozenset(('lt;', 'gt;', 'amp;', 'apos;', 'quot;'))
+xmlEntities = frozenset(['lt;', 'gt;', 'amp;', 'apos;', 'quot;'])

 entities = {
    "AElig": "\xc6",
@@ -3088,8 +3086,8 @@ tokenTypes = {
    "ParseError": 7
 }

-tagTokenTypes = frozenset((tokenTypes["StartTag"], tokenTypes["EndTag"],
-                           tokenTypes["EmptyTag"]))
+tagTokenTypes = frozenset([tokenTypes["StartTag"], tokenTypes["EndTag"],
+                           tokenTypes["EmptyTag"]])


 prefixes = dict([(v, k) for k, v in namespaces.items()])
@@ -1,8 +1,5 @@
 from __future__ import absolute_import, division, unicode_literals

-from gettext import gettext
-_ = gettext
-
 from . import _base
 from ..constants import cdataElements, rcdataElements, voidElements

@@ -23,24 +20,24 @@ class Filter(_base.Filter):
            if type in ("StartTag", "EmptyTag"):
                name = token["name"]
                if contentModelFlag != "PCDATA":
-                    raise LintError(_("StartTag not in PCDATA content model flag: %(tag)s") % {"tag": name})
+                    raise LintError("StartTag not in PCDATA content model flag: %(tag)s" % {"tag": name})
                if not isinstance(name, str):
-                    raise LintError(_("Tag name is not a string: %(tag)r") % {"tag": name})
+                    raise LintError("Tag name is not a string: %(tag)r" % {"tag": name})
                if not name:
-                    raise LintError(_("Empty tag name"))
+                    raise LintError("Empty tag name")
                if type == "StartTag" and name in voidElements:
-                    raise LintError(_("Void element reported as StartTag token: %(tag)s") % {"tag": name})
+                    raise LintError("Void element reported as StartTag token: %(tag)s" % {"tag": name})
                elif type == "EmptyTag" and name not in voidElements:
-                    raise LintError(_("Non-void element reported as EmptyTag token: %(tag)s") % {"tag": token["name"]})
+                    raise LintError("Non-void element reported as EmptyTag token: %(tag)s" % {"tag": token["name"]})
                if type == "StartTag":
                    open_elements.append(name)
                for name, value in token["data"]:
                    if not isinstance(name, str):
-                        raise LintError(_("Attribute name is not a string: %(name)r") % {"name": name})
+                        raise LintError("Attribute name is not a string: %(name)r" % {"name": name})
                    if not name:
-                        raise LintError(_("Empty attribute name"))
+                        raise LintError("Empty attribute name")
                    if not isinstance(value, str):
-                        raise LintError(_("Attribute value is not a string: %(value)r") % {"value": value})
+                        raise LintError("Attribute value is not a string: %(value)r" % {"value": value})
                if name in cdataElements:
                    contentModelFlag = "CDATA"
                elif name in rcdataElements:
@@ -51,43 +48,43 @@ class Filter(_base.Filter):
            elif type == "EndTag":
                name = token["name"]
                if not isinstance(name, str):
-                    raise LintError(_("Tag name is not a string: %(tag)r") % {"tag": name})
+                    raise LintError("Tag name is not a string: %(tag)r" % {"tag": name})
                if not name:
-                    raise LintError(_("Empty tag name"))
+                    raise LintError("Empty tag name")
                if name in voidElements:
-                    raise LintError(_("Void element reported as EndTag token: %(tag)s") % {"tag": name})
+                    raise LintError("Void element reported as EndTag token: %(tag)s" % {"tag": name})
                start_name = open_elements.pop()
                if start_name != name:
-                    raise LintError(_("EndTag (%(end)s) does not match StartTag (%(start)s)") % {"end": name, "start": start_name})
+                    raise LintError("EndTag (%(end)s) does not match StartTag (%(start)s)" % {"end": name, "start": start_name})
                contentModelFlag = "PCDATA"

            elif type == "Comment":
                if contentModelFlag != "PCDATA":
-                    raise LintError(_("Comment not in PCDATA content model flag"))
+                    raise LintError("Comment not in PCDATA content model flag")

            elif type in ("Characters", "SpaceCharacters"):
                data = token["data"]
                if not isinstance(data, str):
-                    raise LintError(_("Attribute name is not a string: %(name)r") % {"name": data})
+                    raise LintError("Attribute name is not a string: %(name)r" % {"name": data})
                if not data:
-                    raise LintError(_("%(type)s token with empty data") % {"type": type})
+                    raise LintError("%(type)s token with empty data" % {"type": type})
                if type == "SpaceCharacters":
                    data = data.strip(spaceCharacters)
                    if data:
-                        raise LintError(_("Non-space character(s) found in SpaceCharacters token: %(token)r") % {"token": data})
+                        raise LintError("Non-space character(s) found in SpaceCharacters token: %(token)r" % {"token": data})

            elif type == "Doctype":
                name = token["name"]
                if contentModelFlag != "PCDATA":
-                    raise LintError(_("Doctype not in PCDATA content model flag: %(name)s") % {"name": name})
+                    raise LintError("Doctype not in PCDATA content model flag: %(name)s" % {"name": name})
                if not isinstance(name, str):
-                    raise LintError(_("Tag name is not a string: %(tag)r") % {"tag": name})
+                    raise LintError("Tag name is not a string: %(tag)r" % {"tag": name})
                # XXX: what to do with token["data"] ?

            elif type in ("ParseError", "SerializeError"):
                pass

            else:
-                raise LintError(_("Unknown token type: %(type)s") % {"type": type})
+                raise LintError("Unknown token type: %(type)s" % {"type": type})

            yield token
@@ -18,6 +18,7 @@ from .constants import cdataElements, rcdataElements
 from .constants import tokenTypes, ReparseException, namespaces
 from .constants import htmlIntegrationPointElements, mathmlTextIntegrationPointElements
 from .constants import adjustForeignAttributes as adjustForeignAttributesMap
+from .constants import E


 def parse(doc, treebuilder="etree", encoding=None,
@@ -129,6 +130,17 @@ class HTMLParser(object):

        self.framesetOK = True

+    @property
+    def documentEncoding(self):
+        """The name of the character encoding
+        that was used to decode the input stream,
+        or :obj:`None` if that is not determined yet.
+
+        """
+        if not hasattr(self, 'tokenizer'):
+            return None
+        return self.tokenizer.stream.charEncoding[0]
+
    def isHTMLIntegrationPoint(self, element):
        if (element.name == "annotation-xml" and
                element.namespace == namespaces["mathml"]):
@@ -245,7 +257,7 @@ class HTMLParser(object):
        # XXX The idea is to make errorcode mandatory.
        self.errors.append((self.tokenizer.stream.position(), errorcode, datavars))
        if self.strict:
-            raise ParseError
+            raise ParseError(E[errorcode] % datavars)

    def normalizeToken(self, token):
        """ HTML5 specific normalizations to the token stream """
@@ -868,7 +880,7 @@ def getPhases(debug):
            self.startTagHandler = utils.MethodDispatcher([
                ("html", self.startTagHtml),
                (("base", "basefont", "bgsound", "command", "link", "meta",
-                  "noframes", "script", "style", "title"),
+                  "script", "style", "title"),
                 self.startTagProcessInHead),
                ("body", self.startTagBody),
                ("frameset", self.startTagFrameset),
@@ -1205,8 +1217,7 @@ def getPhases(debug):
            attributes["name"] = "isindex"
            self.processStartTag(impliedTagToken("input", "StartTag",
                                                 attributes=attributes,
-                                                 selfClosing=
-                                                 token["selfClosing"]))
+                                                 selfClosing=token["selfClosing"]))
            self.processEndTag(impliedTagToken("label"))
            self.processStartTag(impliedTagToken("hr", "StartTag"))
            self.processEndTag(impliedTagToken("form"))
@@ -28,7 +28,18 @@ asciiLettersBytes = frozenset([item.encode("ascii") for item in asciiLetters])
 asciiUppercaseBytes = frozenset([item.encode("ascii") for item in asciiUppercase])
 spacesAngleBrackets = spaceCharactersBytes | frozenset([b">", b"<"])

-invalid_unicode_re = re.compile("[\u0001-\u0008\u000B\u000E-\u001F\u007F-\u009F\uD800-\uDFFF\uFDD0-\uFDEF\uFFFE\uFFFF\U0001FFFE\U0001FFFF\U0002FFFE\U0002FFFF\U0003FFFE\U0003FFFF\U0004FFFE\U0004FFFF\U0005FFFE\U0005FFFF\U0006FFFE\U0006FFFF\U0007FFFE\U0007FFFF\U0008FFFE\U0008FFFF\U0009FFFE\U0009FFFF\U000AFFFE\U000AFFFF\U000BFFFE\U000BFFFF\U000CFFFE\U000CFFFF\U000DFFFE\U000DFFFF\U000EFFFE\U000EFFFF\U000FFFFE\U000FFFFF\U0010FFFE\U0010FFFF]")
+
+invalid_unicode_no_surrogate = "[\u0001-\u0008\u000B\u000E-\u001F\u007F-\u009F\uFDD0-\uFDEF\uFFFE\uFFFF\U0001FFFE\U0001FFFF\U0002FFFE\U0002FFFF\U0003FFFE\U0003FFFF\U0004FFFE\U0004FFFF\U0005FFFE\U0005FFFF\U0006FFFE\U0006FFFF\U0007FFFE\U0007FFFF\U0008FFFE\U0008FFFF\U0009FFFE\U0009FFFF\U000AFFFE\U000AFFFF\U000BFFFE\U000BFFFF\U000CFFFE\U000CFFFF\U000DFFFE\U000DFFFF\U000EFFFE\U000EFFFF\U000FFFFE\U000FFFFF\U0010FFFE\U0010FFFF]"
+
+if utils.supports_lone_surrogates:
+    # Use one extra step of indirection and create surrogates with
+    # unichr. Not using this indirection would introduce an illegal
+    # unicode literal on platforms not supporting such lone
+    # surrogates.
+    invalid_unicode_re = re.compile(invalid_unicode_no_surrogate +
+                                    eval('"\\uD800-\\uDFFF"'))
+else:
+    invalid_unicode_re = re.compile(invalid_unicode_no_surrogate)

 non_bmp_invalid_codepoints = set([0x1FFFE, 0x1FFFF, 0x2FFFE, 0x2FFFF, 0x3FFFE,
                                  0x3FFFF, 0x4FFFE, 0x4FFFF, 0x5FFFE, 0x5FFFF,
@@ -164,13 +175,18 @@ class HTMLUnicodeInputStream(object):

        """

-        # Craziness
-        if len("\U0010FFFF") == 1:
+        if not utils.supports_lone_surrogates:
+            # Such platforms will have already checked for such
+            # surrogate errors, so no need to do this checking.
+            self.reportCharacterErrors = None
+            self.replaceCharactersRegexp = None
+        elif len("\U0010FFFF") == 1:
            self.reportCharacterErrors = self.characterErrorsUCS4
-            self.replaceCharactersRegexp = re.compile("[\uD800-\uDFFF]")
+            self.replaceCharactersRegexp = re.compile(eval('"[\\uD800-\\uDFFF]"'))
        else:
            self.reportCharacterErrors = self.characterErrorsUCS2
-            self.replaceCharactersRegexp = re.compile("([\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?<![\uD800-\uDBFF])[\uDC00-\uDFFF])")
+            self.replaceCharactersRegexp = re.compile(
+                eval('"([\\uD800-\\uDBFF](?![\\uDC00-\\uDFFF])|(?<![\\uD800-\\uDBFF])[\\uDC00-\\uDFFF])"'))

        # List of where new lines occur
        self.newLines = [0]
@@ -265,11 +281,12 @@ class HTMLUnicodeInputStream(object):
                self._bufferedCharacter = data[-1]
                data = data[:-1]

-        self.reportCharacterErrors(data)
+        if self.reportCharacterErrors:
+            self.reportCharacterErrors(data)

-        # Replace invalid characters
-        # Note U+0000 is dealt with in the tokenizer
-        data = self.replaceCharactersRegexp.sub("\ufffd", data)
+            # Replace invalid characters
+            # Note U+0000 is dealt with in the tokenizer
+            data = self.replaceCharactersRegexp.sub("\ufffd", data)

        data = data.replace("\r\n", "\n")
        data = data.replace("\r", "\n")
@@ -2,11 +2,26 @@ from __future__ import absolute_import, division, unicode_literals

 import re
 from xml.sax.saxutils import escape, unescape
+from six.moves import urllib_parse as urlparse

 from .tokenizer import HTMLTokenizer
 from .constants import tokenTypes


+content_type_rgx = re.compile(r'''
+                               ^
+                               # Match a content type <application>/<type>
+                               (?P<content_type>[-a-zA-Z0-9.]+/[-a-zA-Z0-9.]+)
+                               # Match any character set and encoding
+                               (?:(?:;charset=(?:[-a-zA-Z0-9]+)(?:;(?:base64))?)
+                                 |(?:;(?:base64))?(?:;charset=(?:[-a-zA-Z0-9]+))?)
+                               # Assume the rest is data
+                               ,.*
+                               $
+                               ''',
+                              re.VERBOSE)
+
+
 class HTMLSanitizerMixin(object):
    """ sanitization of XHTML+MathML+SVG and of inline style attributes."""

@@ -100,8 +115,8 @@ class HTMLSanitizerMixin(object):
                      'xml:base', 'xml:lang', 'xml:space', 'xmlns', 'xmlns:xlink', 'y',
                      'y1', 'y2', 'zoomAndPan']

-    attr_val_is_uri = ['href', 'src', 'cite', 'action', 'longdesc', 'poster',
-                       'xlink:href', 'xml:base']
+    attr_val_is_uri = ['href', 'src', 'cite', 'action', 'longdesc', 'poster', 'background', 'datasrc',
+                       'dynsrc', 'lowsrc', 'ping', 'poster', 'xlink:href', 'xml:base']

    svg_attr_val_allows_ref = ['clip-path', 'color-profile', 'cursor', 'fill',
                               'filter', 'marker', 'marker-start', 'marker-mid', 'marker-end',
@@ -138,7 +153,9 @@ class HTMLSanitizerMixin(object):
    acceptable_protocols = ['ed2k', 'ftp', 'http', 'https', 'irc',
                            'mailto', 'news', 'gopher', 'nntp', 'telnet', 'webcal',
                            'xmpp', 'callto', 'feed', 'urn', 'aim', 'rsync', 'tag',
-                            'ssh', 'sftp', 'rtsp', 'afs']
+                            'ssh', 'sftp', 'rtsp', 'afs', 'data']
+
+    acceptable_content_types = ['image/png', 'image/jpeg', 'image/gif', 'image/webp', 'image/bmp', 'text/plain']

    # subclasses may define their own versions of these constants
    allowed_elements = acceptable_elements + mathml_elements + svg_elements
@@ -147,6 +164,7 @@ class HTMLSanitizerMixin(object):
    allowed_css_keywords = acceptable_css_keywords
    allowed_svg_properties = acceptable_svg_properties
    allowed_protocols = acceptable_protocols
+    allowed_content_types = acceptable_content_types

    # Sanitize the +html+, escaping all elements not in ALLOWED_ELEMENTS, and
    # stripping out all # attributes not in ALLOWED_ATTRIBUTES. Style
@@ -189,10 +207,17 @@ class HTMLSanitizerMixin(object):
                                       unescape(attrs[attr])).lower()
                # remove replacement characters from unescaped characters
                val_unescaped = val_unescaped.replace("\ufffd", "")
-                if (re.match("^[a-z0-9][-+.a-z0-9]*:", val_unescaped) and
-                    (val_unescaped.split(':')[0] not in
-                     self.allowed_protocols)):
-                    del attrs[attr]
+                uri = urlparse.urlparse(val_unescaped)
+                if uri:
+                    if uri.scheme not in self.allowed_protocols:
+                        del attrs[attr]
+                    if uri.scheme == 'data':
+                        m = content_type_rgx.match(uri.path)
+                        if not m:
+                            del attrs[attr]
+                        elif m.group('content_type') not in self.allowed_content_types:
+                            del attrs[attr]
+
            for attr in self.svg_attr_val_allows_ref:
                if attr in attrs:
                    attrs[attr] = re.sub(r'url\s*\(\s*[^#\s][^)]+?\)',
@@ -245,7 +270,7 @@ class HTMLSanitizerMixin(object):
            elif prop.split('-')[0].lower() in ['background', 'border', 'margin',
                                                'padding']:
                for keyword in value.split():
-                    if not keyword in self.acceptable_css_keywords and \
+                    if keyword not in self.acceptable_css_keywords and \
                            not re.match("^(#[0-9a-f]+|rgb\(\d+%?,\d*%?,?\d*%?\)?|\d{0,2}\.?\d{0,2}(cm|em|ex|in|mm|pc|pt|px|%|,|\))?)$", keyword):
                        break
                else:
@@ -1,9 +1,6 @@
 from __future__ import absolute_import, division, unicode_literals
 from six import text_type

-import gettext
-_ = gettext.gettext
-
 try:
    from functools import reduce
 except ImportError:
@@ -35,7 +32,7 @@ else:
                v = utils.surrogatePairToCodepoint(v)
            else:
                v = ord(v)
-            if not v in encode_entity_map or k.islower():
+            if v not in encode_entity_map or k.islower():
                # prefer &lt; over &LT; and similarly for &amp;, &gt;, etc.
                encode_entity_map[v] = k

@@ -208,7 +205,7 @@ class HTMLSerializer(object):
                if token["systemId"]:
                    if token["systemId"].find('"') >= 0:
                        if token["systemId"].find("'") >= 0:
-                            self.serializeError(_("System identifer contains both single and double quote characters"))
+                            self.serializeError("System identifer contains both single and double quote characters")
                        quote_char = "'"
                    else:
                        quote_char = '"'
@@ -220,7 +217,7 @@ class HTMLSerializer(object):
            elif type in ("Characters", "SpaceCharacters"):
                if type == "SpaceCharacters" or in_cdata:
                    if in_cdata and token["data"].find("</") >= 0:
-                        self.serializeError(_("Unexpected </ in CDATA"))
+                        self.serializeError("Unexpected </ in CDATA")
                    yield self.encode(token["data"])
                else:
                    yield self.encode(escape(token["data"]))
@@ -231,7 +228,7 @@ class HTMLSerializer(object):
                if name in rcdataElements and not self.escape_rcdata:
                    in_cdata = True
                elif in_cdata:
-                    self.serializeError(_("Unexpected child element of a CDATA element"))
+                    self.serializeError("Unexpected child element of a CDATA element")
                for (attr_namespace, attr_name), attr_value in token["data"].items():
                    # TODO: Add namespace support here
                    k = attr_name
@@ -279,20 +276,20 @@ class HTMLSerializer(object):
                if name in rcdataElements:
                    in_cdata = False
                elif in_cdata:
-                    self.serializeError(_("Unexpected child element of a CDATA element"))
+                    self.serializeError("Unexpected child element of a CDATA element")
                yield self.encodeStrict("</%s>" % name)

            elif type == "Comment":
                data = token["data"]
                if data.find("--") >= 0:
-                    self.serializeError(_("Comment contains --"))
+                    self.serializeError("Comment contains --")
                yield self.encodeStrict("<!--%s-->" % token["data"])

            elif type == "Entity":
                name = token["name"]
                key = name + ";"
-                if not key in entities:
-                    self.serializeError(_("Entity %s not recognized" % name))
+                if key not in entities:
+                    self.serializeError("Entity %s not recognized" % name)
                if self.resolve_entities and key not in xmlEntities:
                    data = entities[key]
                else:
@@ -158,7 +158,7 @@ def getDomBuilder(DomImplementation):
            else:
                # HACK: allow text nodes as children of the document node
                if hasattr(self.dom, '_child_node_types'):
-                    if not Node.TEXT_NODE in self.dom._child_node_types:
+                    if Node.TEXT_NODE not in self.dom._child_node_types:
                        self.dom._child_node_types = list(self.dom._child_node_types)
                        self.dom._child_node_types.append(Node.TEXT_NODE)
                self.dom.appendChild(self.dom.createTextNode(data))
@@ -10,8 +10,12 @@ returning an iterator generating tokens.

 from __future__ import absolute_import, division, unicode_literals

+__all__ = ["getTreeWalker", "pprint", "dom", "etree", "genshistream", "lxmletree",
+           "pulldom"]
+
 import sys

+from .. import constants
 from ..utils import default_etree

 treeWalkerCache = {}
@@ -55,3 +59,89 @@ def getTreeWalker(treeType, implementation=None, **kwargs):
            # XXX: NEVER cache here, caching is done in the etree submodule
            return etree.getETreeModule(implementation, **kwargs).TreeWalker
    return treeWalkerCache.get(treeType)
+
+
+def concatenateCharacterTokens(tokens):
+    pendingCharacters = []
+    for token in tokens:
+        type = token["type"]
+        if type in ("Characters", "SpaceCharacters"):
+            pendingCharacters.append(token["data"])
+        else:
+            if pendingCharacters:
+                yield {"type": "Characters", "data": "".join(pendingCharacters)}
+                pendingCharacters = []
+            yield token
+    if pendingCharacters:
+        yield {"type": "Characters", "data": "".join(pendingCharacters)}
+
+
+def pprint(walker):
+    """Pretty printer for tree walkers"""
+    output = []
+    indent = 0
+    for token in concatenateCharacterTokens(walker):
+        type = token["type"]
+        if type in ("StartTag", "EmptyTag"):
+            # tag name
+            if token["namespace"] and token["namespace"] != constants.namespaces["html"]:
+                if token["namespace"] in constants.prefixes:
+                    ns = constants.prefixes[token["namespace"]]
+                else:
+                    ns = token["namespace"]
+                name = "%s %s" % (ns, token["name"])
+            else:
+                name = token["name"]
+            output.append("%s<%s>" % (" " * indent, name))
+            indent += 2
+            # attributes (sorted for consistent ordering)
+            attrs = token["data"]
+            for (namespace, localname), value in sorted(attrs.items()):
+                if namespace:
+                    if namespace in constants.prefixes:
+                        ns = constants.prefixes[namespace]
+                    else:
+                        ns = namespace
+                    name = "%s %s" % (ns, localname)
+                else:
+                    name = localname
+                output.append("%s%s=\"%s\"" % (" " * indent, name, value))
+            # self-closing
+            if type == "EmptyTag":
+                indent -= 2
+
+        elif type == "EndTag":
+            indent -= 2
+
+        elif type == "Comment":
+            output.append("%s<!-- %s -->" % (" " * indent, token["data"]))
+
+        elif type == "Doctype":
+            if token["name"]:
+                if token["publicId"]:
+                    output.append("""%s<!DOCTYPE %s "%s" "%s">""" %
+                                  (" " * indent,
+                                   token["name"],
+                                   token["publicId"],
+                                   token["systemId"] if token["systemId"] else ""))
+                elif token["systemId"]:
+                    output.append("""%s<!DOCTYPE %s "" "%s">""" %
+                                  (" " * indent,
+                                   token["name"],
+                                   token["systemId"]))
+                else:
+                    output.append("%s<!DOCTYPE %s>" % (" " * indent,
+                                                       token["name"]))
+            else:
+                output.append("%s<!DOCTYPE >" % (" " * indent,))
+
+        elif type == "Characters":
+            output.append("%s\"%s\"" % (" " * indent, token["data"]))
+
+        elif type == "SpaceCharacters":
+            assert False, "concatenateCharacterTokens should have got rid of all Space tokens"
+
+        else:
+            raise ValueError("Unknown token type, %s" % type)
+
+    return "\n".join(output)
@@ -1,8 +1,8 @@
 from __future__ import absolute_import, division, unicode_literals
 from six import text_type, string_types

-import gettext
-_ = gettext.gettext
+__all__ = ["DOCUMENT", "DOCTYPE", "TEXT", "ELEMENT", "COMMENT", "ENTITY", "UNKNOWN",
+           "TreeWalker", "NonRecursiveTreeWalker"]

 from xml.dom import Node

@@ -58,7 +58,7 @@ class TreeWalker(object):
               "namespace": to_text(namespace),
               "data": attrs}
        if hasChildren:
-            yield self.error(_("Void element has children"))
+            yield self.error("Void element has children")

    def startTag(self, namespace, name, attrs):
        assert namespace is None or isinstance(namespace, string_types), type(namespace)
@@ -122,7 +122,7 @@ class TreeWalker(object):
        return {"type": "Entity", "name": text_type(name)}

    def unknown(self, nodeType):
-        return self.error(_("Unknown node type: ") + nodeType)
+        return self.error("Unknown node type: " + nodeType)


 class NonRecursiveTreeWalker(TreeWalker):
@@ -2,9 +2,6 @@ from __future__ import absolute_import, division, unicode_literals

 from xml.dom import Node

-import gettext
-_ = gettext.gettext
-
 from . import _base


@@ -7,8 +7,6 @@ except ImportError:
        from ordereddict import OrderedDict
    except ImportError:
        OrderedDict = dict
-import gettext
-_ = gettext.gettext

 import re

@@ -4,9 +4,6 @@ from six import text_type
 from lxml import etree
 from ..treebuilders.etree import tag_regexp

-from gettext import gettext
-_ = gettext
-
 from . import _base

 from .. import ihatexml
@@ -130,7 +127,7 @@ class TreeWalker(_base.NonRecursiveTreeWalker):
    def getNodeDetails(self, node):
        if isinstance(node, tuple):  # Text node
            node, key = node
-            assert key in ("text", "tail"), _("Text nodes are text or tail, found %s") % key
+            assert key in ("text", "tail"), "Text nodes are text or tail, found %s" % key
            return _base.TEXT, ensure_str(getattr(node, key))

        elif isinstance(node, Root):
@@ -169,7 +166,7 @@ class TreeWalker(_base.NonRecursiveTreeWalker):
                    attrs, len(node) > 0 or node.text)

    def getFirstChild(self, node):
-        assert not isinstance(node, tuple), _("Text nodes have no children")
+        assert not isinstance(node, tuple), "Text nodes have no children"

        assert len(node) or node.text, "Node has no children"
        if node.text:
@@ -180,7 +177,7 @@ class TreeWalker(_base.NonRecursiveTreeWalker):
    def getNextSibling(self, node):
        if isinstance(node, tuple):  # Text node
            node, key = node
-            assert key in ("text", "tail"), _("Text nodes are text or tail, found %s") % key
+            assert key in ("text", "tail"), "Text nodes are text or tail, found %s" % key
            if key == "text":
                # XXX: we cannot use a "bool(node) and node[0] or None" construct here
                # because node[0] might evaluate to False if it has no child element
@@ -196,7 +193,7 @@ class TreeWalker(_base.NonRecursiveTreeWalker):
    def getParentNode(self, node):
        if isinstance(node, tuple):  # Text node
            node, key = node
-            assert key in ("text", "tail"), _("Text nodes are text or tail, found %s") % key
+            assert key in ("text", "tail"), "Text nodes are text or tail, found %s" % key
            if key == "text":
                return node
            # else: fallback to "normal" processing
@@ -2,6 +2,8 @@ from __future__ import absolute_import, division, unicode_literals

 from types import ModuleType

+from six import text_type
+
 try:
    import xml.etree.cElementTree as default_etree
 except ImportError:
@@ -9,7 +11,26 @@ except ImportError:


 __all__ = ["default_etree", "MethodDispatcher", "isSurrogatePair",
-           "surrogatePairToCodepoint", "moduleFactoryFactory"]
+           "surrogatePairToCodepoint", "moduleFactoryFactory",
+           "supports_lone_surrogates"]
+
+
+# Platforms not supporting lone surrogates (\uD800-\uDFFF) should be
+# caught by the below test. In general this would be any platform
+# using UTF-16 as its encoding of unicode strings, such as
+# Jython. This is because UTF-16 itself is based on the use of such
+# surrogates, and there is no mechanism to further escape such
+# escapes.
+try:
+    _x = eval('"\\uD800"')
+    if not isinstance(_x, text_type):
+        # We need this with u"" because of http://bugs.jython.org/issue2039
+        _x = eval('u"\\uD800"')
+        assert isinstance(_x, text_type)
+except:
+    supports_lone_surrogates = False
+else:
+    supports_lone_surrogates = True


 class MethodDispatcher(dict):
@@ -25,7 +25,7 @@ if __name__=="__main__":
    exclude=['*.pyc','*~','*.xcf','*[0-9].png','*.po','*.pot','*default.mo','*Thumbs.db']
    
    os.chdir('calibre-plugin')
-    files=['plugin-defaults.ini','plugin-example.ini','about.txt',
+    files=['plugin-defaults.ini','plugin-example.ini','about.html',
           'images','translations']
    files.extend(glob('*.py'))
    files.extend(glob('plugin-import-name-*.txt'))
@@ -36,7 +36,7 @@ if __name__=="__main__":

    os.chdir('../included_dependencies')
    # 'a' for append
-    files=['six.py','bs4','html5lib','chardet']
+    files=['gif.py','six.py','bs4','html5lib','chardet']
    createZipFile("../"+filename,"a",
                  files,
                  exclude=exclude)
@@ -10,13 +10,11 @@ https://github.com/pypa/sampleproject
 # Always prefer setuptools over distutils
 from setuptools import setup, find_packages
 # To use a consistent encoding
-from codecs import open
+import codecs
 from os import path

-here = path.abspath(path.dirname(__file__))
-
 # Get the long description from the relevant file
-with open(path.join(here, 'DESCRIPTION.rst'), encoding='utf-8') as f:
+with codecs.open('DESCRIPTION.rst', encoding='utf-8') as f:
    long_description = f.read()

 setup(
@@ -25,7 +23,7 @@ setup(
    # Versions should comply with PEP440.  For a discussion on single-sourcing
    # the version across setup.py and the project code, see
    # https://packaging.python.org/en/latest/single_source_version.html
-    version="2.2.2",
+    version="2.2.13",

    description='A tool for downloading fanfiction to eBook formats',
    long_description=long_description,
@@ -1,6 +1,6 @@
-# ffd-retief-hrd fanfictiondownloader fanficfare
+# ffd-retief-hrd fanficfare
 application: fanficfare
-version: 2-2-2
+version: 2-2-13
 runtime: python27
 api_version: 1
 threadsafe: true
@@ -41,6 +41,8 @@ class DownloadMeta(db.Model):
 	completed =  db.BooleanProperty(default=False)
 	date = db.DateTimeProperty(auto_now_add=True)
 	version = db.StringProperty()
+	ch_begin = db.StringProperty()
+	ch_end = db.StringProperty()
 	# data_chunks is implicit from DownloadData def.

 class DownloadData(db.Model):
@@ -27,38 +27,17 @@
 	      much easier. </p>
 	  </div>

-	  <h3>New Name and URL</h3>
-          <p>
-            This version changes the name of the project from
-            FanFictionDownLoader(FFDL) to FanFicFare.  Along with
-            the name change, there are changes to the internal
-            structuring of the project code, but not to the functionality.
-          </p>
-          <p>
-            If you had saved configuration (including your
-            username/password for fic sites) in FFDL, you will need to
-            copy it over to FanFicFare.  Visit
-            the <a href="http://fanfictiondownloader.appspot.com/editconfig">FFDL
-            User Config</a> page, copy your settings and then paste
-            them on
-            the <a href="http://fanficfare.appspot.com/editconfig">FanFicFare
-            User Config</a> page.
-          </p>
 	  <h3>Changes:</h3>
 	  <ul>
-            <li>Change the name to FanFicFare.</li>
-            <li>Fixes for literotica changes.</li>
-            <li>Fix error in debug output that uses wrong variable.</li>
-            <li>Change date format for onedirectionfanfiction.com.</li>
-            <li>Explicit html parse for BS4 when reading in existing epub.  For cal1.48.</li>
-            <li>Fix for PI - eFiction Base adapter Anthology Series string.</li>
-            <li>Update translations.</li>
-            <li>Change some menu & gui details to be clearer.</li>
-            <li>Fix for ffnet's latest change to story/cover image.</li>
-            <li>Fix problems found during trawl through all sites -- thehookupzone.net and libraryofmoria.com now use eFiction Base.</li>
-            <li>Remove defunct sites - netraptor.org www.simplyundeniable.com www.wizardtales.net www.checkmated.com stargate-atlantis.org www.henneth-annun.net www.twiwrite.net www.dotmoon.net</li>
-            <li>Add option to check ffnet auth img vs cover img and not use author images as book covers.  Defaults to on, add <b>skip_author_cover:false</b> to <b>[fanfiction.net]</b> section if you like author image covers.</li>
-            <li>Add site tgstorytime.com as an eFiction Base Adapter.</li>
+            <li>New site: <a href="http://www.quotev.com/">quotev.com</a>, thanks cryzed!</li>
+            <li>Convert dark-solace.org(Elysium fields) to eFiction Base adapter to correct for site changes.</li>
+            <li>Fix eFiction Base Author URLs.</li>
+            <li>Switch chapter #post-12345 chapter URLs to permalinks for XenForoForum Base adapters.</li>
+            <li>Record forum thread tags in XenForoForum Base adapters. Appears on title page by default, but not calibre tags.</li>
+            <li>Change output_filename_safepattern to replace leading '.' for benefit of *nix OSes.</li>
+            <li>Fix mediaminer.org for site changes.</li>
+            <li>Fix to hpfandom.net change story URL when given chapter URL instead of story start URL.</li>
+            <li>Allow forum.questionablequesting.com for questionablequesting.com.</li>
 	  </ul>
 	  <p>
 	    Questions?  Check out our 
@@ -68,8 +47,8 @@
 	    If you have any problems with this application, please
 	    report them in
 	    the <a href="http://groups.google.com/group/fanfic-downloader">FanFicFare Google Group</a>.  The
-	    <a href="http://fanfictiondownloader.appspot.com">FanFictionDownLoader(FFDL)
-	    Version</a> is also available for you to use if necessary.
+	    <a href="http://2-2-12.fanficfare.appspot.com">previous version
+	    </a> is also available for you to use if necessary.
 	  </p>
 	  <div id='error'>
 	    {{ error_message }}
@@ -130,7 +109,7 @@
 	    Once you have calibre installed and running, inside
 	    calibre, you can go to 'Get plugins to enhance calibre' or
 	    'Get new plugins' and
-	    install <a href="http://www.mobileread.com/forums/showthread.php?t=163261">FanFicFare</a>.
+	    install <a href="http://www.mobileread.com/forums/showthread.php?t=259221">FanFicFare</a>.
 	    
 	  </p>
      </div>
@@ -62,7 +62,7 @@ class UserConfigServer(webapp2.RequestHandler):
    
    def getUserConfig(self,user,url,fileformat):

-        configuration = Configuration(adapters.getConfigSectionFor(url),fileformat)
+        configuration = Configuration(adapters.getConfigSectionsFor(url),fileformat)
        
        logging.debug('reading defaults.ini config file')
        configuration.read('fanficfare/defaults.ini')
@@ -366,6 +366,16 @@ class FanfictionDownloader(UserConfigServer):
            self.redirect('/')
            return

+        # Allow chapter range with URL.
+        # test1.com?sid=5[4-6]
+        mc = re.match(r"^(?P<url>.*?)(?:\[(?P<begin>\d+)?(?P<comma>[,-])?(?P<end>\d+)?\])?$",url)
+        #print("url:(%s) begin:(%s) end:(%s)"%(mc.group('url'),mc.group('begin'),mc.group('end')))
+        url = mc.group('url')
+        ch_begin = mc.group('begin')
+        ch_end = mc.group('end')
+        if ch_begin and not mc.group('comma'):
+            ch_end = ch_begin
+        
        logging.info("Queuing Download: %s" % url)
        login = self.request.get('login')
        password = self.request.get('password')
@@ -385,6 +395,7 @@ class FanfictionDownloader(UserConfigServer):
                return

            adapter = adapters.getAdapter(configuration,url)
+            adapter.setChaptersRange(ch_begin,ch_end)
            logging.info('Created an adaper: %s' % adapter)

            if login or password:
@@ -408,6 +419,8 @@ class FanfictionDownloader(UserConfigServer):
            download.title = story.getMetadata('title')
            download.author = story.getMetadata('author')
            download.url = story.getMetadata('storyUrl')
+            download.ch_begin = ch_begin
+            download.ch_end = ch_end
            download.put()

            taskqueue.add(url='/fdowntask',
@@ -490,6 +503,7 @@ class FanfictionDownloaderTask(UserConfigServer):
        try:
            configuration = self.getUserConfig(user,url,format)
            adapter = adapters.getAdapter(configuration,url)
+            adapter.setChaptersRange(download.ch_begin,download.ch_end)

            logging.info('Created an adapter: %s' % adapter)