Update translations, bump versions.

Add MassEffect2.in Adapter

MANIFEST.in

@@ -0,0 +1 @@
+include DESCRIPTION.rst

README.md

@@ -10,8 +10,10 @@ FFDL, AKA fanficdownloader) project previously hosted as a
 This program is available as a calibre plugin, a command-line
 interface, and a web service.
 
-*Currently, the project is the middle of being moved from code.google
-to GitHub and being renamed.  For now, continue to use
-FanFictionDownLoader(FFDL).*
+FanFicFare has now been launched.  New versions, features and updates
+will all be in FanFicFare.
 
-[code.google project]: https://code.google.com/p/fanficdownloader/
+There's additional info in the project [wiki] pages.
+
+[code.google project]: http://google-opensource.blogspot.com/2015/03/farewell-to-google-code.html
+[wiki]: https://github.com/JimmXinu/FanFicFare/wiki

calibre-plugin/__init__.py

@@ -42,7 +42,7 @@ class FanFicFareBase(InterfaceActionBase):
     description         = _('UI plugin to download FanFiction stories from various sites.')
     supported_platforms = ['windows', 'osx', 'linux']
     author              = 'Jim Miller'
-    version             = (2, 2, 0)
+    version             = (2, 2, 12)
     minimum_calibre_version = (1, 48, 0)
 
     #: This field defines the GUI plugin class that contains all the code

calibre-plugin/about.html

@@ -0,0 +1,32 @@
+<hr />
+
+<p>Plugin created by Jim Miller, borrowing heavily from Grant Drake's
+  '<a href="http://www.mobileread.com/forums/showthread.php?t=134856">Reading List</a>',
+  '<a href="http://www.mobileread.com/forums/showthread.php?t=126727">Extract ISBN</a>' and
+  '<a href="http://www.mobileread.com/forums/showthread.php?t=134000">Count Pages</a>'
+  plugins.</p>
+
+<p>
+  Calibre officially distributes plugins from the mobileread.com forum site.
+  The official distro channel for this plugin is there: <a href="http://www.mobileread.com/forums/showthread.php?t=259221">FanFicFare</a>
+</p>
+
+<p> I also monitor the
+  <a href="http://groups.google.com/group/fanfic-downloader">general users
+    group</a> for the downloader.  That covers the web application and CLI, too.
+</p>
+
+<p>
+  The source for this plugin is available at it's 
+  <a href="https://github.com/JimmXinu/FanFicFare">project home</a>.
+</p>
+
+<hr />
+
+<p>
+  See the <a href="https://github.com/JimmXinu/FanFicFare/wiki/Supportedsites">list of supported sites</a>.
+</p>
+
+<p>
+  Read the <a href="https://github.com/JimmXinu/FanFicFare/wiki/FAQs">FAQs</a>.
+</p>

calibre-plugin/about.txt

@@ -1,28 +0,0 @@
-<hr />
-
-<p>Plugin created by Jim Miller, borrowing heavily from Grant Drake's
-'<a href="http://www.mobileread.com/forums/showthread.php?t=134856">Reading List</a>',
-'<a href="http://www.mobileread.com/forums/showthread.php?t=126727">Extract ISBN</a>' and
-'<a href="http://www.mobileread.com/forums/showthread.php?t=134000">Count Pages</a>'
-plugins.</p>
-
-<p>
-Calibre officially distributes plugins from the mobileread.com forum site.
-The official distro channel for this plugin is there: <a href="http://www.mobileread.com/forums/showthread.php?t=163261">FanFictionDownLoader</a>
-</p>
-
-<p> I also monitor the
-<a href="http://groups.google.com/group/fanfic-downloader">general users
-group</a> for the downloader.  That covers the web application and CLI, too.
-</p>
-
-The source for this plugin is available at it's 
-<a href="https://github.com/JimmXinu/FanFicFare">project home</a>.
-<hr />
-
-<p>
-See the <a href="https://github.com/JimmXinu/FanFicFare/wiki/Supportedsites">list of supported sites</a>.
-</p>
-<p>
-Read the <a href="https://github.com/JimmXinu/FanFicFare/wiki/FAQs">FAQs</a>.
-</p>

calibre-plugin/basicinihighlighter.py

@@ -1,5 +1,5 @@
-#!/usr/bin/env python
-# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+# -*- coding: utf-8 -*-
+
 from __future__ import (unicode_literals, division,
                         print_function)
 

calibre-plugin/common_utils.py

@@ -1,5 +1,5 @@
-#!/usr/bin/env python
-# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+# -*- coding: utf-8 -*-
+
 from __future__ import (unicode_literals, division, absolute_import,
                         print_function)
 

calibre-plugin/config.py

@@ -1,5 +1,5 @@
-#!/usr/bin/env python
-# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+# -*- coding: utf-8 -*-
+
 from __future__ import (unicode_literals, division, absolute_import,
                         print_function)
 
@@ -17,12 +17,12 @@ try:
     from PyQt5.Qt import (QDialog, QWidget, QVBoxLayout, QHBoxLayout, QGridLayout,
                           QLabel, QLineEdit, QFont, QWidget, QTextEdit, QComboBox,
                           QCheckBox, QPushButton, QTabWidget, QScrollArea,
-                          QDialogButtonBox, QGroupBox, Qt )
+                          QDialogButtonBox, QGroupBox, QButtonGroup, QRadioButton, Qt)
 except ImportError as e:
     from PyQt4.Qt import (QDialog, QWidget, QVBoxLayout, QHBoxLayout, QGridLayout,
                           QLabel, QLineEdit, QFont, QWidget, QTextEdit, QComboBox,
                           QCheckBox, QPushButton, QTabWidget, QScrollArea,
-                          QDialogButtonBox, QGroupBox, Qt )
+                          QDialogButtonBox, QGroupBox, QButtonGroup, QRadioButton, Qt)
 try:
     from calibre.gui2 import QVariant
     del QVariant
@@ -39,9 +39,15 @@ else:
             return [convert_qvariant(i) for i in x.toList()]
         return x.toPyObject()
 
-from calibre.gui2.ui import get_gui
 from calibre.gui2 import dynamic, info_dialog, question_dialog
-from calibre.constants import numeric_version as calibre_version
+from calibre.gui2.ui import get_gui
+from calibre.gui2.complete2 import EditWithComplete
+
+try:
+    from calibre.ebooks.covers import generate_cover as cal_generate_cover
+    HAS_CALGC=True
+except:
+    HAS_CALGC=False
 
 # pulls in translation files for _() strings
 try:
@@ -49,12 +55,14 @@ try:
 except NameError:
     pass # load_translations() added in calibre 1.9
 
+from calibre.library.field_metadata import FieldMetadata
+field_metadata = FieldMetadata()
+
 # There are a number of things used several times that shouldn't be
 # translated.  This is just a way to make that easier by keeping them
 # out of the _() strings.
 # I'm tempted to override _() to include them...
 no_trans = { 'pini':'personal.ini',
-             'imgset':'\n\n[epub]\ninclude_images:true\nkeep_summary_html:true\nmake_firstimage_cover:true\n\n',
              'gcset':'generate_cover_settings',
              'ccset':'custom_columns_settings',
              'gc':'Generate Cover',
@@ -70,13 +78,18 @@ no_trans = { 'pini':'personal.ini',
              'p':'password',
              }
 
-from calibre_plugins.fanficfare_plugin.prefs import prefs, PREFS_NAMESPACE
+STD_COLS_SKIP = ['size','cover','news','ondevice','path','series_sort','sort']
+
+from calibre_plugins.fanficfare_plugin.prefs \
+    import (prefs, PREFS_NAMESPACE, updatecalcover_order, calcover_save_options,
+            gencalcover_order, SAVE_YES, SAVE_NO)
+
 from calibre_plugins.fanficfare_plugin.dialogs \
     import (UPDATE, UPDATEALWAYS, collision_order, save_collisions, RejectListDialog,
             EditTextDialog, IniTextDialog, RejectUrlEntry)
     
 from calibre_plugins.fanficfare_plugin.fanficfare.adapters \
-    import getConfigSections
+    import getSiteSections
 
 from calibre_plugins.fanficfare_plugin.common_utils \
     import ( KeyboardConfigDialog, PrefsViewerDialog )
@@ -84,8 +97,6 @@ from calibre_plugins.fanficfare_plugin.common_utils \
 from calibre_plugins.fanficfare_plugin.fff_util \
     import (test_config)
 
-from calibre.gui2.complete2 import EditWithComplete  #MultiCompleteLineEdit
-
 class RejectURLList:
     def __init__(self,prefs):
         self.prefs = prefs
@@ -205,10 +216,8 @@ class ConfigWidget(QWidget):
         if 'Reading List' not in plugin_action.gui.iactions:
             self.readinglist_tab.setEnabled(False)
 
-        self.generatecover_tab = GenerateCoverTab(self, plugin_action)
-        tab_widget.addTab(self.generatecover_tab, 'Generate Cover')
-        if 'Generate Cover' not in plugin_action.gui.iactions:
-            self.generatecover_tab.setEnabled(False)
+        self.calibrecover_tab = CalibreCoverTab(self, plugin_action)
+        tab_widget.addTab(self.calibrecover_tab, _('Calibre Cover'))
 
         self.countpages_tab = CountPagesTab(self, plugin_action)
         tab_widget.addTab(self.countpages_tab, 'Count Pages')
@@ -234,7 +243,6 @@ class ConfigWidget(QWidget):
         prefs['fileform'] = unicode(self.basic_tab.fileform.currentText())
         prefs['collision'] = save_collisions[unicode(self.basic_tab.collision.currentText())]
         prefs['updatemeta'] = self.basic_tab.updatemeta.isChecked()
-        prefs['updatecover'] = self.basic_tab.updatecover.isChecked()
         prefs['updateepubcover'] = self.basic_tab.updateepubcover.isChecked()
         prefs['keeptags'] = self.basic_tab.keeptags.isChecked()
         prefs['suppressauthorsort'] = self.basic_tab.suppressauthorsort.isChecked()
@@ -247,11 +255,11 @@ class ConfigWidget(QWidget):
         prefs['updatedefault'] = self.basic_tab.updatedefault.isChecked()
         prefs['deleteotherforms'] = self.basic_tab.deleteotherforms.isChecked()
         prefs['adddialogstaysontop'] = self.basic_tab.adddialogstaysontop.isChecked()
-        prefs['includeimages'] = self.basic_tab.includeimages.isChecked()
         prefs['lookforurlinhtml'] = self.basic_tab.lookforurlinhtml.isChecked()
         prefs['checkforseriesurlid'] = self.basic_tab.checkforseriesurlid.isChecked()
         prefs['checkforurlchange'] = self.basic_tab.checkforurlchange.isChecked()
         prefs['injectseries'] = self.basic_tab.injectseries.isChecked()
+        prefs['matchtitleauth'] = self.basic_tab.matchtitleauth.isChecked()
         prefs['smarten_punctuation'] = self.basic_tab.smarten_punctuation.isChecked()
         prefs['reject_always'] = self.basic_tab.reject_always.isChecked()
 
@@ -264,6 +272,7 @@ class ConfigWidget(QWidget):
             prefs['addtolists'] = self.readinglist_tab.addtolists.isChecked()
             prefs['addtoreadlists'] = self.readinglist_tab.addtoreadlists.isChecked()
             prefs['addtolistsonread'] = self.readinglist_tab.addtolistsonread.isChecked()
+            prefs['autounnew'] = self.readinglist_tab.autounnew.isChecked()
 
         # personal.ini
         ini = self.personalini_tab.personalini
@@ -273,17 +282,25 @@ class ConfigWidget(QWidget):
             # if they've removed everything, reset to default.
             prefs['personal.ini'] = get_resources('plugin-example.ini')
 
-        # Generate Covers tab
-        prefs['gcnewonly'] = self.generatecover_tab.gcnewonly.isChecked()
+        prefs['cal_cols_pass_in'] = self.personalini_tab.cal_cols_pass_in.isChecked()
+            
+        # Covers tab
+        prefs['updatecalcover'] = calcover_save_options[unicode(self.calibrecover_tab.updatecalcover.currentText())]
+        # for backward compatibility:
+        prefs['updatecover'] = prefs['updatecalcover'] == SAVE_YES
+        prefs['gencalcover'] = calcover_save_options[unicode(self.calibrecover_tab.gencalcover.currentText())]
+        prefs['calibre_gen_cover'] = self.calibrecover_tab.calibre_gen_cover.isChecked()
+        prefs['plugin_gen_cover'] = self.calibrecover_tab.plugin_gen_cover.isChecked()
+        prefs['gcnewonly'] = self.calibrecover_tab.gcnewonly.isChecked()
         gc_site_settings = {}
-        for (site,combo) in self.generatecover_tab.gc_dropdowns.iteritems():
+        for (site,combo) in self.calibrecover_tab.gc_dropdowns.iteritems():
             val = unicode(convert_qvariant(combo.itemData(combo.currentIndex())))
             if val != 'none':
                 gc_site_settings[site] = val
                 #print("gc_site_settings[%s]:%s"%(site,gc_site_settings[site]))
         prefs['gc_site_settings'] = gc_site_settings
-        prefs['allow_gc_from_ini'] = self.generatecover_tab.allow_gc_from_ini.isChecked()
-        prefs['gc_polish_cover'] = self.generatecover_tab.gc_polish_cover.isChecked()
+        prefs['allow_gc_from_ini'] = self.calibrecover_tab.allow_gc_from_ini.isChecked()
+        prefs['gc_polish_cover'] = self.calibrecover_tab.gc_polish_cover.isChecked()
 
         # Count Pages tab
         countpagesstats = []
@@ -308,10 +325,15 @@ class ConfigWidget(QWidget):
             colsnewonly[col] = checkbox.isChecked()
         prefs['std_cols_newonly'] = colsnewonly
 
+        prefs['set_author_url'] =self.std_columns_tab.set_author_url.isChecked()
+
         # Custom Columns tab
         # error column
         prefs['errorcol'] = unicode(convert_qvariant(self.cust_columns_tab.errorcol.itemData(self.cust_columns_tab.errorcol.currentIndex())))
 
+        # metadata column
+        prefs['savemetacol'] = unicode(convert_qvariant(self.cust_columns_tab.savemetacol.itemData(self.cust_columns_tab.savemetacol.currentIndex())))
+
         # cust cols tab
         colsmap = {}
         for (col,combo) in self.cust_columns_tab.custcol_dropdowns.iteritems():
@@ -328,12 +350,14 @@ class ConfigWidget(QWidget):
         
         prefs['allow_custcol_from_ini'] = self.cust_columns_tab.allow_custcol_from_ini.isChecked()
 
-        prefs['imapserver'] = unicode(self.imap_tab.imapserver.text())
-        prefs['imapuser'] = unicode(self.imap_tab.imapuser.text())
-        prefs['imappass'] = unicode(self.imap_tab.imappass.text())
-        prefs['imapfolder'] = unicode(self.imap_tab.imapfolder.text())
+        prefs['imapserver'] = unicode(self.imap_tab.imapserver.text()).strip()
+        prefs['imapuser'] = unicode(self.imap_tab.imapuser.text()).strip()
+        prefs['imappass'] = unicode(self.imap_tab.imappass.text()).strip()
+        prefs['imapfolder'] = unicode(self.imap_tab.imapfolder.text()).strip()
         prefs['imapmarkread'] = self.imap_tab.imapmarkread.isChecked()
         prefs['imapsessionpass'] = self.imap_tab.imapsessionpass.isChecked()
+        prefs['auto_reject_from_email'] = self.imap_tab.auto_reject_from_email.isChecked()
+        prefs['download_from_email_immediately'] = self.imap_tab.download_from_email_immediately.isChecked()
         
         prefs.save_to_db()
 
@@ -396,21 +420,17 @@ class BasicTab(QWidget):
         horz.addWidget(self.collision)
         self.l.addLayout(horz)
 
+        horz = QHBoxLayout()
         self.updatemeta = QCheckBox(_('Default Update Calibre &Metadata?'),self)
         self.updatemeta.setToolTip(_("On each download, FanFicFare offers an option to update Calibre's metadata (title, author, URL, tags, custom columns, etc) from the web site. <br />This sets whether that will default to on or off. <br />Columns set to 'New Only' in the column tabs will only be set for new books."))
         self.updatemeta.setChecked(prefs['updatemeta'])
-        self.l.addWidget(self.updatemeta)
+        horz.addWidget(self.updatemeta)
 
         self.updateepubcover = QCheckBox(_('Default Update EPUB Cover when Updating EPUB?'),self)
         self.updateepubcover.setToolTip(_("On each download, FanFicFare offers an option to update the book cover image <i>inside</i> the EPUB from the web site when the EPUB is updated.<br />This sets whether that will default to on or off."))
         self.updateepubcover.setChecked(prefs['updateepubcover'])
-        self.l.addWidget(self.updateepubcover)
-
-        self.smarten_punctuation = QCheckBox(_('Smarten Punctuation (EPUB only)'),self)
-        self.smarten_punctuation.setToolTip(_("Run Smarten Punctuation from Calibre's Polish Book feature on each EPUB download and update."))
-        self.smarten_punctuation.setChecked(prefs['smarten_punctuation'])
-        if calibre_version >= (0, 9, 39):
-            self.l.addWidget(self.smarten_punctuation)
+        horz.addWidget(self.updateepubcover)
+        self.l.addLayout(horz)
 
         cali_gb = groupbox = QGroupBox(_("Updating Calibre Options"))
         self.l = QVBoxLayout()
@@ -421,11 +441,6 @@ class BasicTab(QWidget):
         self.deleteotherforms.setChecked(prefs['deleteotherforms'])
         self.l.addWidget(self.deleteotherforms)
         
-        self.updatecover = QCheckBox(_('Update Calibre Cover when Updating Metadata?'),self)
-        self.updatecover.setToolTip(_("Update calibre book cover image from EPUB when metadata is updated.  (EPUB only.)\nDoesn't go looking for new images on 'Update Calibre Metadata Only'."))
-        self.updatecover.setChecked(prefs['updatecover'])
-        self.l.addWidget(self.updatecover)
-
         self.keeptags = QCheckBox(_('Keep Existing Tags when Updating Metadata?'),self)
         self.keeptags.setToolTip(_("Existing tags will be kept and any new tags added.\n%(cmplt)s and %(inprog)s tags will be still be updated, if known.\n%(lul)s tags will be updated if %(lus)s in %(is)s.\n(If Tags is set to 'New Only' in the Standard Columns tab, this has no effect.)")%no_trans)
         self.keeptags.setChecked(prefs['keeptags'])
@@ -466,16 +481,16 @@ class BasicTab(QWidget):
         self.showmarked.setChecked(prefs['showmarked'])
         self.l.addWidget(self.showmarked)
 
+        self.smarten_punctuation = QCheckBox(_('Smarten Punctuation (EPUB only)'),self)
+        self.smarten_punctuation.setToolTip(_("Run Smarten Punctuation from Calibre's Polish Book feature on each EPUB download and update."))
+        self.smarten_punctuation.setChecked(prefs['smarten_punctuation'])
+        self.l.addWidget(self.smarten_punctuation)
+
         self.autoconvert = QCheckBox(_("Automatically Convert new/update books?"),self)
         self.autoconvert.setToolTip(_("Automatically call calibre's Convert for new/update books.\nConverts to the current output format as chosen in calibre's\nPreferences->Behavior settings."))
         self.autoconvert.setChecked(prefs['autoconvert'])
         self.l.addWidget(self.autoconvert)
 
-        self.show_est_time = QCheckBox(_("Show estimated time left?"),self)
-        self.show_est_time.setToolTip(_("When a Progress Bar is shown, show a rough estimate of the time left."))
-        self.show_est_time.setChecked(prefs['show_est_time'])
-        self.l.addWidget(self.show_est_time)
-
         gui_gb = groupbox = QGroupBox(_("GUI Options"))
         self.l = QVBoxLayout()
         groupbox.setLayout(self.l)
@@ -495,21 +510,25 @@ class BasicTab(QWidget):
         self.adddialogstaysontop.setChecked(prefs['adddialogstaysontop'])
         self.l.addWidget(self.adddialogstaysontop)
 
+        self.show_est_time = QCheckBox(_("Show estimated time left?"),self)
+        self.show_est_time.setToolTip(_("When a Progress Bar is shown, show a rough estimate of the time left."))
+        self.show_est_time.setChecked(prefs['show_est_time'])
+        self.l.addWidget(self.show_est_time)
+
         misc_gb = groupbox = QGroupBox(_("Misc Options"))
         self.l = QVBoxLayout()
         groupbox.setLayout(self.l)
 
-        # this is a cheat to make it easier for users to realize there's a new include_images features.
-        self.includeimages = QCheckBox(_("Include images in EPUBs?"),self)
-        self.includeimages.setToolTip(_("Download and include images in EPUB stories.  This is equivalent to adding:%(imgset)s ...to the top of %(pini)s.  Your settings in %(pini)s will override this.")%no_trans)
-        self.includeimages.setChecked(prefs['includeimages'])
-        self.l.addWidget(self.includeimages)
-
         self.injectseries = QCheckBox(_("Inject calibre Series when none found?"),self)
         self.injectseries.setToolTip(_("If no series is found, inject the calibre series (if there is one) so \nit appears on the FanFicFare title page(not cover)."))
         self.injectseries.setChecked(prefs['injectseries'])
         self.l.addWidget(self.injectseries)
 
+        self.matchtitleauth = QCheckBox(_("Search by Title/Author(s) for If Story Already Exists?"),self)
+        self.matchtitleauth.setToolTip(_("When checking <i>If Story Already Exists</i> FanFicFare will first match by URL Identifier.  But if not found, it can also search existing books by Title and Author(s)."))
+        self.matchtitleauth.setChecked(prefs['matchtitleauth'])
+        self.l.addWidget(self.matchtitleauth)
+
         rej_gb = groupbox = QGroupBox(_("Reject List"))
         self.l = QVBoxLayout()
         groupbox.setLayout(self.l)
@@ -639,15 +658,25 @@ class PersonalIniTab(QWidget):
         self.ini_button.clicked.connect(self.add_ini_button)
         self.l.addWidget(self.ini_button)
         
+        label = QLabel(_("Changes will only be saved if you click 'OK' to leave Customize FanFicFare."))
+        label.setWordWrap(True)
+        self.l.addWidget(label)
+        
         self.defaults = QPushButton(_('View Defaults')+' (plugin-defaults.ini)', self)
         self.defaults.setToolTip(_("View all of the plugin's configurable settings\nand their default settings."))
         self.defaults.clicked.connect(self.show_defaults)
         self.l.addWidget(self.defaults)
 
-        label = QLabel(_("Changes will only be saved if you click 'OK' to leave Customize FanFicFare."))
-        label.setWordWrap(True)
-        self.l.addWidget(label)
+        self.cal_cols_pass_in = QCheckBox(_('Pass Calibre Columns into FanFicFare on Update/Overwrite')%no_trans,self)
+        self.cal_cols_pass_in.setToolTip(_("If checked, when updating/overwriting an existing book, FanFicFare will have the Calibre Columns available to use in replace_metadata, title_page, etc.<br>Click the button below to see the Calibre Column namess.")%no_trans)
+        self.cal_cols_pass_in.setChecked(prefs['cal_cols_pass_in'])
+        self.l.addWidget(self.cal_cols_pass_in)
         
+        self.showcalcols = QPushButton(_('Show Calibre Column Names'), self)
+        self.showcalcols.setToolTip(_("FanFicFare can pass the Calibre Columns into the download/update process.<br>This will show you the columns available by name."))
+        self.showcalcols.clicked.connect(self.show_showcalcols)
+        self.l.addWidget(self.showcalcols)
+
         self.l.insertStretch(-1)
         # let edit box fill the space.
         
@@ -673,6 +702,27 @@ class PersonalIniTab(QWidget):
         if d.result() == d.Accepted:
             self.personalini = d.get_plain_text()
                 
+    def show_showcalcols(self):
+        lines=[]#[('calibre_std_user_categories',_('User Categories'))]
+        for k,f in field_metadata.iteritems():
+            if f['name'] and k not in STD_COLS_SKIP: # only if it has a human readable name.
+                lines.append(('calibre_std_'+k,f['name']))
+            
+        for k, column in self.plugin_action.gui.library_view.model().custom_columns.iteritems():
+            if k != prefs['savemetacol']:
+                # custom always have name.
+                lines.append(('calibre_cust_'+k[1:],column['name']))
+
+        lines.sort() # sort by key.
+
+        EditTextDialog(self,
+                       '\n'.join(['%s (%s)'%(l,k) for (k,l) in lines]),
+                       icon=self.windowIcon(),
+                       title=_('Calibre Column Entry Names'),
+                       label=_('Label (entry_name)'),
+                       read_only=True,
+                       save_size_name='fff:showcalcols').exec_()
+        
 class ReadingListTab(QWidget):
 
     def __init__(self, parent_dialog, plugin_action):
@@ -708,6 +758,7 @@ class ReadingListTab(QWidget):
         self.send_lists_box.update_items_cache(reading_lists)
         self.send_lists_box.setText(prefs['send_lists'])
         horz.addWidget(self.send_lists_box)
+        self.send_lists_box.setCursorPosition(0)
         self.l.addLayout(horz)
         
         self.addtoreadlists = QCheckBox(_('Add new/updated stories to "To Read" Reading List(s).'),self)
@@ -724,6 +775,7 @@ class ReadingListTab(QWidget):
         self.read_lists_box.update_items_cache(reading_lists)
         self.read_lists_box.setText(prefs['read_lists'])
         horz.addWidget(self.read_lists_box)
+        self.read_lists_box.setCursorPosition(0)
         self.l.addLayout(horz)
         
         self.addtolistsonread = QCheckBox(_('Add stories back to "Send to Device" Reading List(s) when marked "Read".'),self)
@@ -731,16 +783,26 @@ class ReadingListTab(QWidget):
         self.addtolistsonread.setChecked(prefs['addtolistsonread'])
         self.l.addWidget(self.addtolistsonread)
             
+        self.autounnew = QCheckBox(_('Automatically run Remove "New" Chapter Marks when marking books "Read".'),self)
+        self.autounnew.setToolTip(_('Menu option to remove from "To Read" lists will also remove "(new)" chapter marks created by personal.ini <i>mark_new_chapters</i> setting.'))
+        self.autounnew.setChecked(prefs['autounnew'])
+        self.l.addWidget(self.autounnew)
+            
         self.l.insertStretch(-1)
         
-class GenerateCoverTab(QWidget):
+class CalibreCoverTab(QWidget):
 
     def __init__(self, parent_dialog, plugin_action):
         self.parent_dialog = parent_dialog
         self.plugin_action = plugin_action
         QWidget.__init__(self)
         
-        self.l = QVBoxLayout()
+        self.gencov_elements=[] ## used to disable/enable when gen
+                                ## cover is off/on.  This is more
+                                ## about being a visual que than real
+                                ## necessary function.
+        
+        topl = self.l = QVBoxLayout()
         self.setLayout(self.l)
 
         try:
@@ -748,8 +810,116 @@ class GenerateCoverTab(QWidget):
             gc_settings = gc_plugin.get_saved_setting_names()
         except KeyError:
             gc_settings= []
+
             
-        label = QLabel(_('The %(gc)s plugin can create cover images for books using various metadata and configurations.  If you have GC installed, FanFicFare can run GC on new downloads and metadata updates.  Pick a GC setting by site or Default.')%no_trans)
+        label = QLabel(_("The Calibre cover image for a downloaded book can come"
+                         " from the story site(if EPUB and images are enabled), or"
+                         " from either Calibre's built-in random cover generator or"
+                         " the %(gc)s plugin.")%no_trans)
+        label.setWordWrap(True)
+        self.l.addWidget(label)
+        self.l.addSpacing(5)
+        
+        tooltip = _("Update Calibre book cover image from EPUB when Calibre metadata is updated.\n"
+                    "Doesn't go looking for new images on 'Update Calibre Metadata Only'.\n"
+                    "Cover in EPUB could be from site or previously injected into the EPUB.\n"
+                    "This comes before Generate Cover so %(gc)s(Plugin) use the image if configured to.")%no_trans
+        horz = QHBoxLayout()
+        label = QLabel(_('Update Calibre Cover (from EPUB):'))
+        label.setToolTip(tooltip)
+        horz.addWidget(label)
+        self.updatecalcover = QComboBox(self)
+        for i in updatecalcover_order:
+            self.updatecalcover.addItem(i)
+        # back compat.  If has own value, use.
+        if prefs['updatecalcover']:
+            self.updatecalcover.setCurrentIndex(self.updatecalcover.findText(calcover_save_options[prefs['updatecalcover']]))
+        elif prefs['updatecover']: # doesn't have own val, set YES if old value set.
+            self.updatecalcover.setCurrentIndex(self.updatecalcover.findText(calcover_save_options[SAVE_YES]))
+        else: # doesn't have own value, old value not set, NO.
+            self.updatecalcover.setCurrentIndex(self.updatecalcover.findText(calcover_save_options[SAVE_NO]))
+        self.updatecalcover.setToolTip(tooltip)
+        label.setBuddy(self.updatecalcover)
+        horz.addWidget(self.updatecalcover)
+        self.l.addLayout(horz)
+
+        tooltip = _("Generate a Calibre book cover image when Calibre metadata is updated.<br />"
+                    "Defaults to 'Yes, Always' for backward compatibility and because %(gc)s(Plugin)"
+                    " will only run if configured for Default or site.")%no_trans
+        horz = QHBoxLayout()
+        label = QLabel(_('Generate Calibre Cover:'))
+        label.setToolTip(tooltip)
+        horz.addWidget(label)
+        self.gencalcover = QComboBox(self)
+        for i in gencalcover_order:
+            self.gencalcover.addItem(i)
+        # back compat.  If has own value, use.
+        # if prefs['gencalcover']:
+        self.gencalcover.setCurrentIndex(self.gencalcover.findText(calcover_save_options[prefs['gencalcover']]))
+        # elif prefs['gencover']: # doesn't have own val, set YES if old value set.
+        #     self.gencalcover.setCurrentIndex(self.gencalcover.findText(calcover_save_options[SAVE_YES]))
+        # else: # doesn't have own value, old value not set, NO.
+        #     self.gencalcover.setCurrentIndex(self.gencalcover.findText(calcover_save_options[SAVE_NO]))
+            
+        self.gencalcover.setToolTip(tooltip)
+        label.setBuddy(self.gencalcover)
+        horz.addWidget(self.gencalcover)
+        self.l.addLayout(horz)
+        self.gencalcover.currentIndexChanged.connect(self.endisable_elements)
+
+        # can't be local or it's destroyed when __init__ is done and
+        # connected things don't fire.
+        self.gencov_rdgrp = QButtonGroup()
+        self.gencov_gb = QGroupBox()
+        horz = QHBoxLayout()
+        self.gencov_gb.setLayout(horz)
+        
+        self.plugin_gen_cover = QRadioButton(_('Plugin %(gc)s')%no_trans,self)
+        self.plugin_gen_cover.setToolTip(_("Use plugin to create covers.  Additional settings are below."))
+        self.gencov_rdgrp.addButton(self.plugin_gen_cover)
+        # always, new only, when no cover from site, inject yes/no...
+        self.plugin_gen_cover.setChecked(prefs['plugin_gen_cover'])
+        horz.addWidget(self.plugin_gen_cover)
+        self.gencov_elements.append(self.plugin_gen_cover)
+
+        self.calibre_gen_cover = QRadioButton(_('Calibre Generate Cover'),self)
+        self.calibre_gen_cover.setToolTip(_("Call Calibre's Edit Metadata Generate cover"
+                                            " feature to create a random cover each time"
+                                            " a story is downloaded or updated.<br />"
+                                            "Right click or long click the 'Generate cover'"
+                                            " button in Calibre's Edit Metadata to customize."))
+        self.gencov_rdgrp.addButton(self.calibre_gen_cover)
+        # always, new only, when no cover from site, inject yes/no...
+        self.calibre_gen_cover.setChecked(prefs['calibre_gen_cover'])
+        horz.addWidget(self.calibre_gen_cover)
+        self.gencov_elements.append(self.calibre_gen_cover)
+
+        #self.l.addLayout(horz)
+        self.l.addWidget(self.gencov_gb)
+
+        self.gcnewonly = QCheckBox(_("Generate Covers Only for New Books")%no_trans,self)
+        self.gcnewonly.setToolTip(_("Default is to generate a cover any time the calibre metadata is"
+                                    " updated.<br \>Used for both Calibre and Plugin generated covers."))
+        self.gcnewonly.setChecked(prefs['gcnewonly'])
+        self.l.addWidget(self.gcnewonly)
+        self.gencov_elements.append(self.gcnewonly)
+
+        self.gc_polish_cover = QCheckBox(_("Inject/update the cover inside EPUB"),self)
+        self.gc_polish_cover.setToolTip(_("Calibre's Polish feature will be used to inject or update the generated"
+                                          " cover into the EPUB ebook file.<br \>Used for both Calibre and Plugin generated covers."))
+        self.gc_polish_cover.setChecked(prefs['gc_polish_cover'])
+        self.l.addWidget(self.gc_polish_cover)
+        self.gencov_elements.append(self.gc_polish_cover)
+
+        self.gcp_gb = QGroupBox(_("%(gc)s(Plugin) Settings")%no_trans)
+        topl.addWidget(self.gcp_gb)
+        self.l = QVBoxLayout()
+        self.gcp_gb.setLayout(self.l)
+        self.gencov_elements.append(self.gcp_gb)
+
+        self.gencov_rdgrp.buttonClicked.connect(self.endisable_elements)
+            
+        label = QLabel(_('The %(gc)s plugin can create cover images for books using various metadata (including existing cover image).  If you have %(gc)s installed, FanFicFare can run %(gc)s on new downloads and metadata updates.  Pick a %(gc)s setting by site and/or one to use by Default.')%no_trans)
         label.setWordWrap(True)
         self.l.addWidget(label)
         self.l.addSpacing(5)
@@ -765,14 +935,14 @@ class GenerateCoverTab(QWidget):
         
         self.gc_dropdowns = {}
 
-        sitelist = getConfigSections()
+        sitelist = getSiteSections()
         sitelist.sort()
         sitelist.insert(0,_("Default"))
         for site in sitelist:
             horz = QHBoxLayout()
             label = QLabel(site)
             if site == _("Default"):
-                s = _("On Metadata update, run %(gc)s with this setting, if not selected for specific site.")%no_trans
+                s = _("On Metadata update, run %(gc)s with this setting, if there isn't a more specific setting below.")%no_trans
             else:
                 no_trans['site']=site # not ideal, but, meh.
                 s = _("On Metadata update, run %(gc)s with this setting for %(site)s stories.")%no_trans
@@ -798,20 +968,34 @@ class GenerateCoverTab(QWidget):
         
         self.sl.insertStretch(-1)
         
-        self.gcnewonly = QCheckBox(_("Run %(gc)s Only on New Books")%no_trans,self)
-        self.gcnewonly.setToolTip(_("Default is to run GC any time the calibre metadata is updated."))
-        self.gcnewonly.setChecked(prefs['gcnewonly'])
-        self.l.addWidget(self.gcnewonly)
-
         self.allow_gc_from_ini = QCheckBox(_('Allow %(gcset)s from %(pini)s to override')%no_trans,self)
-        self.allow_gc_from_ini.setToolTip(_("The %(pini)s parameter %(gcset)s allows you to choose a GC setting based on metadata rather than site, but it's much more complex.<br \>%(gcset)s is ignored when this is off.")%no_trans)
+        self.allow_gc_from_ini.setToolTip(_("The %(pini)s parameter %(gcset)s allows you to choose a %(gc)s setting based on metadata"
+                                            " rather than site, but it's much more complex.<br \>%(gcset)s is ignored when this is off.")%no_trans)
         self.allow_gc_from_ini.setChecked(prefs['allow_gc_from_ini'])
         self.l.addWidget(self.allow_gc_from_ini)
-            
-        self.gc_polish_cover = QCheckBox(_("Use calibre's Polish feature to inject/update the cover"),self)
-        self.gc_polish_cover.setToolTip(_("Calibre's Polish feature will be used to inject or update the generated cover into the ebook, EPUB only."))
-        self.gc_polish_cover.setChecked(prefs['gc_polish_cover'])
-        self.l.addWidget(self.gc_polish_cover)
+
+        # keep at end.
+        self.endisable_elements()
+
+    def endisable_elements(self,button=None):
+        "Clearing house function for setting elements of Calibre"
+        "Cover tab enabled/disabled depending on all factors."
+
+        ## First, cover gen on/off 
+        for e in self.gencov_elements:
+            e.setEnabled(calcover_save_options[unicode(self.gencalcover.currentText())] != SAVE_NO)
+
+        # next, disable plugin settings when using calibre gen cov.
+        if not self.plugin_gen_cover.isChecked():
+            self.gcp_gb.setEnabled(False)
+
+        # disable (but not enable) unsupported options.
+        if not HAS_CALGC:
+            self.calibre_gen_cover.setEnabled(False)
+        if not 'Generate Cover' in self.plugin_action.gui.iactions:
+            self.plugin_gen_cover.setEnabled(False)
+            self.gcp_gb.setEnabled(False)
+        
             
 class CountPagesTab(QWidget):
 
@@ -1061,7 +1245,6 @@ class CustomColumnsTab(QWidget):
         self.allow_custcol_from_ini.setChecked(prefs['allow_custcol_from_ini'])
         self.l.addWidget(self.allow_custcol_from_ini)
         
-        self.l.addSpacing(5)
         label = QLabel(_("Special column:"))
         label.setWordWrap(True)
         self.l.addWidget(label)
@@ -1080,6 +1263,21 @@ class CustomColumnsTab(QWidget):
         self.errorcol.setCurrentIndex(self.errorcol.findData(prefs['errorcol']))
         horz.addWidget(self.errorcol)
         self.l.addLayout(horz)
+        
+        horz = QHBoxLayout()
+        label = QLabel(_("Saved Metadata Column:"))
+        tooltip=_("If set, FanFicFare will save a copy of all its metadata in this column when the book is downloaded or updated.<br/>The metadata from this column can later be used to update custom columns without having to request the metadata from the server again.<br/>(Long Text columns only.)")
+        label.setToolTip(tooltip)
+        horz.addWidget(label)
+        self.savemetacol = QComboBox(self)
+        self.savemetacol.setToolTip(tooltip)
+        self.savemetacol.addItem('','')
+        for key, column in custom_columns.iteritems():
+            if column['datatype'] in ('comments'):
+                self.savemetacol.addItem(column['name'],key)
+        self.savemetacol.setCurrentIndex(self.savemetacol.findData(prefs['savemetacol']))
+        horz.addWidget(self.savemetacol)
+        self.l.addLayout(horz)
 
         #print("prefs['custom_cols'] %s"%prefs['custom_cols'])
 
@@ -1128,6 +1326,17 @@ class StandardColumnsTab(QWidget):
             horz.addWidget(newonlycheck)
             
             self.l.addLayout(horz)
+            
+        self.l.addSpacing(5)
+        label = QLabel(_("Other Standard Column Options"))
+        label.setWordWrap(True)
+        self.l.addWidget(label)
+        self.l.addSpacing(5)
+
+        self.set_author_url = QCheckBox(_('Set Calibre Author URL'),self)
+        self.set_author_url.setToolTip(_("Set Calibre Author URL to Author's URL on story site."))
+        self.set_author_url.setChecked(prefs['set_author_url'])
+        self.l.addWidget(self.set_author_url)
         
         self.l.insertStretch(-1)
 
@@ -1200,6 +1409,18 @@ class ImapTab(QWidget):
         self.l.addWidget(self.imapmarkread,row,0,1,-1)
         row+=1
             
+        self.auto_reject_from_email = QCheckBox(_('Discard URLs on Reject List'),self)
+        self.auto_reject_from_email.setToolTip(_('If checked, FanFicFare will silently discard story URLs from emails that are on your Reject URL List.<br>Otherwise they will appear and you will see the normal Reject URL dialog.<br>The Emails will still be marked Read if configured to.'))
+        self.auto_reject_from_email.setChecked(prefs['auto_reject_from_email'])
+        self.l.addWidget(self.auto_reject_from_email,row,0,1,-1)
+        row+=1
+        
+        self.download_from_email_immediately = QCheckBox(_('Download from Email Immediately'),self)
+        self.download_from_email_immediately.setToolTip(_('If checked, FanFicFare will start downloading story URLs from emails immediately.<br>Otherwise the usual Download from URLs dialog will appear.'))
+        self.download_from_email_immediately.setChecked(prefs['download_from_email_immediately'])
+        self.l.addWidget(self.download_from_email_immediately,row,0,1,-1)
+        row+=1
+        
         label = QLabel(_("<b>It's safest if you create a separate email account that you use only "
                          "for your story update notices.  FanFicFare and calibre cannot guarantee that "
                          "malicious code cannot get your email password once you've entered it. "

calibre-plugin/dialogs.py

@@ -1,5 +1,5 @@
-#!/usr/bin/env python
-# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+# -*- coding: utf-8 -*-
+
 from __future__ import (unicode_literals, division,
                         print_function)
 
@@ -79,14 +79,16 @@ UPDATE=_('Update EPUB if New Chapters')
 UPDATEALWAYS=_('Update EPUB Always')
 OVERWRITE=_('Overwrite if Newer')
 OVERWRITEALWAYS=_('Overwrite Always')
-CALIBREONLY=_('Update Calibre Metadata Only')
+CALIBREONLY=_('Update Calibre Metadata from Web Site')
+CALIBREONLYSAVECOL=_('Update Calibre Metadata from Saved Metadata Column')
 collision_order=[SKIP,
                  ADDNEW,
                  UPDATE,
                  UPDATEALWAYS,
                  OVERWRITE,
                  OVERWRITEALWAYS,
-                 CALIBREONLY,]
+                 CALIBREONLY,
+                 CALIBREONLYSAVECOL,]
 
 # best idea I've had for how to deal with config/pref saving the
 # collision name in english.
@@ -97,6 +99,7 @@ SAVE_UPDATEALWAYS='Update EPUB Always'
 SAVE_OVERWRITE='Overwrite if Newer'
 SAVE_OVERWRITEALWAYS='Overwrite Always'
 SAVE_CALIBREONLY='Update Calibre Metadata Only'
+SAVE_CALIBREONLYSAVECOL='Update Calibre Metadata Only(Saved Column)'
 save_collisions={
     SKIP:SAVE_SKIP,
     ADDNEW:SAVE_ADDNEW,
@@ -112,6 +115,7 @@ save_collisions={
     SAVE_OVERWRITE:OVERWRITE,
     SAVE_OVERWRITEALWAYS:OVERWRITEALWAYS,
     SAVE_CALIBREONLY:CALIBREONLY,
+    SAVE_CALIBREONLYSAVECOL:CALIBREONLYSAVECOL,
     }
     
 anthology_collision_order=[UPDATE,
@@ -122,7 +126,7 @@ gpstyle='QGroupBox {border:0; padding-top:10px; padding-bottom:0px; margin-botto
 
 class RejectUrlEntry:
 
-    matchpat=re.compile(r"^(?P<url>[^,]+)(,(?P<fullnote>(((?P<title>.+) by (?P<auth>.+?)( - (?P<note>.+))?)|.*)))?$")
+    matchpat=re.compile(r"^(?P<url>[^,]+?)(,(?P<fullnote>(((?P<title>.+?) by (?P<auth>.+?)( - (?P<note>.+))?)|.*)))?$")
     
     def __init__(self,url_or_line,note=None,title=None,auth=None,
                  addreasontext=None,fromline=False,book_id=None):
@@ -254,6 +258,7 @@ class AddNewDialog(SizePersistedDialog):
         self.l.addWidget(self.url)
 
         self.merge = self.newmerge = False
+        self.extraoptions = {}
         
         # elements to hide when doing merge.
         self.mergehide = []
@@ -303,7 +308,7 @@ class AddNewDialog(SizePersistedDialog):
         self.collision.setToolTip("CollisionToolTip")
         # add collision options
         self.set_collisions()
-        i = self.collision.findText(save_collisions[prefs['collision']])
+        i = self.collision.findText(save_collisions[self.prefs['collision']])
         if i > -1:
             self.collision.setCurrentIndex(i)
         self.collisionlabel.setBuddy(self.collision)
@@ -317,14 +322,14 @@ class AddNewDialog(SizePersistedDialog):
         horz = QHBoxLayout()
         self.updatemeta = QCheckBox(_('Update Calibre &Metadata?'),self)
         self.updatemeta.setToolTip(_("Update metadata for existing stories in Calibre from web site?\n(Columns set to 'New Only' in the column tabs will only be set for new books.)"))
-        self.updatemeta.setChecked(prefs['updatemeta'])
+        self.updatemeta.setChecked(self.prefs['updatemeta'])
         horz.addWidget(self.updatemeta)
         self.mergehide.append(self.updatemeta)
         self.mergeupdateshow.append(self.updatemeta)
 
         self.updateepubcover = QCheckBox(_('Update EPUB Cover?'),self)
         self.updateepubcover.setToolTip(_('Update book cover image from site or defaults (if found) <i>inside</i> the EPUB when EPUB is updated.'))
-        self.updateepubcover.setChecked(prefs['updateepubcover'])
+        self.updateepubcover.setChecked(self.prefs['updateepubcover'])
         horz.addWidget(self.updateepubcover)
         self.mergehide.append(self.updateepubcover)
         
@@ -354,7 +359,10 @@ class AddNewDialog(SizePersistedDialog):
                     extrapayload=None):
         # rather than mutex in fff_plugin, just bail here if it's
         # already in use.
-        if self.isVisible(): return
+        if self.isVisible():
+            if url_list_text: # add to open box.
+               self.url.setText( '\n'.join([self.get_urlstext(), url_list_text]) )
+            return
 
         try:
             self.go_signal.disconnect()
@@ -398,6 +406,9 @@ class AddNewDialog(SizePersistedDialog):
 
         if not self.merge:
             self.fileform.setCurrentIndex(self.fileform.findText(self.prefs['fileform']))
+        else:
+            # always epub on self.merge (anthology)
+            self.fileform.setCurrentIndex(self.fileform.findText('epub'))
 
         # add collision options
         self.set_collisions()
@@ -424,12 +435,19 @@ class AddNewDialog(SizePersistedDialog):
         prev=self.collision.currentText()
         self.collision.clear()
         if self.merge:
-            order = anthology_collision_order
+            order = list(anthology_collision_order)
         else:
-            order = collision_order
+            order = list(collision_order)
+            ## Remove options that aren't valid.
+            if self.fileform.currentText() != 'epub':
+                order.remove(UPDATE)
+                order.remove(UPDATEALWAYS)
+            if self.prefs['savemetacol'] == '':
+                order.remove(CALIBREONLYSAVECOL)
+
         for o in order:
-            if self.merge or self.fileform.currentText() == 'epub' or o not in [UPDATE,UPDATEALWAYS]:
-                self.collision.addItem(o)
+            self.collision.addItem(o)
+            
         i = self.collision.findText(prev)
         if i > -1:
             self.collision.setCurrentIndex(i)
@@ -780,7 +798,7 @@ class UpdateExistingDialog(SizePersistedDialog):
         self.fileform.addItem('mobi')
         self.fileform.addItem('html')
         self.fileform.addItem('txt')
-        self.fileform.setCurrentIndex(self.fileform.findText(prefs['fileform']))
+        self.fileform.setCurrentIndex(self.fileform.findText(self.prefs['fileform']))
         self.fileform.setToolTip(_('Choose output format to create.  May set default from plugin configuration.'))
         self.fileform.activated.connect(self.set_collisions)
         label.setBuddy(self.fileform)
@@ -792,7 +810,7 @@ class UpdateExistingDialog(SizePersistedDialog):
         self.collision.setToolTip(_("What sort of update to perform.  May set default from plugin configuration."))
         # add collision options
         self.set_collisions()
-        i = self.collision.findText(save_collisions[prefs['collision']])
+        i = self.collision.findText(save_collisions[self.prefs['collision']])
         if i > -1:
             self.collision.setCurrentIndex(i)
         label.setBuddy(self.collision)
@@ -800,12 +818,12 @@ class UpdateExistingDialog(SizePersistedDialog):
 
         self.updatemeta = QCheckBox(_('Update Calibre &Metadata?'),self)
         self.updatemeta.setToolTip(_("Update metadata for existing stories in Calibre from web site?\n(Columns set to 'New Only' in the column tabs will only be set for new books.)"))
-        self.updatemeta.setChecked(prefs['updatemeta'])
+        self.updatemeta.setChecked(self.prefs['updatemeta'])
         gbl.addWidget(self.updatemeta)
                 
         self.updateepubcover = QCheckBox(_('Update EPUB Cover?'),self)
         self.updateepubcover.setToolTip(_('Update book cover image from site or defaults (if found) <i>inside</i> the EPUB when EPUB is updated.'))
-        self.updateepubcover.setChecked(prefs['updateepubcover'])
+        self.updateepubcover.setChecked(self.prefs['updateepubcover'])
         gbl.addWidget(self.updateepubcover)
 
 
@@ -824,10 +842,18 @@ class UpdateExistingDialog(SizePersistedDialog):
     def set_collisions(self):
         prev=self.collision.currentText()
         self.collision.clear()
-        for o in collision_order:
-            if o not in [ADDNEW,SKIP] and \
-                    (self.fileform.currentText() == 'epub' or o not in [UPDATE,UPDATEALWAYS]):
-                self.collision.addItem(o)        
+        order = list(collision_order)
+        order.remove(ADDNEW)
+        order.remove(SKIP)
+        if self.fileform.currentText() != 'epub':
+            order.remove(UPDATE)
+            order.remove(UPDATEALWAYS)
+        if self.prefs['savemetacol'] == '':
+            order.remove(CALIBREONLYSAVECOL)
+            
+        for o in order:
+            self.collision.addItem(o)
+            
         i = self.collision.findText(prev)
         if i > -1:
             self.collision.setCurrentIndex(i)
@@ -994,6 +1020,7 @@ class RejectListTableWidget(QTableWidget):
         note_cell.set_separator(None)
         note_cell.setToolTip(_('Select or Edit Reject Note.'))
         self.setCellWidget(row, 3, note_cell)
+        note_cell.setCursorPosition(0)
         
     def remove_selected_rows(self):
         self.setFocus()
@@ -1069,6 +1096,7 @@ class RejectListDialog(SizePersistedDialog):
             label.setToolTip(_("This will be added to whatever note you've set for each URL above."))
             horz.addWidget(label)
             horz.addWidget(self.reason_edit)
+            self.reason_edit.setCursorPosition(0)
             horz.insertStretch(-1)
             layout.addLayout(horz)
                     
@@ -1127,6 +1155,7 @@ class EditTextDialog(SizePersistedDialog):
 
     def __init__(self, parent, text,
                  icon=None, title=None, label=None, tooltip=None,
+                 read_only=False,
                  rejectreasons=[],reasonslabel=None,
                  save_size_name='fff:edit text dialog',
                  ):
@@ -1143,6 +1172,7 @@ class EditTextDialog(SizePersistedDialog):
         
         self.textedit = QTextEdit(self)
         self.textedit.setLineWrapMode(QTextEdit.NoWrap)
+        self.textedit.setReadOnly(read_only)
         self.textedit.setText(text)
         self.l.addWidget(self.textedit)
 
@@ -1168,6 +1198,7 @@ class EditTextDialog(SizePersistedDialog):
                 self.l.addLayout(horz)
             else:
                 self.l.addWidget(self.reason_edit)
+            self.reason_edit.setCursorPosition(0)
             
         button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
         button_box.accepted.connect(self.accept)

calibre-plugin/fff_plugin.py

@@ -1,5 +1,5 @@
-#!/usr/bin/env python
-# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+# -*- coding: utf-8 -*-
+
 from __future__ import (unicode_literals, division, absolute_import,
                         print_function)
 
@@ -13,24 +13,24 @@ logger = logging.getLogger(__name__)
 import time, os, copy, threading, re, platform, sys
 from StringIO import StringIO
 from functools import partial
-from datetime import datetime, time
+from datetime import datetime, time, date
 from string import Template
 import urllib
 import email
 import traceback
 
 try:
-    from PyQt5.Qt import (QApplication, QMenu, QTimer)
+    from PyQt5.Qt import (QApplication, QMenu, QTimer, QCursor, Qt)
     from PyQt5.QtCore import QBuffer
 except ImportError as e:
-    from PyQt4.Qt import (QApplication, QMenu, QTimer)
+    from PyQt4.Qt import (QApplication, QMenu, QTimer, QCursor, Qt)
     from PyQt4.QtCore import QBuffer
 
 from calibre.constants import numeric_version as calibre_version
 
 from calibre.ptempfile import PersistentTemporaryFile, PersistentTemporaryDirectory, remove_dir
 from calibre.ebooks.metadata import MetaInformation
-from calibre.ebooks.metadata.meta import get_metadata
+from calibre.ebooks.metadata.meta import get_metadata as calibre_get_metadata
 from calibre.gui2 import error_dialog, warning_dialog, question_dialog, info_dialog
 from calibre.gui2.dialogs.message_box import ViewLog
 from calibre.gui2.dialogs.confirm_delete import confirm
@@ -48,20 +48,46 @@ try:
 except NameError:
     pass # load_translations() added in calibre 1.9
 
-from calibre_plugins.fanficfare_plugin.common_utils import (set_plugin_icon_resources, get_icon,
-                                                            create_menu_action_unique, get_library_uuid)
+try:
+    # should be present from cal2.3.0 on.
+    from calibre.ebooks.covers import generate_cover as cal_generate_cover
+    HAS_CALGC=True
+except:
+    HAS_CALGC=False
 
-from calibre_plugins.fanficfare_plugin.fanficfare import adapters, exceptions
-from calibre_plugins.fanficfare_plugin.fanficfare.epubutils import get_dcsource, get_dcsource_chaptercount, get_story_url_from_html
-from calibre_plugins.fanficfare_plugin.fanficfare.geturls import get_urls_from_page, get_urls_from_html, get_urls_from_text, get_urls_from_imap
+from calibre.library.field_metadata import FieldMetadata
+field_metadata = FieldMetadata()
+    
+from calibre_plugins.fanficfare_plugin.common_utils import (
+    set_plugin_icon_resources, get_icon, create_menu_action_unique,
+    get_library_uuid)
+
+from calibre_plugins.fanficfare_plugin.fanficfare import (
+    adapters, exceptions)
+
+from calibre_plugins.fanficfare_plugin.fanficfare.epubutils import (
+    get_dcsource, get_dcsource_chaptercount, get_story_url_from_html,
+    reset_orig_chapters_epub)
+
+from calibre_plugins.fanficfare_plugin.fanficfare.geturls import (
+    get_urls_from_page, get_urls_from_html,get_urls_from_text,
+    get_urls_from_imap)
+
+from calibre_plugins.fanficfare_plugin.fff_util import (
+    get_fff_adapter, get_fff_config, get_fff_personalini)
+
+from calibre_plugins.fanficfare_plugin.config import (
+    permitted_values, rejecturllist, STD_COLS_SKIP)
+
+from calibre_plugins.fanficfare_plugin.prefs import (
+    prefs, SAVE_YES, SAVE_NO, SAVE_YES_IF_IMG, SAVE_YES_UNLESS_IMG)
 
-from calibre_plugins.fanficfare_plugin.fff_util import (get_fff_adapter, get_fff_config, get_fff_personalini)
-from calibre_plugins.fanficfare_plugin.config import (permitted_values, rejecturllist)
-from calibre_plugins.fanficfare_plugin.prefs import prefs
 from calibre_plugins.fanficfare_plugin.dialogs import (
     AddNewDialog, UpdateExistingDialog,
-    LoopProgressDialog, UserPassDialog, AboutDialog, CollectURLDialog, RejectListDialog, EmailPassDialog,
+    LoopProgressDialog, UserPassDialog, AboutDialog, CollectURLDialog,
+    RejectListDialog, EmailPassDialog,
     OVERWRITE, OVERWRITEALWAYS, UPDATE, UPDATEALWAYS, ADDNEW, SKIP, CALIBREONLY,
+    CALIBREONLYSAVECOL,
     NotGoingToDownload, RejectUrlEntry )
 
 # because calibre immediately transforms html into zip and don't want
@@ -310,6 +336,12 @@ class FanFicFarePlugin(InterfaceAction):
                                                                       image='minusminus.png',
                                                                       triggered=partial(self.update_lists,add=False))
 
+            self.menu.addSeparator()
+            self.get_list_action = self.create_menu_item_ex(self.menu, _('Remove "New" Chapter Marks from Selected books'),
+                                                            unique_name='Remove "(new)" chapter marks created by personal.ini <i>mark_new_chapters</i> setting.',
+                                                            image='edit-undo.png',
+                                                            triggered=self.unnew_books)
+
             self.menu.addSeparator()
             self.get_list_action = self.create_menu_item_ex(self.menu, _('Get Story URLs from Selected Books'),
                                                             unique_name='Get URLs from Selected Books',
@@ -350,7 +382,7 @@ class FanFicFarePlugin(InterfaceAction):
         # get_resources will return a dictionary mapping names to bytes. Names that
         # are not found in the zip file will not be in the returned dictionary.
 
-        text = get_resources('about.txt')
+        text = get_resources('about.html')
         AboutDialog(self.gui,self.qaction.icon(),self.version + text).exec_()
 
     def create_menu_item_ex(self, parent_menu, menu_text, image=None, tooltip=None,
@@ -390,6 +422,8 @@ class FanFicFarePlugin(InterfaceAction):
                 return
 
             self.update_reading_lists(self.gui.library_view.get_selected_ids(),add)
+            if not add and prefs['autounnew']:
+                self.unnew_books()
 
     def get_urls_from_imap_menu(self):
 
@@ -413,20 +447,42 @@ class FanFicFarePlugin(InterfaceAction):
             if prefs['imapsessionpass']:
                 self.imap_pass = imap_pass
 
+        self.busy_cursor()
+        self.gui.status_bar.show_message(_('Fetching Story URLs from Email...'))
         url_list = get_urls_from_imap(prefs['imapserver'],
                                       prefs['imapuser'],
                                       imap_pass,
                                       prefs['imapfolder'],
                                       prefs['imapmarkread'],)
+        reject_list=set()
+        if prefs['auto_reject_from_email']:
+            # need to normalize for reject list.
+            reject_list = set([x for x in url_list if rejecturllist.check(adapters.getNormalStoryURLSite(x)[0])])
+        url_list = url_list - reject_list
 
-        if url_list:
-            self.add_dialog("\n".join(url_list),merge=False)
+        self.gui.status_bar.show_message(_('No Valid Story URLs Found in Unread Emails.'),3000)
+        self.restore_cursor()
+
+        if prefs['download_from_email_immediately']:
+            ## do imap fetch w/o GUI elements
+            if url_list:
+                self.prep_downloads(self.add_new_dialog.get_fff_options(),
+                                    "\n".join(url_list))
+            else:
+                self.gui.status_bar.show_message(_('Finished Fetching Story URLs from Email.'),3000)
+                
         else:
-            info_dialog(self.gui, _('Get Story URLs from Email'),
-                        _('No Valid Story URLs Found in Unread Emails.'),
-                        show=True,
-                        show_copy_button=False)
-
+            if url_list:
+                self.add_dialog("\n".join(url_list),merge=False)
+            else:            
+                msg = _('No Valid Story URLs Found in Unread Emails.')
+                if reject_list:
+                    msg = msg + '<p>'+(_('(%d Story URLs Skipped, on Rejected URL List)')%len(reject_list))+'</p>'
+                info_dialog(self.gui, _('Get Story URLs from Email'),
+                            msg,
+                            show=True,
+                            show_copy_button=False)
+    
     def get_urls_from_page_menu(self,anthology=False):
 
         urltxt = ""
@@ -444,8 +500,14 @@ class FanFicFarePlugin(InterfaceAction):
             return
         url = u"%s"%d.url.text()
 
+        self.busy_cursor()
+        self.gui.status_bar.show_message(_('Fetching Story URLs from Page...'))
+
         url_list = self.get_urls_from_page(url)
 
+        self.gui.status_bar.show_message(_('Finished Fetching Story URLs from Page.'),3000)
+        self.restore_cursor()
+        
         if url_list:
             self.add_dialog("\n".join(url_list),merge=d.anthology,anthology_url=url)
         else:
@@ -510,6 +572,70 @@ class FanFicFarePlugin(InterfaceAction):
                         show=True,
                         show_copy_button=False)
 
+    def unnew_books(self):
+        '''Get list of URLs from existing books.'''
+        if not self.is_library_view():
+            self.gui.status_bar.show_message(_('Can only UnNew books in library'),
+                                             3000)
+            return
+        
+        if not self.gui.current_view().selectionModel().selectedRows() :
+            self.gui.status_bar.show_message(_('No Selected Books to Get URLs From'),
+                                             3000)
+            return
+
+        book_list = map( partial(self.make_book_id_only),
+                         self.gui.library_view.get_selected_ids() )
+
+        tdir = PersistentTemporaryDirectory(prefix='fanficfare_')
+        LoopProgressDialog(self.gui,
+                           book_list,
+                           partial(self.get_unnew_books_loop, db=self.gui.current_db, tdir=tdir),
+                           partial(self.get_unnew_books_finish, tdir=tdir),
+                           init_label=_("UnNewing books..."),
+                           win_title=_("UnNew Books"),
+                           status_prefix=_("Books UnNewed"))
+
+    def get_unnew_books_loop(self,book,db=None,tdir=None):
+        
+        if book['calibre_id'] and db.has_format(book['calibre_id'],'EPUB',index_is_id=True):
+            tmp = PersistentTemporaryFile(prefix='%s-'%book['calibre_id'],
+                                          suffix='.epub',
+                                          dir=tdir)
+            db.copy_format_to(book['calibre_id'],'EPUB',tmp,index_is_id=True)
+            
+            unnewtmp = PersistentTemporaryFile(prefix='unnew-%s-'%book['calibre_id'],
+                                               suffix='.epub',
+                                               dir=tdir)
+            book['changed']=reset_orig_chapters_epub(tmp,unnewtmp)
+            if book['changed']:
+                db.add_format_with_hooks(book['calibre_id'],
+                                         'EPUB',
+                                         unnewtmp,
+                                         index_is_id=True)
+                if prefs['deleteotherforms']:
+                    fmts = db.formats(book['calibre_id'], index_is_id=True).split(',')
+                    for fmt in fmts:
+                        if fmt.lower() != formmapping['epub'].lower():
+                            logger.debug("deleteotherforms remove f:"+fmt)
+                            db.remove_format(book['calibre_id'], fmt, index_is_id=True)#, notify=False
+                elif prefs['autoconvert']:
+                    ## 'Convert Book'.auto_convert_auto_add doesn't convert if
+                    ## the format is already there.
+                    fmt = calibre_prefs['output_format']
+                    # delete if there, but not if the format we just made.
+                    if fmt.lower() != 'epub' and db.has_format(book['calibre_id'],fmt,index_is_id=True):
+                        logger.debug("autoconvert remove f:"+fmt)
+                        db.remove_format(book['calibre_id'], fmt, index_is_id=True)#, notify=False
+                
+    def get_unnew_books_finish(self, book_list, tdir=None):
+        remove_dir(tdir)
+        if prefs['autoconvert']:
+            changed_ids = [ x['calibre_id'] for x in book_list if x['changed'] ]
+            if changed_ids:
+                self.gui.status_bar.show_message(_('Starting auto conversion of %d books.')%(len(changed_ids)), 3000)
+                self.gui.iactions['Convert Books'].auto_convert_auto_add(changed_ids)
+
     def reject_list_urls(self):
         if self.is_library_view():
             book_list = map( partial(self.make_book_id_only),
@@ -630,6 +756,9 @@ class FanFicFarePlugin(InterfaceAction):
             remove_dir(tdir)
             return
 
+        self.busy_cursor()
+        self.gui.status_bar.show_message(_('Fetching Story URLs for Series...'))
+        
         # get list from identifiers:url/uri if present, but only if
         # it's *not* a valid story URL.
         mergeurl = self.get_story_url(db,book_id)
@@ -638,6 +767,9 @@ class FanFicFarePlugin(InterfaceAction):
 
         url_list_text = "\n".join(url_list)
 
+        self.gui.status_bar.show_message(_('Finished Fetching Story URLs for Series.'),3000)
+        self.restore_cursor()
+        
         #print("urlmapfile:%s"%urlmapfile)
 
         # AddNewDialog collects URLs, format and presents buttons.
@@ -759,10 +891,10 @@ class FanFicFarePlugin(InterfaceAction):
         # No need to do anything with perfs here, but we could.
         prefs
 
-    def make_id_searchstr(self,url):
+    def do_id_search(self,url):
         # older idents can be uri vs url and have | instead of : after
         # http, plus many sites are now switching to https.
-        return 'identifiers:"~ur(i|l):~^%s$"'%re.sub(r'https?\\\:','https?(\:|\|)',re.escape(url))
+        return self.gui.current_db.search_getting_ids('identifiers:"~ur(i|l):~^%s$"'%re.sub(r'https?\\\:','https?(\:|\|)',re.escape(url)),None)
 
     def prep_downloads(self, options, books, merge=False, extrapayload=None):
         '''Fetch metadata for stories from servers, launch BG job when done.'''
@@ -777,6 +909,7 @@ class FanFicFarePlugin(InterfaceAction):
         options['version'] = self.version
         logger.debug(self.version)
         options['personal.ini'] = get_fff_personalini()
+        options['savemetacol'] = prefs['savemetacol']
 
         #print("prep_downloads:%s"%books)
 
@@ -868,9 +1001,9 @@ class FanFicFarePlugin(InterfaceAction):
             # book has already been flagged bad for whatever reason.
             return
 
-        skip_date_update = False
-
         adapter = get_fff_adapter(url,fileform)
+        ## chapter range for title_chapter_range_pattern
+        adapter.setChaptersRange(book['begin'],book['end'])
         ## save and share cookiejar and pagecache between all
         ## downloads.
         if 'pagecache' not in options:
@@ -880,67 +1013,91 @@ class FanFicFarePlugin(InterfaceAction):
             options['cookiejar'] = adapter.get_empty_cookiejar()
         adapter.set_cookiejar(options['cookiejar'])
 
-        # reduce foreground sleep time for ffnet when few books.
-        if 'ffnetcount' in options and \
-                adapter.getConfig('tweak_fg_sleep') and \
-                adapter.getSiteDomain() == 'www.fanfiction.net':
-            minslp = float(adapter.getConfig('min_fg_sleep'))
-            maxslp = float(adapter.getConfig('max_fg_sleep'))
-            dwnlds = float(adapter.getConfig('max_fg_sleep_at_downloads'))
-            m = (maxslp-minslp) / (dwnlds-1)
-            b = minslp - m
-            slp = min(maxslp,m*float(options['ffnetcount'])+b)
-            #print("m:%s b:%s = %s"%(m,b,slp))
-            adapter.set_sleep(slp)
-
-        ## three tries, that's enough if both user/pass & is_adult needed,
-        ## or a couple tries of one or the other
-        for x in range(0,2):
-            try:
-                adapter.getStoryMetadataOnly(get_cover=False)
-            except exceptions.FailedToLogin, f:
-                logger.warn("Login Failed, Need Username/Password.")
-                userpass = UserPassDialog(self.gui,url,f)
-                userpass.exec_() # exec_ will make it act modal
-                if userpass.status:
-                    adapter.username = userpass.user.text()
-                    adapter.password = userpass.passwd.text()
-
-            except exceptions.AdultCheckRequired:
-                if question_dialog(self.gui, _('Are You an Adult?'), '<p>'+
-                                   _("%s requires that you be an adult.  Please confirm you are an adult in your locale:")%url,
-                                   show_copy_button=False):
-                    adapter.is_adult=True
-
-        # let other exceptions percolate up.
-        story = adapter.getStoryMetadataOnly(get_cover=False)
-
-        series = story.getMetadata('series')
-        if not merge and series and prefs['checkforseriesurlid']:
-            # try to find *series anthology* by *seriesUrl* identifier url or uri first.
-            searchstr = self.make_id_searchstr(story.getMetadata('seriesUrl'))
-            identicalbooks = db.search_getting_ids(searchstr, None)
-            # print("searchstr:%s"%searchstr)
-            # print("identicalbooks:%s"%identicalbooks)
-            if len(identicalbooks) > 0 and question_dialog(self.gui, _('Skip Story?'),'''
-                                                              <h3>%s</h3>
-                                                              <p>%s</p>
-                                                              <p>%s</p>
-                                                              <p>%s</p>
-                                                           '''%(
-                                                           _('Skip Anthology Story?'),
-                                                           _('"<b>%s</b>" is in series "<b><a href="%s">%s</a></b>" that you have an anthology book for.')%(story.getMetadata('title'),story.getMetadata('seriesUrl'),series[:series.index(' [')]),
-                                                           _("Click '<b>Yes</b>' to Skip."),
-                                                           _("Click '<b>No</b>' to download anyway.")),
-                                                           show_copy_button=False):
-                book['comment'] = _("Story in Series Anthology(%s).")%series
-                book['title'] = story.getMetadata('title')
-                book['author'] = [story.getMetadata('author')]
-                book['good']=False
-                book['icon']='rotate-right.png'
-                book['status'] = _('Skipped')
-                return
+        if collision in (CALIBREONLY, CALIBREONLYSAVECOL):
+            ## Getting metadata from configured column.
+            custom_columns = self.gui.library_view.model().custom_columns
+            if ( collision in (CALIBREONLYSAVECOL) and
+                 prefs['savemetacol'] != '' and 
+                 prefs['savemetacol'] in custom_columns ):
 
+                savedmeta_book_id = book['calibre_id']
+                # won't have calibre_id if update by URL vs book.
+                if not savedmeta_book_id:
+                    identicalbooks = self.do_id_search(url)
+                    if len(identicalbooks) == 1:
+                        savedmeta_book_id = identicalbooks.pop()
+                        
+                if savedmeta_book_id:
+                    label = custom_columns[prefs['savemetacol']]['label']
+                    savedmetadata = db.get_custom(savedmeta_book_id, label=label, index_is_id=True)
+                else:
+                    savedmetadata = None
+                    
+                if savedmetadata:
+                    # sets flag inside story so getStoryMetadataOnly won't hit server.
+                    adapter.setStoryMetadata(savedmetadata)
+                
+            # let other exceptions percolate up.
+            story = adapter.getStoryMetadataOnly(get_cover=False)
+        else:
+            # reduce foreground sleep time for ffnet when few books.
+            if 'ffnetcount' in options and \
+                    adapter.getConfig('tweak_fg_sleep') and \
+                    adapter.getSiteDomain() == 'www.fanfiction.net':
+                minslp = float(adapter.getConfig('min_fg_sleep'))
+                maxslp = float(adapter.getConfig('max_fg_sleep'))
+                dwnlds = float(adapter.getConfig('max_fg_sleep_at_downloads'))
+                m = (maxslp-minslp) / (dwnlds-1)
+                b = minslp - m
+                slp = min(maxslp,m*float(options['ffnetcount'])+b)
+                #print("m:%s b:%s = %s"%(m,b,slp))
+                adapter.set_sleep(slp)
+    
+            ## three tries, that's enough if both user/pass & is_adult needed,
+            ## or a couple tries of one or the other
+            for x in range(0,2):
+                try:
+                    adapter.getStoryMetadataOnly(get_cover=False)
+                except exceptions.FailedToLogin, f:
+                    logger.warn("Login Failed, Need Username/Password.")
+                    userpass = UserPassDialog(self.gui,url,f)
+                    userpass.exec_() # exec_ will make it act modal
+                    if userpass.status:
+                        adapter.username = userpass.user.text()
+                        adapter.password = userpass.passwd.text()
+    
+                except exceptions.AdultCheckRequired:
+                    if question_dialog(self.gui, _('Are You an Adult?'), '<p>'+
+                                       _("%s requires that you be an adult.  Please confirm you are an adult in your locale:")%url,
+                                       show_copy_button=False):
+                        adapter.is_adult=True
+    
+            # let other exceptions percolate up.
+            story = adapter.getStoryMetadataOnly(get_cover=False)
+    
+            series = story.getMetadata('series')
+            if not merge and series and prefs['checkforseriesurlid']:
+                # try to find *series anthology* by *seriesUrl* identifier url or uri first.
+                identicalbooks = self.do_id_search(story.getMetadata('seriesUrl'))
+                # print("identicalbooks:%s"%identicalbooks)
+                if len(identicalbooks) > 0 and question_dialog(self.gui, _('Skip Story?'),'''
+                                                                  <h3>%s</h3>
+                                                                  <p>%s</p>
+                                                                  <p>%s</p>
+                                                                  <p>%s</p>
+                                                               '''%(
+                                                               _('Skip Anthology Story?'),
+                                                               _('"<b>%s</b>" is in series "<b><a href="%s">%s</a></b>" that you have an anthology book for.')%(story.getMetadata('title'),story.getMetadata('seriesUrl'),series[:series.index(' [')]),
+                                                               _("Click '<b>Yes</b>' to Skip."),
+                                                               _("Click '<b>No</b>' to download anyway.")),
+                                                               show_copy_button=False):
+                    book['comment'] = _("Story in Series Anthology(%s).")%series
+                    book['title'] = story.getMetadata('title')
+                    book['author'] = [story.getMetadata('author')]
+                    book['good']=False
+                    book['icon']='rotate-right.png'
+                    book['status'] = _('Skipped')
+                    return
 
         ################################################################################################################################################33
 
@@ -950,7 +1107,10 @@ class FanFicFarePlugin(InterfaceAction):
 
         # all_metadata duplicates some data, but also includes extra_entries, etc.
         book['all_metadata'] = story.getAllMetadata(removeallentities=True)
-
+        if prefs['savemetacol'] != '':
+            # get metadata to save in configured column.
+            book['savemetacol'] = story.dump_html_metadata()
+        
         book['title'] = story.getMetadata("title", removeallentities=True)
         book['author_sort'] = book['author'] = story.getList("author", removeallentities=True)
         book['publisher'] = story.getMetadata("site")
@@ -977,7 +1137,7 @@ class FanFicFarePlugin(InterfaceAction):
             book['timestamp'] = None # need *something* there for calibre.
 
         if not merge:# skip all the collision code when d/ling for merging.
-            if collision in (CALIBREONLY):
+            if collision in (CALIBREONLY, CALIBREONLYSAVECOL):
                 book['icon'] = 'metadata.png'
                 book['status'] = _('Meta')
 
@@ -995,11 +1155,9 @@ class FanFicFarePlugin(InterfaceAction):
                 logger.debug("from URL(%s)"%url)
 
                 # try to find by identifier url or uri first.
-                searchstr = self.make_id_searchstr(url)
-                identicalbooks = db.search_getting_ids(searchstr, None)
-                # print("searchstr:%s"%searchstr)
+                identicalbooks = self.do_id_search(url)
                 # print("identicalbooks:%s"%identicalbooks)
-                if len(identicalbooks) < 1:
+                if len(identicalbooks) < 1 and prefs['matchtitleauth']:
                     # find dups
                     authlist = story.getList("author", removeallentities=True)
                     mi = MetaInformation(story.getMetadata("title", removeallentities=True),
@@ -1018,7 +1176,7 @@ class FanFicFarePlugin(InterfaceAction):
                     raise NotGoingToDownload(_("More than one identical book by Identifer URL or title/author(s)--can't tell which book to update/overwrite."),"minusminus.png")
 
                 ## changed: add new book when CALIBREONLY if none found.
-                if collision == CALIBREONLY and not identicalbooks:
+                if collision in (CALIBREONLY, CALIBREONLYSAVECOL) and not identicalbooks:
                     collision = ADDNEW
                     options['collision'] = ADDNEW
 
@@ -1073,7 +1231,7 @@ class FanFicFarePlugin(InterfaceAction):
                                 return
 
             if book_id != None and collision != ADDNEW:
-                if collision in (CALIBREONLY):
+                if collision in (CALIBREONLY, CALIBREONLYSAVECOL):
                     book['comment'] = _('Metadata collected.')
                     # don't need temp file created below.
                     return
@@ -1091,9 +1249,6 @@ class FanFicFarePlugin(InterfaceAction):
                         if chaptercount == urlchaptercount:
                             if collision == UPDATE:
                                 raise NotGoingToDownload(_("Already contains %d chapters.")%chaptercount,'edit-undo.png')
-                            else:
-                                # UPDATEALWAYS
-                                skip_date_update = True
                         elif chaptercount > urlchaptercount:
                             raise NotGoingToDownload(_("Existing epub contains %d chapters, web site only has %d. Use Overwrite to force update.") % (chaptercount,urlchaptercount),'dialog_error.png')
                         elif chaptercount == 0:
@@ -1132,6 +1287,52 @@ class FanFicFarePlugin(InterfaceAction):
                     #print("calibre_series:%s [%s]"%book['calibre_series'])
 
         if book['good']: # there shouldn't be any !'good' books at this point.
+
+            ## Filling calibre_std_* and calibre_cust_* metadata
+            book['calibre_columns']={}
+            if prefs['cal_cols_pass_in']:
+                # std columns
+                mi = db.get_metadata(book['calibre_id'],index_is_id=True)
+                # book['calibre_columns']['calibre_std_identifiers']=\
+                #     {'val':', '.join(["%s:%s"%(k,v) for (k,v) in mi.get_identifiers().iteritems()]),
+                #                      'label':_('Ids')}
+                for k in mi.standard_field_keys():
+                # for k in mi:
+                    if k in STD_COLS_SKIP:
+                        continue
+                    (label,value,v,fmd) = mi.format_field_extended(k)
+                    if not label and k in field_metadata:
+                        label=field_metadata[k]['name']
+                    key='calibre_std_'+k
+                    
+                    # if k == 'user_categories':
+                    #     value=u', '.join(mi.get(k))
+                    #     label=_('User Categories')
+        
+                    if label: # only if it has a human readable name.
+                        if value is None or not book['calibre_id']:
+                            ## if existing book, populate existing calibre column
+                            ## values in metadata, else '' to hide.
+                            value=''                    
+                        book['calibre_columns'][key]={'val':value,'label':label}
+                        #logger.debug("%s(%s): %s"%(label,key,value))
+        
+                # custom columns
+                for k, column in self.gui.library_view.model().custom_columns.iteritems():
+                    if k != prefs['savemetacol']:
+                        key='calibre_cust_'+k[1:]
+                        label=column['name']
+                        value=db.get_custom(book['calibre_id'],
+                                            label=column['label'],
+                                            index_is_id=True)
+                        # custom always have name.
+                        if value is None or not book['calibre_id']:
+                            ## if existing book, populate existing calibre column
+                            ## values in metadata, else '' to hide.
+                            value=''
+                        book['calibre_columns'][key]={'val':value,'label':label}
+                        # logger.debug("%s(%s): %s"%(label,key,value))
+            
             # if still 'good', make a temp file to write the output to.
             # For HTML format users, make the filename inside the zip something reasonable.
             # For crazy long titles/authors, limit it to 200chars.
@@ -1143,17 +1344,6 @@ class FanFicFarePlugin(InterfaceAction):
             logger.debug("outfile:"+tmp.name)
             book['outfile'] = tmp.name
 
-            # cookiejar = PersistentTemporaryFile(prefix=story.formatFileName("${title}-${author}-",allowunsafefilename=False)[:100],
-            #                                     suffix='.cookiejar',
-            #                                     dir=options['tdir'])
-            # adapter.save_cookiejar(cookiejar.name)
-            # book['cookiejar'] = cookiejar.name
-            # pagecache = PersistentTemporaryFile(prefix=story.formatFileName("${title}-${author}-",allowunsafefilename=False)[:100],
-            #                                     suffix='.pagecache',
-            #                                     dir=options['tdir'])
-            # adapter.save_pagecache(pagecache.name)
-            # book['pagecache'] = pagecache.name
-
         return
 
     def start_download_job(self,book_list,
@@ -1170,7 +1360,7 @@ class FanFicFarePlugin(InterfaceAction):
 
         ## No need to BG process when CALIBREONLY!  Fake it.
         #print("options:%s"%options)
-        if options['collision'] == CALIBREONLY:
+        if options['collision'] in (CALIBREONLY, CALIBREONLYSAVECOL):
             class NotJob(object):
                 def __init__(self,result):
                     self.failed=False
@@ -1255,10 +1445,10 @@ class FanFicFarePlugin(InterfaceAction):
         logger.debug("add/update %s %s"%(book['title'],book['url']))
         mi = self.make_mi_from_book(book)
 
-        if options['collision'] != CALIBREONLY:
+        if options['collision'] not in (CALIBREONLY, CALIBREONLYSAVECOL):
             self.add_book_or_update_format(book,options,prefs,mi)
 
-        if options['collision'] == CALIBREONLY or \
+        if options['collision'] in (CALIBREONLY, CALIBREONLYSAVECOL) or \
                 ( (options['updatemeta'] or book['added']) and book['good'] ):
             try:
                 self.update_metadata(db, book['calibre_id'], book, mi, options)
@@ -1285,7 +1475,7 @@ class FanFicFarePlugin(InterfaceAction):
         failed_list = filter(lambda x : not x['good'] , book_list)
         failed_ids = [ x['calibre_id'] for x in failed_list ]
 
-        if options['collision'] != CALIBREONLY and \
+        if options['collision'] not in (CALIBREONLY, CALIBREONLYSAVECOL) and \
                 (prefs['addtolists'] or prefs['addtoreadlists']):
             self.update_reading_lists(all_ids,add=True)
 
@@ -1349,7 +1539,7 @@ class FanFicFarePlugin(InterfaceAction):
             if countpagesstats:
                 cp_plugin.count_statistics(all_ids,countpagesstats)
 
-        if prefs['autoconvert'] and options['collision'] != CALIBREONLY:
+        if prefs['autoconvert'] and options['collision'] not in (CALIBREONLY, CALIBREONLYSAVECOL):
             self.gui.status_bar.show_message(_('Starting auto conversion of %d books.')%(len(all_ids)), 3000)
             self.gui.iactions['Convert Books'].auto_convert_auto_add(all_ids)
 
@@ -1609,15 +1799,6 @@ class FanFicFarePlugin(InterfaceAction):
             if not oldmi.languages:
                 mi.languages=['en']
 
-        if options['fileform'] == 'epub' and prefs['updatecover']:
-            existingepub = db.format(book_id,'EPUB',index_is_id=True, as_file=True)
-            epubmi = get_metadata(existingepub,'EPUB')
-            if epubmi.cover_data[1] is not None:
-                try:
-                    db.set_cover(book_id, epubmi.cover_data[1])
-                except:
-                    logger.info("Failed to set_cover, skipping")
-
         # implement 'newonly' flags here by setting to the current
         # value again.
         if not book['added']:
@@ -1640,6 +1821,11 @@ class FanFicFarePlugin(InterfaceAction):
         #print("all_metadata: %s"%book['all_metadata'])
         custom_columns = self.gui.library_view.model().custom_columns
 
+        # save metadata to configured column
+        if 'savemetacol' in book and prefs['savemetacol'] != '' and prefs['savemetacol'] in custom_columns:
+            label = custom_columns[prefs['savemetacol']]['label']
+            self.set_custom(db, book_id, 'comment', book['savemetacol'], label=label, commit=True)
+
         #print("prefs['custom_cols'] %s"%prefs['custom_cols'])
         for col, meta in prefs['custom_cols'].iteritems():
             #print("setting %s to %s"%(col,meta))
@@ -1740,7 +1926,7 @@ class FanFicFarePlugin(InterfaceAction):
 
         # set author link if found.  All current adapters have authorUrl, except anonymous on AO3.
         # Moved down so author's already in the DB.
-        if 'authorUrl' in book['all_metadata']:
+        if 'authorUrl' in book['all_metadata'] and prefs['set_author_url']:
             authurls = book['all_metadata']['authorUrl'].split(", ")
             authorlist = [ a.replace('&',';') for a in book['author'] ]
             authorids = db.new_api.get_item_ids('authors',authorlist)
@@ -1757,73 +1943,105 @@ class FanFicFarePlugin(InterfaceAction):
 
         db.commit()
 
-        if 'Generate Cover' in self.gui.iactions and (book['added'] or not prefs['gcnewonly']):
-
-            #logger.debug("Do Generate Cover added:%s gcnewonly:%s"%(book['added'],prefs['gcnewonly']))
-
-            # force a refresh if generating cover so complex composite
-            # custom columns are current and correct
-            db.refresh_ids([book_id])
-
-            gc_plugin = self.gui.iactions['Generate Cover']
-            setting_name = None
-            if prefs['allow_gc_from_ini']:
-                if not configuration: # might already have it from allow_custcol_from_ini
-                    configuration = get_fff_config(book['url'],options['fileform'])
-
-                # template => regexp to match => GC Setting to use.
-                # generate_cover_settings:
-                # ${category} => Buffy:? the Vampire Slayer => Buffy
-                # for line in configuration.getConfig('generate_cover_settings').splitlines():
-                #     if "=>" in line:
-                #         (template,regexp,setting) = map( lambda x: x.strip(), line.split("=>") )
-                for (template,regexp,setting) in configuration.get_generate_cover_settings():
-                    value = Template(template).safe_substitute(book['all_metadata']).encode('utf8')
-                    # print("%s(%s) => %s => %s"%(template,value,regexp,setting))
-                    if re.search(regexp,value):
-                        setting_name = setting
-                        break
+        # updating calibre cover from book.
+        if options['fileform'] == 'epub' and (
+            (prefs['updatecover'] and not prefs['updatecalcover']) ## backward compat
+            or prefs['updatecalcover'] == SAVE_YES ## yes, always
+            or (prefs['updatecalcover'] == SAVE_YES_IF_IMG ## yes, if image.
+                and book['all_metadata']['cover_image'] )): # in ('specific','first','default','old')
+            existingepub = db.format(book_id,'EPUB',index_is_id=True, as_file=True)
+            epubmi = calibre_get_metadata(existingepub,'EPUB')
+            if epubmi.cover_data[1] is not None:
+                try:
+                    db.set_cover(book_id, epubmi.cover_data[1])
+                except:
+                    logger.info("Failed to set_cover, skipping")
 
+        # First, should cover generation happen at all?
+        if (book['added'] or not prefs['gcnewonly']) and ( # skip if not new book and gcnewonly is True
+            prefs['gencalcover'] == SAVE_YES ## yes, always
+            or (prefs['gencalcover'] == SAVE_YES_UNLESS_IMG ## yes, unless image.
+                and book['all_metadata']['cover_image'] not in ('specific','first','default')) ):
+        
+            cover_generated = False # flag for polish below.
+            # Yes, should do gencov.  Which?
+            if prefs['calibre_gen_cover'] and HAS_CALGC:
+                # calibre's builtin, if available.
+                cdata = cal_generate_cover(mi)
+                db.set_cover(book_id, cdata)
+                cover_generated = True
+            elif prefs['plugin_gen_cover'] and 'Generate Cover' in self.gui.iactions:
+                # plugin, if available.
+    
+                #logger.debug("Do Generate Cover added:%s gcnewonly:%s"%(book['added'],prefs['gcnewonly']))
+    
+                # force a refresh if generating cover so complex composite
+                # custom columns are current and correct
+                db.refresh_ids([book_id])
+    
+                gc_plugin = self.gui.iactions['Generate Cover']
+                setting_name = None
+                if prefs['allow_gc_from_ini']:
+                    if not configuration: # might already have it from allow_custcol_from_ini
+                        configuration = get_fff_config(book['url'],options['fileform'])
+    
+                    # template => regexp to match => GC Setting to use.
+                    # generate_cover_settings:
+                    # ${category} => Buffy:? the Vampire Slayer => Buffy
+                    # for line in configuration.getConfig('generate_cover_settings').splitlines():
+                    #     if "=>" in line:
+                    #         (template,regexp,setting) = map( lambda x: x.strip(), line.split("=>") )
+                    for (template,regexp,setting) in configuration.get_generate_cover_settings():
+                        value = Template(template).safe_substitute(book['all_metadata']).encode('utf8')
+                        print("%s(%s) => %s => %s"%(template,value,regexp,setting))
+                        if re.search(regexp,value):
+                            setting_name = setting
+                            break
+    
+                    if setting_name:
+                        logger.debug("Generate Cover Setting from generate_cover_settings(%s)"%setting_name)
+                        if setting_name not in gc_plugin.get_saved_setting_names():
+                            logger.info("GC Name %s not found, discarding! (check personal.ini for typos)"%setting_name)
+                            setting_name = None
+    
+                if not setting_name and book['all_metadata']['site'] in prefs['gc_site_settings']:
+                    setting_name =  prefs['gc_site_settings'][book['all_metadata']['site']]
+                    logger.debug("Generate Cover Setting from site(%s)"%setting_name)
+    
+                if not setting_name and 'Default' in prefs['gc_site_settings']:
+                    setting_name =  prefs['gc_site_settings']['Default']
+                    logger.debug("Generate Cover Setting from Default(%s)"%setting_name)
+    
                 if setting_name:
-                    logger.debug("Generate Cover Setting from generate_cover_settings(%s)"%setting_name)
-                    if setting_name not in gc_plugin.get_saved_setting_names():
-                        logger.info("GC Name %s not found, discarding! (check personal.ini for typos)"%setting_name)
-                        setting_name = None
-
-            if not setting_name and book['all_metadata']['site'] in prefs['gc_site_settings']:
-                setting_name =  prefs['gc_site_settings'][book['all_metadata']['site']]
-                logger.debug("Generate Cover Setting from site(%s)"%setting_name)
-
-            if not setting_name and 'Default' in prefs['gc_site_settings']:
-                setting_name =  prefs['gc_site_settings']['Default']
-                logger.debug("Generate Cover Setting from Default(%s)"%setting_name)
-
-            if setting_name:
-                logger.debug("Running Generate Cover with settings %s."%setting_name)
-                realmi = db.get_metadata(book_id, index_is_id=True)
-                gc_plugin.generate_cover_for_book(realmi,saved_setting_name=setting_name)
-
-                if prefs['gc_polish_cover'] and \
-                        options['fileform'] == "epub":
-                    # set cover inside epub from calibre's polish feature
-                    from calibre.ebooks.oeb.polish.main import polish, ALL_OPTS
-                    from calibre.utils.logging import Log
-                    from collections import namedtuple
-
-                    # Couldn't find a better way to get the cover path.
-                    cover_path = os.path.join(db.library_path, db.path(book_id, index_is_id=True), 'cover.jpg')
-                    data = {'cover':cover_path}
-                    #print("cover_path:%s"%cover_path)
-                    opts = ALL_OPTS.copy()
-                    opts.update(data)
-                    O = namedtuple('Options', ' '.join(ALL_OPTS.iterkeys()))
-                    opts = O(**opts)
-
-                    log = Log(level=Log.DEBUG)
-                    outfile = db.format_abspath(book_id, formmapping[options['fileform']], index_is_id=True)
-                    #print("polish cover outfile:%s"%outfile)
-                    polish({outfile:outfile}, opts, log, logger.info)
-
+                    logger.debug("Running Generate Cover with settings %s."%setting_name)
+                    realmi = db.get_metadata(book_id, index_is_id=True)
+                    gc_plugin.generate_cover_for_book(realmi,saved_setting_name=setting_name)
+                    cover_generated = True
+    
+            if cover_generated and prefs['gc_polish_cover'] and \
+                    options['fileform'] == "epub":
+                # set cover inside epub from calibre's polish feature
+                from calibre.ebooks.oeb.polish.main import polish, ALL_OPTS
+                from calibre.utils.logging import Log
+                from collections import namedtuple
+    
+                # Couldn't find a better way to get the cover path.
+                cover_path = os.path.join(db.library_path,
+                                          db.path(book_id, index_is_id=True),
+                                          'cover.jpg')
+                data = {'cover':cover_path}
+                #print("cover_path:%s"%cover_path)
+                opts = ALL_OPTS.copy()
+                opts.update(data)
+                O = namedtuple('Options', ' '.join(ALL_OPTS.iterkeys()))
+                opts = O(**opts)
+    
+                log = Log(level=Log.DEBUG)
+                outfile = db.format_abspath(book_id,
+                                            formmapping[options['fileform']],
+                                            index_is_id=True)
+                #print("polish cover outfile:%s"%outfile)
+                polish({outfile:outfile}, opts, log, logger.info)
 
     def get_clean_reading_lists(self,lists):
         if lists == None or lists.strip() == "" :
@@ -2018,7 +2236,7 @@ class FanFicFarePlugin(InterfaceAction):
             existingepub = None
             if path == None and db.has_format(book_id,'EPUB',index_is_id=True):
                 existingepub = db.format(book_id,'EPUB',index_is_id=True, as_file=True)
-                mi = get_metadata(existingepub,'EPUB')
+                mi = calibre_get_metadata(existingepub,'EPUB')
                 identifiers = mi.get_identifiers()
                 if 'url' in identifiers:
                     # print("url from get_metadata:%s"%identifiers['url'].replace('|',':'))
@@ -2172,6 +2390,12 @@ class FanFicFarePlugin(InterfaceAction):
 
         return book
 
+    def busy_cursor(self):
+        QApplication.setOverrideCursor(QCursor(Qt.WaitCursor))
+
+    def restore_cursor(self):
+        QApplication.restoreOverrideCursor()
+    
 def split_text_to_urls(urls):
     # remove dups while preserving order.
     dups=set()

calibre-plugin/fff_util.py

@@ -1,5 +1,5 @@
-#!/usr/bin/env python
-# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+# -*- coding: utf-8 -*-
+
 from __future__ import (unicode_literals, division, absolute_import,
                         print_function)
 
@@ -18,25 +18,17 @@ from calibre_plugins.fanficfare_plugin.fanficfare.configurable import Configurat
 from calibre_plugins.fanficfare_plugin.prefs import prefs
 
 def get_fff_personalini():
-    if prefs['includeimages']:
-        # this is a cheat to make it easier for users.
-        return '''[epub]
-include_images:true
-keep_summary_html:true
-make_firstimage_cover:true
-''' + prefs['personal.ini']
-    else:
-        return prefs['personal.ini']
+    return prefs['personal.ini']
 
 def get_fff_config(url,fileform="epub",personalini=None):
     if not personalini:
         personalini = get_fff_personalini()
-    site='unknown'
+    sections=['unknown']
     try:
-        site = adapters.getConfigSectionFor(url)
+        sections = adapters.getConfigSectionsFor(url)
     except Exception as e:
-        logger.debug("Failed trying to get ini config for url(%s): %s, using section [%s] instead"%(url,e,site))
-    configuration = Configuration(site,fileform)
+        logger.debug("Failed trying to get ini config for url(%s): %s, using section %s instead"%(url,e,sections))
+    configuration = Configuration(sections,fileform)
     configuration.readfp(StringIO(get_resources("plugin-defaults.ini")))
     configuration.readfp(StringIO(personalini))
 

calibre-plugin/inihighlighter.py

@@ -1,5 +1,5 @@
-#!/usr/bin/env python
-# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+# -*- coding: utf-8 -*-
+
 from __future__ import (unicode_literals, division,
                         print_function)
 

calibre-plugin/jobs.py

@@ -1,5 +1,5 @@
-#!/usr/bin/env python
-# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+# -*- coding: utf-8 -*-
+
 from __future__ import (unicode_literals, division, absolute_import,
                         print_function)
 
@@ -11,6 +11,7 @@ import logging
 logger = logging.getLogger(__name__)
 
 import time, traceback
+from StringIO import StringIO
     
 from calibre.utils.ipc.server import Server
 from calibre.utils.ipc.job import ParallelJob
@@ -99,7 +100,7 @@ def do_download_for_worker(book,options,notification=lambda x,y:x):
     with fffbase:
         
         from calibre_plugins.fanficfare_plugin.dialogs import (NotGoingToDownload,
-                OVERWRITE, OVERWRITEALWAYS, UPDATE, UPDATEALWAYS, ADDNEW, SKIP, CALIBREONLY)
+                OVERWRITE, OVERWRITEALWAYS, UPDATE, UPDATEALWAYS, ADDNEW, SKIP, CALIBREONLY, CALIBREONLYSAVECOL)
         from calibre_plugins.fanficfare_plugin.fanficfare import adapters, writers, exceptions
         from calibre_plugins.fanficfare_plugin.fanficfare.epubutils import get_update_data
         
@@ -151,9 +152,12 @@ def do_download_for_worker(book,options,notification=lambda x,y:x):
             outfile = book['outfile']
     
             ## No need to download at all.  Shouldn't ever get down here.
-            if options['collision'] in (CALIBREONLY):
+            if options['collision'] in (CALIBREONLY, CALIBREONLYSAVECOL):
                 logger.info("Skipping CALIBREONLY 'update' down inside worker--this shouldn't be happening...")
                 book['comment'] = 'Metadata collected.'
+                book['all_metadata'] = story.getAllMetadata(removeallentities=True)
+                if options['savemetacol'] != '':
+                    book['savemetacol'] = story.dump_html_metadata()
                 
             ## checks were done earlier, it's new or not dup or newer--just write it.
             elif options['collision'] in (ADDNEW, SKIP, OVERWRITE, OVERWRITEALWAYS) or \
@@ -161,23 +165,20 @@ def do_download_for_worker(book,options,notification=lambda x,y:x):
     
                 # preserve logfile even on overwrite.
                 if 'epub_for_update' in book:
-                    (urlignore,
-                     chaptercountignore,
-                     oldchaptersignore,
-                     oldimgsignore,
-                     oldcoverignore,
-                     calibrebookmarkignore,
-                     # only logfile set in adapter, so others aren't used.
-                     adapter.logfile) = get_update_data(book['epub_for_update'])
-    
+                    
+                    adapter.logfile = get_update_data(book['epub_for_update'])[6]
                     # change the existing entries id to notid so
                     # write_epub writes a whole new set to indicate overwrite.
                     if adapter.logfile:
                         adapter.logfile = adapter.logfile.replace("span id","span notid")
                 
                 logger.info("write to %s"%outfile)
+                inject_cal_cols(book,story,configuration)
                 writer.writeStory(outfilename=outfile, forceOverwrite=True)
                 book['comment'] = 'Download %s completed, %s chapters.'%(options['fileform'],story.getMetadata("numChapters"))
+                book['all_metadata'] = story.getAllMetadata(removeallentities=True)
+                if options['savemetacol'] != '':
+                    book['savemetacol'] = story.dump_html_metadata()
                 
             ## checks were done earlier, just update it.
             elif 'epub_for_update' in book and options['collision'] in (UPDATE, UPDATEALWAYS):
@@ -191,12 +192,17 @@ def do_download_for_worker(book,options,notification=lambda x,y:x):
                  adapter.oldimgs,
                  adapter.oldcover,
                  adapter.calibrebookmark,
-                 adapter.logfile) = get_update_data(book['epub_for_update'])
+                 adapter.logfile,
+                 adapter.oldchaptersmap,
+                 adapter.oldchaptersdata) = get_update_data(book['epub_for_update'])[0:9]
     
                 # dup handling from fff_plugin needed for anthology updates.
                 if options['collision'] == UPDATE:
                     if chaptercount == urlchaptercount:
                         book['comment']=_("Already contains %d chapters.  Reuse as is.")%chaptercount
+                        book['all_metadata'] = story.getAllMetadata(removeallentities=True)
+                        if options['savemetacol'] != '':
+                            book['savemetacol'] = story.dump_html_metadata()
                         book['outfile'] = book['epub_for_update'] # for anthology merge ops.
                         return book
     
@@ -211,11 +217,15 @@ def do_download_for_worker(book,options,notification=lambda x,y:x):
                 logger.info("Do update - epub(%d) vs url(%d)" % (chaptercount, urlchaptercount))
                 logger.info("write to %s"%outfile)
     
+                inject_cal_cols(book,story,configuration)
                 writer.writeStory(outfilename=outfile, forceOverwrite=True)
                 
                 book['comment'] = _('Update %s completed, added %s chapters for %s total.')%\
                     (options['fileform'],(urlchaptercount-chaptercount),urlchaptercount)
-            
+                book['all_metadata'] = story.getAllMetadata(removeallentities=True)
+                if options['savemetacol'] != '':
+                    book['savemetacol'] = story.dump_html_metadata()
+
             if options['smarten_punctuation'] and options['fileform'] == "epub" \
                     and calibre_version >= (0, 9, 39):
                 # for smarten punc
@@ -249,3 +259,22 @@ def do_download_for_worker(book,options,notification=lambda x,y:x):
             
         #time.sleep(10)
     return book
+
+## calibre's columns for an existing book are pased in and injected
+## into the story's metadata.  For convenience, we also add labels and
+## valid_entries for them in a special [injected] section that has
+## even less precedence than [defaults]
+def inject_cal_cols(book,story,configuration):
+    configuration.remove_section('injected')
+    if 'calibre_columns' in book:
+        injectini = ['[injected]']
+        extra_valid = []
+        for k, v in book['calibre_columns'].iteritems():
+            story.setMetadata(k,v['val'])
+            injectini.append('%s_label:%s'%(k,v['label']))
+            extra_valid.append(k)
+        if extra_valid: # if empty, there's nothing to add.
+            injectini.append("add_to_extra_valid_entries:,"+','.join(extra_valid))
+            configuration.readfp(StringIO('\n'.join(injectini)))
+            #print("added:\n%s\n"%('\n'.join(injectini)))
+    

calibre-plugin/plugin-defaults.ini

@@ -85,9 +85,9 @@ authorId_label:Author ID
 ## show up in Calibre as tags.  Also carried into mobi when converted.
 extratags_label:Extra Tags
 ## The version of fanficdownloader
-version_label:FFDL Version
+version_label:Downloader Version
 
-## Date formats used by FFDL.  Published and Update don't have time.
+## Date formats used by FanFicFare.  Published and Update don't have time.
 ## See http://docs.python.org/library/datetime.html#strftime-strptime-behavior
 ## Note that ini format requires % to be escaped as %%.
 dateCreated_format:%%Y-%%m-%%d %%H:%%M:%%S
@@ -172,7 +172,7 @@ connect_timeout:60.0
 ## Note that if metakey == conditionalkey the conditional is ignored.
 ## You can use \s in the replacement to add explicit spaces.  (The config parser
 ## tends to discard trailing spaces.)
-## replace_metadata <entry>_LIST options: FFDL replace_metadata lines
+## replace_metadata <entry>_LIST options: FanFicFare replace_metadata lines
 ## operate on individual list items for list entries. But if you
 ## want to do a replacement on the joined string for the whole list,
 ## you can by using <entry>_LIST. Example, if you added
@@ -232,12 +232,21 @@ connect_timeout:60.0
 ## Make sure to keep at least one space at the start of each line and
 ## to escape % to %%, if used.
 ## template => regexp to match => GC Setting to use.
-## To use this, make sure you go to the Generate Cover tab in FFDL
+## To use this, make sure you go to the Generate Cover tab in FanFicFare
 ## config and check 'Allow generate_cover_settings from personal.ini
 ## to override'
+## You can check for an existing cover image in the metadata
+## cover_image.  cover_image can be:
+## specific -- The site has an image that is specifically this story's cover
+## first -- The first image in the story or story desc is used as cover
+## default -- A default_cover_image was used
+## old -- There was already a cover image in the epub.
+## This is useful because GC plugin can use the existing image.
 #generate_cover_settings:
 # ${category} => Buffy:? [tT]he Vampire Slayer => BuffyCover
 # ${category} => Star Trek => StarTrekCover
+# ${cover_image} => => CompleteCover
+# ${cover_image} => (specific|first|default) => CoverWithOrigImage
 
 ## If set false, the summary will have all html stripped.
 ## Both this and include_images must be true to get images in the
@@ -252,6 +261,11 @@ keep_summary_html:true
 ## kept.  Some sites(AO3) use <a title="Translation">Foreign language</a>
 #keep_title_attr: false
 
+## If a chapter range was given, use this pattern for the book title.
+## replace_metadata and include/exclude will be applied *after* this.
+## Set to empty value to disable.
+title_chapter_range_pattern:${title} (Ch ${first}-${last})
+
 ## Don't like the numbers at the start of chapter titles on some
 ## sites?  You can use strip_chapter_numbers to strip them off.  Just
 ## want to make them all look the same?  Strip them off, then add them
@@ -284,12 +298,31 @@ chapter_title_strip_pattern:^[0-9]+[\.: -]+
 ## etc
 #chapter_title_strip_pattern:^([0-9]+[\.: -]+)?(Chapter *[0-9]+[\.:, -]*)?
 
-## Uses a python template substitution.  The ${index} is the 'chapter'
-## number and ${title} is the chapter title, after applying
-## chapter_title_strip_pattern.  Those are the only variables available.
-## "The Beginning" => "1. The Beginning" 
+## If true, when updating an epub that already has old chapters, new
+## chapters will be marked in the TOC and chapter header by using
+## chapter_title_new_pattern and chapter_title_addnew_pattern to set the chapter.
+mark_new_chapters:false
+
+## chapter title patterns use python template substitution.  The
+## ${index} is the 'chapter' number and ${title} is the chapter title,
+## after applying chapter_title_strip_pattern.  Those are the only
+## variables available.
+
+## The basic pattern used when not using add_chapter_numbers or
+## mark_new_chapters
+chapter_title_def_pattern:${title}
+
+## Pattern used with add_chapter_numbers, but not mark_new_chapters
 chapter_title_add_pattern:${index}. ${title}
 
+## Pattern used with mark_new_chapters, but not add_chapter_numbers
+## (new) is just text and can be changed.
+chapter_title_new_pattern:(new) ${title}
+
+## Pattern used with add_chapter_numbers and mark_new_chapters
+## (new) is just text and can be changed.
+chapter_title_addnew_pattern:${index}. (new) ${title}
+
 ## Uses a python template substitution.  The ${title} is the default
 ## title of a new anthology, <series name> in the case of a series, or
 ## the first book title otherwise.  This is only applied to new
@@ -306,20 +339,88 @@ anthology_tags:Anthology
 ## ships_LIST to change the / back to something else if you want.
 sort_ships:false
 
-## join_string_<entry> options -- FFDL list entries are comma
+## join_string_<entry> options -- FanFicFare list entries are comma
 ## separated by default. You can use this to change that. For example,
 ## if you want authors separated with ' & ' instead, use
 ## join_string_calibre_author:\s&\s. (\s == space)
 #join_string_author:,\s
 
-## keep_in_order_<entry> options: FFDL sorts list entries by default
+## keep_in_order_<entry> options: FanFicFare sorts list entries by default
 ## (except for author/authorUrl/authorId). But if you want to use an
 ## extra entry derived from author, it ends up sorted. For example, if
 ## you added calibre_author: keep_in_order_calibre_author:true
 #keep_in_order_author:true
 
 ## User-agent
-user_agent:FFDL/2.0
+user_agent:FFF/2.X
+
+## Added for [base_xenforoforum], but can be used with other sites,
+## too.  Limit the 'description' to the first X *characters*
+## collected.  Character count includes HTML tags, so it can be
+## non-intuitive.
+#description_limit:1000
+
+[base_efiction]
+## At the time of writing, eFiction Base adapters allow downloading
+## the whole story in bulk using the 'Print' feature. If 'bulk_load'
+## is set to 'true', both metadata and chapters can be loaded in one
+## step
+bulk_load:true
+
+[base_xenforoforum]
+## Currently only forums.spacebattles.com and forums.sufficientvelocity.com
+
+cover_exclusion_regexp:/clear.png
+
+## I saw lots of chapters name simply '1.1' etc during testing.
+strip_chapter_numbers:false
+
+## Copy title to tagsfromtitle for parsing tags.
+add_to_extra_valid_entries:,tagsfromtitle
+
+## '.NOREPL' tells the system to *not* apply title's
+## in/exclude/replace_metadata -- Only works on include_in_ lines.
+include_in_tagsfromtitle:title.NOREPL
+
+tagsfromtitle_label:Tags from Title
+
+## might want to do this, maybe not.  Will often include category, but
+## also often include non-category stuff.
+# include_in_category:tagsfromtitle 
+
+add_to_include_metadata_pre:
+# only keep tagsfromtitle with ( or [ in.
+ tagsfromtitle=~[\[\(]
+ 
+add_to_replace_metadata:
+# remove anything outside () or []
+ tagsfromtitle=>^.*?([\(\[]([^\]\)]+)[\)\]]).*?$=>\1
+# remove () []
+ tagsfromtitle=>[\(\)\[\]]=>
+# change (spaces)slash(or semicolon)(spaces) to comma
+ tagsfromtitle=> *[/;] *=>,
+ tagsfromtitle=> [xX] =>,
+ 
+# remove [] or () blocks and leading/trailing spaces/dashes/colons
+ title=>[-: ]*[\(\[]([^\]\)]+)[\)\]][-: ]*=>
+# remove 'Thread' and the next word, usually "Thread 2", "Thread
+# four", "Thread iv", etc
+ title,tagsfromtitle=>[-: ]*[Tt]hread [^ ]+[-: ]*=>
+
+add_to_extra_titlepage_entries:,tagsfromtitle
+
+## '.SPLIT' tells the system to split by ','
+add_to_include_subject_tags:,tagsfromtitle.SPLIT
+
+## base_xenforoforum reads Published and Updated datetimes from
+## Threadmarks if used, or from the posted & updated times of the
+## 'first' post if no threadmarks.
+datePublished_format:%%Y-%%m-%%d %%H:%%M:%%S
+dateUpdated_format:%%Y-%%m-%%d %%H:%%M:%%S
+
+## Only take the first X characters of the 'first' post to use as
+## the description.
+description_limit:500
 
 ## Each output format has a section that overrides [defaults]
 [html]
@@ -330,7 +431,7 @@ user_agent:FFDL/2.0
 ## include_images is *not* available in the web service in any format.
 #include_images:false
 
-## This switch prevents FFDL from doing any processing on the images.
+## This switch prevents FanFicFare from doing any processing on the images.
 ## Usually they would be converted to jpg, resized and optionally made
 ## grayscale.
 no_image_processing: true
@@ -442,12 +543,12 @@ output_css:
 ## include images from img tags in the body and summary of
 ## stories.  Images will be converted to jpg for size if possible.
 ## include_images is *only* available in epub and html output format.
-#include_images:false
+include_images:true
 
 ## If set, the first image found will be made the cover image.  If
 ## keep_summary_html is true, any images in summary will be before any
 ## in chapters.
-#make_firstimage_cover: false
+make_firstimage_cover: true
 
 ## If set, the epub will never have a cover, even include_images is on
 ## and the site has specific cover images.
@@ -479,6 +580,22 @@ image_max_size: 580, 725
 ## space.
 #grayscale_images: false
 
+## jpg or png
+## -- jpg produces smaller images, and may be supported by more
+## readers, but it's older and doesn't allow transparency.
+## Transparency removed as if remove_transparency: true
+## -- png is newer but does allow transparency, but only in CLI.
+## It doesn't work in calibre PI due to limitations of the API.
+convert_images_to: jpg
+
+## Remove transparency and fill with background_color if true.
+remove_transparency: true
+
+## This switch prevents FanFicFare from doing any processing on the images.
+## Usually they would be converted to jpg, resized and optionally made
+## grayscale.
+#no_image_processing: false
+
 ## if the <img> tag doesn't have a div or a p around it, nook gets
 ## confused and displays it on every page after that under the text
 ## for the rest of the chapter.  I doubt adding a div around the img
@@ -549,10 +666,12 @@ extratags: FanFiction,Testing,HTML
 #is_adult:true
 
 ## archiveofourown.org stories allow chapters to be added out of
-## order.  So the newest chapter may not be the last one.  FFDL update
+## order.  So the newest chapter may not be the last one.  FanFicFare update
 ## doesn't like that.  If do_update_hook is uncommented and set true,
 ## the adapter will discard all existing chapters from the newest one
 ## on when updating to enforce accurate chapters.
+## Starting July 2015, FFF stores chapter URLs in the chapter files.
+## Stories downloaded after that shouldn't need this setting anymore.
 #do_update_hook:false
 
 ## AO3 adapter defines a few extra metadata entries.
@@ -568,6 +687,18 @@ hits_label:Hits
 collections_label:Collections
 bookmarks_label:Bookmarks
 
+## AO3 doesn't have anything it calls 'genre'.  The adapter used to be
+## hardcoded to include the site specific metadata freeformtags &
+## ao3categories in the standard metadata field genre.  By making it
+## configurable, users can change it.
+include_in_genre: freeformtags, ao3categories
+
+## AO3 uses the word 'category' differently than most sites.  The
+## adapter used to be hardcoded to include the site specific metadata
+## fandom in the standard metadata field category.  By making it
+## configurable, users can change it.
+include_in_category:fandoms
+
 ## freeformtags was previously typo'ed as freefromtags.  This way,
 ## freefromtags will still work for people who've used it.
 include_in_freefromtags:freeformtags
@@ -579,7 +710,7 @@ include_in_freefromtags:freeformtags
 #extra_subject_tags:fandoms,freeformtags,ao3categories
 
 ## AO3 chapters can include several different types of notes.  We've
-## traditional included them all in the chapter text, but this allows
+## traditionally included them all in the chapter text, but this allows
 ## you to customize which you include.  Copy this parameter to your
 ## personal.ini and list the ones you don't want.
 #exclude_notes:authorheadnotes,chaptersummary,chapterheadnotes,chapterfootnotes,authorfootnotes
@@ -729,11 +860,6 @@ extraships:Spike/Buffy
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
-## Virtually all eFiction-based sites allow downloading the whole story in
-## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
-## metadata and chapters can be loaded in one step
-bulk_load:true
-
 [dramione.org]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -758,7 +884,7 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 
 ## Some adapters collect additional meta information beyond the
 ## standard ones.  They need to be defined in extra_valid_entries to
-## tell the rest of the FFDL system about them.  They can be used in
+## tell the rest of the FanFicFare system about them.  They can be used in
 ## include_subject_tags, titlepage_entries, extra_titlepage_entries,
 ## logpage_entries, extra_logpage_entries, and include_in_* config
 ## items.  You can also add additional entries here to build up
@@ -914,11 +1040,6 @@ extraships:Harry Potter/Hermione Granger
 #username:YourName
 #password:yourpassword
 
-## Virtually all eFiction-based sites allow downloading the whole story in
-## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
-## metadata and chapters can be loaded in one step
-bulk_load:true
-
 extra_valid_entries: readings,romance
 extra_titlepage_entries: readings,romance
 readings_label: Readings
@@ -1008,6 +1129,12 @@ extra_valid_entries:size
 # don't show twitter icon.
 cover_exclusion_regexp:/res/css/bir.png
 
+[forums.spacebattles.com]
+## see [base_xenforoforum]
+
+[forums.sufficientvelocity.com]
+## see [base_xenforoforum]
+
 [grangerenchanted.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -1054,6 +1181,18 @@ extracategories:In Death
 ## cover image.  This lets you exclude them.
 cover_exclusion_regexp:/public/style_emoticons/.*
 
+[it-could-happen.net]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+## RPF == http://en.wikipedia.org/wiki/Real_person_fiction
+## Site dedicated to these categories/characters/ships
+extracategories:Glee RPF
+extracharacters:Darren Criss, Chris Colfer
+extraships:Darren Criss/Chris Colfer
+
 [ksarchive.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:Star Trek
@@ -1073,11 +1212,6 @@ eroticatags_label:Erotica Tags
 extra_titlepage_entries: eroticatags
 
 [lotrfanfiction.com]
-## Virtually all eFiction-based sites allow downloading the whole story in
-## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
-## metadata and chapters can be loaded in one step
-bulk_load:true
-
 extra_valid_entries: readings
 readings_label: Readings
 
@@ -1131,6 +1265,12 @@ extracategories:NCIS
 extracategories:Buffy: The Vampire Slayer
 extracharacters:Willow
 
+[ninelives.dark-solace.org]
+## Site dedicated to these categories/characters/ships
+extracategories:The Walking Dead
+extracharacters:Carol,Daryl
+extraships:Carol/Daryl
+
 [nocturnal-light.net]
 ## Extra metadata that this adapter knows about.  See [dramione.org]
 ## for examples of how to use them.
@@ -1319,6 +1459,14 @@ universe_as_series: true
 ## cover image.  This lets you exclude them.
 cover_exclusion_regexp:/css/bir.png
 
+[tgstorytime.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Transgender
+
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.
+#is_adult:true
+
 [thehexfiles.net]
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter
@@ -1338,11 +1486,6 @@ extraships:Harry Potter/Draco Malfoy
 ## personal.ini, not defaults.ini.
 #is_adult:true
 
-## Virtually all eFiction-based sites allow downloading the whole story in
-## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
-## metadata and chapters can be loaded in one step
-bulk_load:true
-
 ## Site dedicated to these categories/characters/ships
 extracategories:Criminal Minds
 
@@ -1352,11 +1495,6 @@ extracategories:Criminal Minds
 ## personal.ini, not defaults.ini.
 #is_adult:true
 
-## Virtually all eFiction-based sites allow downloading the whole story in
-## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
-## metadata and chapters can be loaded in one step
-bulk_load:true
-
 extra_valid_entries: readings,challenge
 extra_titlepage_entries: readings,challenge
 challenge_label: Challenge
@@ -1425,6 +1563,8 @@ extra_titlepage_entries:readings,awards
 awards_label:Awards
 readings_label:Readings
 
+## some sites include images that we don't ever want becoming the
+## cover image.  This lets you exclude them.
 cover_exclusion_regexp:art/.*Awards.jpg
 
 [voracity2.e-fic.com]
@@ -1536,7 +1676,7 @@ extra_valid_entries:reviews,favs,follows
 ## to be romantic pairings.
 ships_label:Pairings
 
-## Date formats used by FFDL.  Published and Update don't usually have
+## Date formats used by FanFicFare.  Published and Update don't usually have
 ## time, but they do now on ffnet.
 ## See http://docs.python.org/library/datetime.html#strftime-strptime-behavior
 ## Note that ini format requires % to be escaped as %%.
@@ -1590,7 +1730,7 @@ cover_exclusion_regexp:(/imageu/|d_60_90\.jpg)
 ## reducing this sleep.
 slow_down_sleep_time:4
 
-## Date formats used by FFDL.  Published and Update don't usually have
+## Date formats used by FanFicFare.  Published and Update don't usually have
 ## time, but they do now on ffnet.
 ## See http://docs.python.org/library/datetime.html#strftime-strptime-behavior
 ## Note that ini format requires % to be escaped as %%.
@@ -1610,7 +1750,7 @@ dateUpdated_format:%%Y-%%m-%%d %%H:%%M:%%S
 #fail_on_password: false
 
 ## fimfiction.net stories allow chapters to be added out of order.  So
-## the newest chapter may not be the last one.  FFDL update doesn't
+## the newest chapter may not be the last one.  FanFicFare update doesn't
 ## like that.  If do_update_hook is uncommented and set true, the
 ## adapter will discard all existing chapters from the newest one on
 ## when updating to enforce accurate chapters.
@@ -1621,6 +1761,10 @@ dateUpdated_format:%%Y-%%m-%%d %%H:%%M:%%S
 ## to on, but can be switched off if it is found to cause problems.
 fix_fimf_blockquotes:true
 
+## some sites include images that we don't ever want becoming the
+## cover image.  This lets you exclude them.
+cover_exclusion_regexp:/images/emoticons/
+
 ## Site dedicated to these categories/characters/ships
 extracategories:My Little Pony: Friendship is Magic
 
@@ -1704,10 +1848,27 @@ extraships:InuYasha/Kagome
 ## Site dedicated to these categories/characters/ships
 extracategories:Lord of the Rings
 
-## Virtually all eFiction-based sites allow downloading the whole story in
-## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
-## metadata and chapters can be loaded in one step
-bulk_load:true
+[www.masseffect2.in]
+## Site dedicated to this fandom.
+extracategories: Mass Effect
+
+## Ucoz.com engine, upon which MassEffect2.in is based, imposes an unspecified limit on request frequency.
+## Reports vary from `5 requests per second' to `2 requests per second for more than 10 per minute'.
+## With default settings, a several-hours IP ban may follow, so set it higher.
+slow_down_sleep_time: 2
+
+## Whether to exclude editor signature from the bottom of chapter text.
+exclude_editor_signature: false
+
+## Stories on the site almost never have cover image, and for the stories which do,
+## this may be adjusted in `personal.ini' before downloading.
+never_make_cover: true
+
+## Titles for ratings identified by 1- or 2-letter codes from `ERATING system'
+## (`система Р.Е.Й.Т.И.Н.Г.').  MassEffect2.in and some other sites adopted it,
+## but changed titles and update them occasionally.
+rating_titles: R=RESTRICTED (16+), E=EXEMPT (18+), I=ART HOUSE, T=To every, A=IN=Иной мир, Nn=Новый мир, G=О\, Господи!
+adult_ratings: E,R
 
 [www.mediaminer.org]
 
@@ -1743,7 +1904,7 @@ extraships:Nick Stokes/Greg Sanders
 
 ## phoenixsong.net, oddly, can have high rated chapters (login
 ## required) in the middle of a lower rated story.  Use this to force
-## FFDL to always login to phoenixsong.net so those stories download
+## FanFicFare to always login to phoenixsong.net so those stories download
 ## correctly.  If you have a login, this is recommended.
 #force_login:true
 

calibre-plugin/prefs.py

@@ -1,5 +1,5 @@
-#!/usr/bin/env python
-# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+# -*- coding: utf-8 -*-
+
 from __future__ import (unicode_literals, division, absolute_import,
                         print_function)
 
@@ -15,9 +15,32 @@ import copy
 from calibre.utils.config import JSONConfig
 from calibre.gui2.ui import get_gui
 
-from calibre_plugins.fanficfare_plugin.dialogs import UPDATE
+from calibre_plugins.fanficfare_plugin.dialogs import SAVE_UPDATE
 from calibre_plugins.fanficfare_plugin.common_utils import get_library_uuid
 
+# Show translated strings, but save the same string in prefs so your
+# prefs are the same in different languages.
+YES=_('Yes, Always')
+SAVE_YES='Yes'
+YES_IF_IMG=_('Yes, if EPUB has a cover image')
+SAVE_YES_IF_IMG='Yes, if img'
+YES_UNLESS_IMG=_('Yes, unless FanFicFare found a cover image')
+SAVE_YES_UNLESS_IMG='Yes, unless img'
+NO=_('No')
+SAVE_NO='No'
+calcover_save_options = {
+    YES:SAVE_YES,
+    SAVE_YES:YES,
+    YES_IF_IMG:SAVE_YES_IF_IMG,
+    SAVE_YES_IF_IMG:YES_IF_IMG,
+    YES_UNLESS_IMG:SAVE_YES_UNLESS_IMG,
+    SAVE_YES_UNLESS_IMG:YES_UNLESS_IMG,
+    NO:SAVE_NO,
+    SAVE_NO:NO,
+    }
+updatecalcover_order=[YES,YES_IF_IMG,NO]
+gencalcover_order=[YES,YES_UNLESS_IMG,NO]
+
 # if don't have any settings for FanFicFarePlugin, copy from
 # predecessor FanFictionDownLoaderPlugin.
 FFDL_PREFS_NAMESPACE = 'FanFictionDownLoaderPlugin'
@@ -28,6 +51,7 @@ PREFS_KEY_SETTINGS = 'settings'
 # take from here.
 default_prefs = {}
 default_prefs['personal.ini'] = get_resources('plugin-example.ini')
+default_prefs['cal_cols_pass_in'] = False
 default_prefs['rejecturls'] = ''
 default_prefs['rejectreasons'] = '''Sucked
 Boring
@@ -35,7 +59,6 @@ Dup from another site'''
 default_prefs['reject_always'] = False
 
 default_prefs['updatemeta'] = True
-default_prefs['updatecover'] = False
 default_prefs['updateepubcover'] = False
 default_prefs['keeptags'] = False
 default_prefs['suppressauthorsort'] = False
@@ -46,14 +69,14 @@ default_prefs['autoconvert'] = False
 default_prefs['urlsfromclip'] = True
 default_prefs['updatedefault'] = True
 default_prefs['fileform'] = 'epub'
-default_prefs['collision'] = UPDATE
+default_prefs['collision'] = SAVE_UPDATE
 default_prefs['deleteotherforms'] = False
 default_prefs['adddialogstaysontop'] = False
-default_prefs['includeimages'] = False
 default_prefs['lookforurlinhtml'] = False
 default_prefs['checkforseriesurlid'] = True
 default_prefs['checkforurlchange'] = True
 default_prefs['injectseries'] = False
+default_prefs['matchtitleauth'] = True
 default_prefs['smarten_punctuation'] = False
 default_prefs['show_est_time'] = False
 
@@ -62,7 +85,13 @@ default_prefs['read_lists'] = ''
 default_prefs['addtolists'] = False
 default_prefs['addtoreadlists'] = False
 default_prefs['addtolistsonread'] = False
+default_prefs['autounnew'] = False
 
+default_prefs['updatecalcover'] = None
+default_prefs['gencalcover'] = SAVE_YES
+default_prefs['updatecover'] = False
+default_prefs['calibre_gen_cover'] = False
+default_prefs['plugin_gen_cover'] = True
 default_prefs['gcnewonly'] = False
 default_prefs['gc_site_settings'] = {}
 default_prefs['allow_gc_from_ini'] = True
@@ -72,11 +101,13 @@ default_prefs['countpagesstats'] = []
 default_prefs['wordcountmissing'] = False
 
 default_prefs['errorcol'] = ''
+default_prefs['savemetacol'] = ''
 default_prefs['custom_cols'] = {}
 default_prefs['custom_cols_newonly'] = {}
 default_prefs['allow_custcol_from_ini'] = True
 
 default_prefs['std_cols_newonly'] = {}
+default_prefs['set_author_url'] = True
 
 default_prefs['imapserver'] = ''
 default_prefs['imapuser'] = ''
@@ -84,6 +115,8 @@ default_prefs['imappass'] = ''
 default_prefs['imapsessionpass'] = False
 default_prefs['imapfolder'] = 'INBOX'
 default_prefs['imapmarkread'] = True
+default_prefs['auto_reject_from_email'] = False
+default_prefs['download_from_email_immediately'] = False
 
 def set_library_config(library_config,db):
     db.prefs.set_namespaced(PREFS_NAMESPACE,

calibre-plugin/translations/README.md

@@ -0,0 +1,6 @@
+# Translations
+
+If you're interested in helping provide translations for this project,
+please use the
+[Transifex](https://www.transifex.com/projects/p/calibre-plugins/resources/)
+website to add translations to this, or other calibre plugins that support it.

fanficfare/adapters/__init__.py

@@ -133,6 +133,13 @@ import adapter_lotrfanfictioncom
 import adapter_fhsarchivecom
 import adapter_fanfictionjunkiesde
 import adapter_devianthearts
+import adapter_tgstorytimecom
+import adapter_itcouldhappennet
+import adapter_forumsspacebattlescom
+import adapter_forumssufficientvelocitycom
+import adapter_questionablequestingcom
+import adapter_ninelivesdarksolaceorg
+import adapter_masseffect2in
 
 ## This bit of complexity allows adapters to be added by just adding
 ## importing.  It eliminates the long if/else clauses we used to need
@@ -192,19 +199,35 @@ def getAdapter(config,url,anyurl=False):
     # No adapter found.
     raise exceptions.UnknownSite( url, [cls.getSiteDomain() for cls in __class_list] )
 
-def getConfigSections():
+def getSiteSections():
+    # doesn't include base sections. Sections rather than site DNS because of squidge/peja
     return [cls.getConfigSection() for cls in __class_list]
 
+def getConfigSections():
+    # does include base sections.
+    sections = set()
+    for cls in __class_list:
+        sections.update(cls.getConfigSections())
+    return sections
+
+def get_bulk_load_sites():
+    # for now, all eFiction Base adapters are assumed to allow bulk_load.
+    sections = set()
+    for cls in filter( lambda x : issubclass(x,base_efiction_adapter.BaseEfictionAdapter),
+                       __class_list):
+        sections.update( [ x.replace('www.','') for x in cls.getConfigSections() ] )
+    return sections
+
 def getSiteExamples():
     l=[]
     for cls in sorted(__class_list, key=lambda x : x.getConfigSection()):
         l.append((cls.getConfigSection(),cls.getSiteExampleURLs().split()))
     return l
 
-def getConfigSectionFor(url):
+def getConfigSectionsFor(url):
     (cls,fixedurl) = getClassFor(url)
     if cls:
-        return cls.getConfigSection()
+        return cls.getConfigSections()
 
     # No adapter found.
     raise exceptions.UnknownSite( url, [cls.getSiteDomain() for cls in __class_list] )

fanficfare/adapters/adapter_archiveofourownorg.py

@@ -230,7 +230,6 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
         fandoms = a.findAll('a',{'class':"tag"})
         for fandom in fandoms:
             self.story.addToList('fandoms',fandom.string)
-            self.story.addToList('category',fandom.string)
 		
         a = metasoup.find('dd',{'class':"warning tags"})
         if a != None:
@@ -243,7 +242,6 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
             genres = a.findAll('a',{'class':"tag"})
             for genre in genres:
                 self.story.addToList('freeformtags',genre.string)
-                self.story.addToList('genre',genre.string)
                 
         a = metasoup.find('dd',{'class':"category tags"})
         if a != None:
@@ -251,7 +249,6 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
             for genre in genres:
                 if genre != "Gen":
                     self.story.addToList('ao3categories',genre.string)
-                    self.story.addToList('genre',genre.string)
 		
         a = metasoup.find('dd',{'class':"character tags"})
         if a != None:

fanficfare/adapters/adapter_devianthearts.py

@@ -32,21 +32,6 @@ class DeviantHeartsAdapter(BaseEfictionAdapter):
     @classmethod
     def getDateFormat(self):
         return "%m/%d/%y" 
-   
-    # def handleMetadataPair(self, key, value):
-    #     if key == 'Warnings':
-    #         for val in re.split("\s*,\s*", value):
-    #             if value == 'None':
-    #                 return
-    #             else:
-    #                 # toss numbers only.
-    #                 self.story.addToList('warnings', filter(lambda x : not x.isdigit() , val))
-     
-    # #     elif 'Categories' in key:
-    # #         for val in re.split("\s*>\s*", value):
-    # #             self.story.addToList('category', val)
-    #     else:
-    #         super(FHSArchiveComAdapter, self).handleMetadataPair(key, value) 
                            
 def getClass():
     return DeviantHeartsAdapter

fanficfare/adapters/adapter_dramioneorg.py

@@ -38,8 +38,9 @@ class DramioneOrgAdapter(BaseSiteAdapter):
     def __init__(self, config, url):
         BaseSiteAdapter.__init__(self, config, url)
 
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
+        self.decode = ["utf8",
+                       "Windows-1252",]
+                               # 1252 is a superset of iso-8859-1.
                                # Most sites that claim to be
                                # iso-8859-1 (and some that claim to be
                                # utf8) are really windows-1252.

fanficfare/adapters/adapter_fanfictionnet.py

@@ -104,7 +104,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
             data = self._fetchUrl(url)
             #logger.debug("\n===================\n%s\n===================\n"%data)
             soup = self.make_soup(data)
-        except urllib2.HTTPError, e:
+        except urllib2.HTTPError as e:
             if e.code == 404:
                 raise exceptions.StoryDoesNotExist(url)
             else:
@@ -135,11 +135,15 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
                                                 chapcount+1)
                 logger.debug('=Trying newer chapter: %s' % tryurl)
                 newdata = self._fetchUrl(tryurl)
-                if "not found. Please check to see you are not using an outdated url." \
-                        not in newdata:
+                if "not found. Please check to see you are not using an outdated url." not in newdata \
+                        and "This request takes too long to process, it is timed out by the server." not in newdata:
                     logger.debug('=======Found newer chapter: %s' % tryurl)
                     soup = self.make_soup(newdata)
-            except:
+            except urllib2.HTTPError as e:
+                if e.code == 503:
+                    raise e
+            except e:
+                logger.warn("Caught an exception reading URL: %s sleeptime(%s) Exception %s."%(unicode(url),sleeptime,unicode(e)))
                 pass
             
         # Find authorid and URL from... author url.

fanficfare/adapters/adapter_ficwadcom.py

@@ -135,7 +135,7 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
         self.story.setMetadata('title', stripHTML(titleh4.a))
 
         # Find authorid and URL from... author url.
-        a = soup.find('span',{'class':'author'}).find('a', href=re.compile(r"^/author/\d+"))
+        a = soup.find('span',{'class':'author'}).find('a', href=re.compile(r"^/a/"))
         self.story.setMetadata('authorId',a['href'].split('/')[2])
         self.story.setMetadata('authorUrl','http://'+self.host+a['href'])
         self.story.setMetadata('author',a.string)

fanficfare/adapters/adapter_fimfictionnet.py

@@ -176,7 +176,10 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
             if coverurl.startswith('//'): # fix for img urls missing 'http:'
                 coverurl = "http:"+coverurl
             if get_cover:
-                self.setCoverImage(self.url,coverurl)
+                # try setting from href, if fails, try using the img src
+                if self.setCoverImage(self.url,coverurl)[0] == "failedtoload":
+                    coverurl = storyImage.find('img')['src']
+                    self.setCoverImage(self.url,coverurl)
 
             coverSource = storyImage.find('a', {'class':'source'})
             if coverSource:

fanficfare/adapters/adapter_forumsspacebattlescom.py

@@ -0,0 +1,35 @@
+#  -*- coding: utf-8 -*-
+
+# Copyright 2015 FanFicFare team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from base_xenforoforum_adapter import BaseXenForoForumAdapter
+
+def getClass():
+    return ForumsSpacebattlesComAdapter
+
+class ForumsSpacebattlesComAdapter(BaseXenForoForumAdapter):
+
+    def __init__(self, config, url):
+        BaseXenForoForumAdapter.__init__(self, config, url)
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','fsb')
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'forums.spacebattles.com'
+

fanficfare/adapters/adapter_forumssufficientvelocitycom.py

@@ -0,0 +1,34 @@
+#  -*- coding: utf-8 -*-
+
+# Copyright 2015 FanFicFare team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from base_xenforoforum_adapter import BaseXenForoForumAdapter
+
+def getClass():
+    return ForumsSufficientVelocityComAdapter
+
+class ForumsSufficientVelocityComAdapter(BaseXenForoForumAdapter):
+
+    def __init__(self, config, url):
+        BaseXenForoForumAdapter.__init__(self, config, url)
+
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','fsv')
+
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'forums.sufficientvelocity.com'

fanficfare/adapters/adapter_grangerenchantedcom.py

@@ -176,7 +176,7 @@ class GrangerEnchantedCom(BaseSiteAdapter):
             raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
             
         # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
+        soup = self.make_soup(data)
         # print data
 
         # Now go hunting for all the meta data and the chapter list.
@@ -217,11 +217,10 @@ class GrangerEnchantedCom(BaseSiteAdapter):
             if 'Summary' in label:
                 ## Everything until the next span class='label'
                 svalue = ""
-                while not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
+                while value and not defaultGetattr(value,'class') == 'label' and '<span class="label">' not in unicode(value):
+                    svalue += unicode(value)
                     value = value.nextSibling
                 self.setDescription(url,svalue)
-                #self.story.setMetadata('description',stripHTML(svalue))
 
             if 'Rated' in label:
                 self.story.setMetadata('rating', value)
@@ -271,7 +270,7 @@ class GrangerEnchantedCom(BaseSiteAdapter):
             series_url = 'http://'+self.host+'/'+self.section+'/'+a['href']
 
             # use BeautifulSoup HTML parser to make everything easier to find.
-            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            seriessoup = self.make_soup(self._fetchUrl(series_url))
             # can't use ^viewstory...$ in case of higher rated stories with javascript href.
             storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
             i=1
@@ -300,8 +299,7 @@ class GrangerEnchantedCom(BaseSiteAdapter):
 
         logger.debug('Getting chapter text from: %s' % url)
 
-        soup = bs.BeautifulSoup(self._fetchUrl(url),
-                                selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        soup = self.make_soup(self._fetchUrl(url))
         
         div = soup.find('div', {'id' : 'story1'})
 

fanficfare/adapters/adapter_itcouldhappennet.py

@@ -0,0 +1,46 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2015 FanFicFare team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Software: eFiction
+import re
+from base_efiction_adapter import BaseEfictionAdapter
+
+class ItCouldHappenNetSiteAdapter(BaseEfictionAdapter):
+
+    @staticmethod
+    def getSiteDomain():
+        return 'it-could-happen.net'
+
+    @classmethod
+    def getSiteAbbrev(seluuf):
+        return 'ich'
+
+    @classmethod
+    def getDateFormat(self):
+        return "%B %d, %Y"
+
+    def handleMetadataPair(self, key, value):
+        # This site is all one 'category' as it's usually defined and
+        # uses Category for what is usually genre.
+        if key == 'Categories':
+            for val in re.split("\s*,\s*", value):
+                self.story.addToList('genre', val)
+        else:
+            super(ItCouldHappenNetSiteAdapter, self).handleMetadataPair(key, value)
+
+def getClass():
+    return ItCouldHappenNetSiteAdapter

fanficfare/adapters/adapter_masseffect2in.py

@@ -0,0 +1,719 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2015 FanFicFare team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import bs4
+import datetime
+import logging
+import re
+import urllib2
+
+from ..htmlcleanup import removeEntities, stripHTML
+from .. import exceptions as exceptions
+from base_adapter import BaseSiteAdapter, makeDate
+
+
+_logger = logging.getLogger(__name__)
+
+
+def getClass():
+    """Returns adapter class defined in this module."""
+    return MassEffect2InAdapter
+
+
+class ParsingError(Exception):
+    """Indicates an error while parsing web page content."""
+    def __init__(self, message):
+        Exception.__init__(self)
+        self.message = message
+
+    def __str__(self):
+        return self.message
+
+
+class MassEffect2InAdapter(BaseSiteAdapter):
+    """
+    Provides support for MassEffect2.in site as story source.
+    Can be used as a template for sites build upon Ucoz.com engine (until no base class extracted).
+    Specializations:
+        1) Russian content (date format, genre names, etc.);
+        2) original `E.R.A.T.I.N.G.' rating scale, used by masseffect2.in
+           and some affiliated sites, denoted with images;
+        3) editor signatures an an option to remove them.
+    """
+
+    WORD_PATTERN = re.compile(u'\w+', re.UNICODE)
+    DOCUMENT_ID_PATTERN = re.compile(u'\d+-\d+-\d+-\d+')
+    SITE_LANGUAGE = u'Russian'
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["utf8"]
+
+        self.story.setMetadata('siteabbrev', 'me2in')
+        self.story.setMetadata('storyId', self._getDocumentId(self.url))
+
+        self._setURL(self._makeDocumentUrl(self.story.getMetadata('storyId')))
+
+        self._chapters = {}
+        self._parsingConfiguration = None
+
+    # Must be @staticmethod, not @classmethod!
+    @staticmethod
+    def getSiteDomain():
+        return 'www.masseffect2.in'
+
+    @classmethod
+    def getSiteExampleURLs(cls):
+        return u' '.join([cls._makeDocumentUrl('19-1-0-1234'),
+                          cls._makeDocumentUrl('24-1-0-4321')])
+
+    def getSiteURLPattern(self):
+        return re.escape(self._makeDocumentUrl('')) + self.DOCUMENT_ID_PATTERN.pattern
+
+    def use_pagecache(self):
+        """Allows use of downloaded page cache.  It is essential for this
+        adapter, because the site does not offers chapter URL list, and many
+        pages have to be fetched and parsed repeatedly."""
+        return True
+
+    def extractChapterUrlsAndMetadata(self):
+        """Extracts chapter URLs and story metadata.  Actually downloads all
+        chapters, which is not exactly right, but necessary due to technical
+        limitations of the site."""
+
+        def followChapters(starting, forward=True):
+            if forward:
+                url = starting.getNextChapterUrl()
+            else:
+                url = starting.getPreviousChapterUrl()
+            if url:
+                url = self._makeDocumentUrl(self._getDocumentId(url))
+                following = self._makeChapter(url)
+                # Do not follow links to related, but different stories (prequels or sequels).
+                if not following.isFromStory(starting.getHeading()):
+                    return
+                if forward:
+                    yield following
+                for chapter in followChapters(following, forward):
+                    yield chapter
+                if not forward:
+                    yield following
+
+        try:
+            startingChapter = self._makeChapter(self.url)
+        except urllib2.HTTPError, error:
+            if error.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            raise
+
+        # We only have one date for each chapter and assume the oldest one
+        # to be publication date and the most recent one to be update date.
+        datePublished = datetime.datetime.max
+        dateUpdated = datetime.datetime.min
+        wordCount = 0
+        # We aim at counting chapters, not chapter parts.
+        chapterCount = 0
+        storyInProgress = False
+
+        chapters = \
+            list(followChapters(startingChapter, forward=False)) + \
+            [startingChapter] + \
+            list(followChapters(startingChapter, forward=True))
+
+        headings = [chapter.getHeading() for chapter in chapters]
+        largestCommonPrefix = _getLargestCommonPrefix(*headings)
+        prefixLength = len(largestCommonPrefix)
+        storyTitleEnd, chapterTitleStart = prefixLength, prefixLength
+        match = re.search(u'[:\.\s]*(?P<chapter>глава\s+)?$', largestCommonPrefix, re.IGNORECASE | re.UNICODE)
+        if match:
+            storyTitleEnd -= len(match.group())
+            label = match.group('chapter')
+            if label:
+                chapterTitleStart -= len(label)
+        storyTitle = largestCommonPrefix[:storyTitleEnd]
+        self.story.setMetadata('title', storyTitle)
+
+        garbagePattern = re.compile(u'(?P<start>^)?[:\.\s]*(?(start)|$)', re.UNICODE)
+        indexPattern = re.compile(u'(?:глава\s)?(?:(?<!\d)(?P<index>\d{1,3})(?=\D|$))', re.IGNORECASE | re.UNICODE)
+
+        for chapter in chapters:
+            url = chapter.getUrl()
+            self._chapters[url] = chapter
+            _logger.debug(u"Processing chapter `%s'.", url)
+
+            try:
+                authorName = chapter.getAuthorName()
+                if authorName:
+                    self.story.extendList('author', [authorName])
+                    authorId = chapter.getAuthorId()
+                    if authorId:
+                        authorUrl = 'http://%s/index/%s' % (self.getSiteDomain(), authorId)
+                    else:
+                        authorId = u''
+                        authorUrl = u''
+                    self.story.extendList('authorId', [authorId])
+                    self.story.extendList('authorUrl', [authorUrl])
+
+                if not self.story.getMetadata('rating'):
+                    ratingTitle = chapter.getRatingTitle()
+                    if ratingTitle:
+                        self.story.setMetadata('rating', ratingTitle)
+
+                if not self.story.getMetadata('description'):
+                    summary = chapter.getSummary()
+                    if summary:
+                        self.story.setMetadata('description', summary)
+
+                datePublished = min(datePublished, chapter.getDate())
+                dateUpdated = max(dateUpdated, chapter.getDate())
+
+                self.story.extendList('genre', chapter.getGenres())
+                self.story.extendList('characters', chapter.getCharacters())
+                self.story.extendList('ships', chapter.getPairings())
+
+                wordCount += self._getWordCount(chapter.getTextElement())
+
+                # Chapter status usually represents the story status, so we want the last chapter status.
+                # Some chapters may have no status attribute.
+                chapterInProgress = chapter.isInProgress()
+                if chapterInProgress is not None:
+                    storyInProgress = chapterInProgress
+
+                # If any chapter is adult, consider the whole story adult.
+                if chapter.isAdult():
+                    self.story.setMetadata('is_adult', True)
+                    warning = chapter.getWarning()
+                    if warning:
+                        self.story.extendList('warnings', [warning])
+
+                chapterTitle = re.sub(garbagePattern, u'', chapter.getHeading()[chapterTitleStart:])
+
+                match = re.search(indexPattern, chapterTitle)
+                if match:
+                    index = int(match.group('index'))
+                    chapterCount = max(chapterCount, index)
+                else:
+                    chapterCount += 1
+
+                self.chapterUrls.append((chapterTitle, url))
+            except ParsingError, error:
+                raise exceptions.FailedToDownload(u"Failed to download chapter `%s': %s" % (url, error))
+
+        # Some metadata are handled separately due to format conversions.
+        self.story.setMetadata('status', 'In Progress' if storyInProgress else 'Completed')
+        self.story.setMetadata('datePublished', datePublished)
+        self.story.setMetadata('dateUpdated', dateUpdated)
+        self.story.setMetadata('numWords', str(wordCount))
+        self.story.setMetadata('numChapters', chapterCount)
+
+        # Site-specific metadata.
+        self.story.setMetadata('language', self.SITE_LANGUAGE)
+
+    def getChapterText(self, url):
+        """Grabs the text for an individual chapter."""
+        if url not in self._chapters:
+            raise exceptions.FailedToDownload(u"No chapter `%s' present!" % url)
+        chapter = self._chapters[url]
+        return self.utf8FromSoup(url, chapter.getTextElement())
+
+    def _makeChapter(self, url):
+        """Creates a chapter object given a URL."""
+        document = self.make_soup(self._fetchUrl(url))
+        chapter = Chapter(self._getParsingConfiguration(), url, document)
+        return chapter
+
+    def _getWordCount(self, element):
+        """Returns word count in plain text extracted from chapter body."""
+        text = stripHTML(element)
+        count = len(re.findall(self.WORD_PATTERN, text))
+        return count
+
+    def _getParsingConfiguration(self):
+        if not self._parsingConfiguration:
+            self._parsingConfiguration = {}
+
+            adultRatings = self.getConfigList('adult_ratings')
+            if not adultRatings:
+                raise exceptions.PersonalIniFailed(
+                    u"Missing `adult_ratings' setting", u"MassEffect2.in", u"?")
+            adultRatings = set(adultRatings)
+            self._parsingConfiguration['adultRatings'] = adultRatings
+
+            ratingTitleDescriptions = self.getConfigList('rating_titles')
+            if ratingTitleDescriptions:
+                ratingTitles = {}
+                for ratingDescription in ratingTitleDescriptions:
+                    parts = ratingDescription.split(u'=')
+                    if len(parts) < 2:
+                        _logger.warning(
+                            u"Invalid `rating_titles' setting, missing `=' in `%s'."
+                            % ratingDescription)
+                        continue
+                    labels = parts[:-1]
+                    title = parts[-1]
+                    for label in labels:
+                        ratingTitles[label] = title
+                        # Duplicate label aliasing in adult rating set.
+                        if label in adultRatings:
+                            adultRatings.add(*labels)
+                self._parsingConfiguration['adultRatings'] = list(adultRatings)
+                self._parsingConfiguration['ratingTitles'] = ratingTitles
+            else:
+                raise exceptions.PersonalIniFailed(
+                    u"Missing `rating_titles' setting", u"MassEffect2.in", u"?")
+
+            self._parsingConfiguration['excludeEditorSignature'] = \
+                self.getConfig('exclude_editor_signature', False)
+
+        return self._parsingConfiguration
+
+    def _getDocumentId(self, url):
+        """Extract document ID from MassEffect2.in URL."""
+        match = re.search(self.DOCUMENT_ID_PATTERN, url)
+        if not match:
+            raise ValueError(u"Failed to extract document ID from `'" % url)
+        documentId = url[match.start():match.end()]
+        return documentId
+
+    @classmethod
+    def _makeDocumentUrl(cls, documentId):
+        """Make a chapter URL given a document ID."""
+        return 'http://%s/publ/%s' % (cls.getSiteDomain(), documentId)
+
+
+class Chapter(object):
+    """Represents a lazily-parsed chapter of a story."""
+    def __init__(self, configuration, url, document):
+        self._configuration = configuration
+        self._url = url
+        self._document = document
+        # Lazy-loaded:
+        self._heading = None
+        self._date = None
+        self._author = None
+        self._attributes = None
+        self._textElement = None
+        self._infoBar = None
+
+    def getHeading(self):
+        return self._extractHeading()
+
+    def getSummary(self):
+        attributes = self.__getAttributes()
+        if 'summary' in attributes:
+            return attributes['summary']
+
+    def getAuthorId(self):
+        author = self._getAuthor()
+        if author:
+            return author['id']
+
+    def getAuthorName(self):
+        author = self._getAuthor()
+        if author:
+            return author['name']
+
+    def getDate(self):
+        return self.__getDate()
+
+    def getRatingTitle(self):
+        attributes = self.__getAttributes()
+        if 'rating' in attributes:
+            return attributes['rating']['title']
+
+    def isAdult(self):
+        attributes = self.__getAttributes()
+        if 'rating' in attributes and attributes['rating']['isAdult']:
+            return True
+        if 'warning' in attributes:
+            return True
+        return False
+
+    def getWarning(self):
+        attributes = self.__getAttributes()
+        if 'warning' in attributes:
+            return attributes['warning']
+
+    def getCharacters(self):
+        return self._getListAttribute('characters')
+
+    def getPairings(self):
+        return self._getListAttribute('pairings')
+
+    def getGenres(self):
+        return self._getListAttribute('genres')
+
+    def isInProgress(self):
+        attributes = self.__getAttributes()
+        if 'isInProgress' in attributes:
+            return attributes['isInProgress']
+
+    def getUrl(self):
+        return self._url
+
+    def getTextElement(self):
+        return self._getTextElement()
+
+    def getPreviousChapterUrl(self):
+        """Download chapters following `Previous chapter' links.
+        Returns a list of chapters' URLs."""
+        return self._getSiblingChapterUrl({'class': 'fl tal'})
+
+    def getNextChapterUrl(self):
+        """Download chapters following `Next chapter' links.
+        Returns a list of chapters' URLs."""
+        return self._getSiblingChapterUrl({'class': 'tar fr'})
+
+    def isFromStory(self, storyTitle, prefixThreshold=-1):
+        """Check if this chapter is from a story different from the given one.
+        Prefix threshold specifies how long common story title prefix shall be
+        for chapters from one story: negative value means implementation-defined
+        optimum, zero inhibits the check, and positive value adjusts threshold."""
+
+        def getFirstWord(string):
+            match = re.search(u'^\s*\w+', string, re.UNICODE)
+            return string[match.start():match.end()]
+
+        thisStoryTitle = self.getHeading()
+        if prefixThreshold != 0:
+            if prefixThreshold < 0:
+                prefixThreshold = min(
+                    len(getFirstWord(storyTitle)), len(getFirstWord(thisStoryTitle)))
+            else:
+                prefixThreshold = min(
+                    prefixThreshold, len(storyTitle), len(thisStoryTitle))
+            result = len(_getLargestCommonPrefix(storyTitle, thisStoryTitle)) >= prefixThreshold
+            return result
+        else:
+            return storyTitle != thisStoryTitle
+
+    def _getListAttribute(self, name):
+        """Return an attribute value as a list or an empty list if the attribute is absent."""
+        attributes = self.__getAttributes()
+        if name in attributes:
+            return attributes[name]
+        return []
+
+    def _extractHeading(self):
+        """Extract header text from the document."""
+        return stripHTML(
+            self._document.find('div', {'class': 'eTitle'}).string)
+
+    def __getHeading(self):
+        """Lazily parse and return heading."""
+        if not self._heading:
+            self._heading = self._extractHeading()
+        return self._heading
+
+    def _getAuthor(self):
+        """Lazily parse and return author's information."""
+        if not self._author:
+            self._author = self._parseAuthor()
+        return self._author
+
+    def _parseAuthor(self):
+        """Locate and parse chapter author's information to a dictionary with author's `id' and `name'."""
+        try:
+            authorLink = self._getInfoBarElement() \
+                .find('i', {'class': 'icon-user'}) \
+                .findNextSibling('a')
+        except AttributeError:
+            raise ParsingError(u'Failed to locate author link.')
+        match = re.search(u'(8-\d+)', authorLink['onclick'])
+        if not match:
+            raise ParsingError(u'Failed to extract author ID.')
+        authorId = match.group(0)
+        authorName = stripHTML(authorLink.text)
+        return {
+            'id': authorId,
+            'name': authorName
+        }
+
+    def __getDate(self):
+        """Lazily parse chapter date."""
+        if not self._date:
+            self._date = self._parseDate()
+        return self._date
+
+    def _parseDate(self):
+        """Locate and parse chapter date."""
+        try:
+            dateText = self._getInfoBarElement() \
+                .find('i', {'class': 'icon-eye'}) \
+                .findPreviousSibling(text=True) \
+                .strip(u'| \n')
+        except AttributeError:
+            raise ParsingError(u'Failed to locate date.')
+        date = makeDate(dateText, '%d.%m.%Y')
+        return date
+
+    def _getInfoBarElement(self):
+        """Locate informational bar element, containing chapter date and author, on the page."""
+        if not self._infoBar:
+            self._infoBar = self._document.find('td', {'class': 'eDetails2'})
+            if not self._infoBar:
+                raise ParsingError(u'No informational bar found.')
+        return self._infoBar
+
+    def __getAttributes(self):
+        """Lazily parse attributes."""
+        if not self._attributes:
+            self._attributes = self._parseAttributes()
+        return self._attributes
+
+    def _parseAttributes(self):
+        """Parse chapter attribute block and return it as a dictionary with standard entries."""
+
+        attributes = {}
+        attributesText = u''
+        try:
+            starter = self._document \
+                .find('div', {'class': 'comm-div'}) \
+                .findNextSibling('div', {'class': 'cb'})
+            bound = starter.findNextSibling('div', {'class': 'cb'})
+
+            def processElement(element):
+                """Return textual representation an *inline* element of chapter attribute block."""
+                result = u''
+                if isinstance(element, bs4.Tag):
+                    if element.name in ('b', 'strong', 'font', 'br'):
+                        result += u"\n"
+                    if element.name == 's':
+                        result += u"<s>%s</s>" % stripHTML(element)
+                    else:
+                        result += stripHTML(element)
+                else:
+                    result += removeEntities(element)
+                return result
+
+            elements = starter.nextSiblingGenerator()
+            for element in elements:
+                if isinstance(element, bs4.Tag):
+                    if element == bound:
+                        break
+                    else:
+                        if element.name in ('div', 'p'):
+                            attributesText += u"\n"
+                            for child in element.childGenerator():
+                                attributesText += processElement(child)
+                            continue
+                attributesText += processElement(element)
+
+            elements = starter.nextGenerator()
+            for element in elements:
+                if isinstance(element, bs4.Tag):
+                    if element == bound:
+                        break
+                    elif element.name == 'img':
+                        rating = self._parseRatingFromImage(element)
+                        if rating:
+                            attributes['rating'] = rating
+                            break
+        except AttributeError or TypeError:
+            raise ParsingError(u'Failed to locate and collect attributes.')
+
+        separators = u"\r\n :;."
+        freestandingText = u''
+        for line in attributesText.split(u'\n'):
+            if line.count(u':') != 1:
+                freestandingText += line
+                continue
+            key, value = line.split(u':', 1)
+            key = key.strip(separators).lower()
+            value = value.strip().strip(separators)
+            parsed = self._parseAttribute(key, value)
+            for parsedKey, parsedValue in parsed.iteritems():
+                attributes[parsedKey] = parsedValue
+
+        freestandingText = freestandingText.strip()
+        if 'summary' not in attributes and freestandingText:
+            attributes['summary'] = freestandingText
+
+        if 'rating' not in attributes:
+            _logger.warning(u"Failed to locate or recognize rating for `%s'!", self.getUrl())
+
+        return attributes
+
+    # Most, but not all, URLs of rating icons match this.
+    RATING_LABEL_PATTERN = re.compile(u'/(?P<rating>[ERATINnG]+)\.png$')
+
+    def _parseRatingFromImage(self, element):
+        """Given an image element, try to parse story rating from it."""
+        # Although deprecated, `has_key()' is required here.
+        if not element.has_attr('src'):
+            return
+        source = element['src']
+        if 'REITiNG' in source:
+            match = re.search(self.RATING_LABEL_PATTERN, source)
+            if not match:
+                return
+            label = match.group('rating')
+            if label in self._configuration['ratingTitles']:
+                return {
+                    'label': label,
+                    'title': self._configuration['ratingTitles'][label],
+                    'isAdult': label in self._configuration['adultRatings']
+                }
+            else:
+                _logger.warning(u"No title found for rating label `%s'!" % label)
+        # TODO: conduct a research on such abnormal URLs.
+        elif source == 'http://www.masseffect2.in/_fr/10/1360399.png':
+            label = 'Nn'
+            return {
+                'label': 'Nn',
+                'title': self._configuration['ratingTitles'][label],
+                'isAdult': label in self._configuration['adultRatings']
+            }
+
+    # Various `et cetera' and `et al' forms in Russian texts.
+    # Intended to be used with whole strings!
+    ETC_PATTERN = re.compile(
+        u'''[и&]\s(?:
+              (?:т\.?\s?[пд]?\.?)|
+              (?:др(?:угие|\.)?)|
+              (?:пр(?:очие|\.)?)|
+              # Note: identically looking letters `K' and `o'
+              # below are from Latin and Cyrillic alphabets.
+              (?:ко(?:мпания)?|[KК][oо°])
+            )$
+        ''',
+        re.IGNORECASE + re.UNICODE + re.VERBOSE)
+
+    # `Author's Notes' and its variants in Russian.
+    ANNOTATION_PATTERN = re.compile(u'аннотация|описание|(?:(?:за|при)мечание\s)?(?:от\s)?автора', re.UNICODE)
+
+    def _parseAttribute(self, key, value):
+        """
+        Parse a single a single record in chapter attributes for chapter metadata.
+        Return a dictionary of canonical attributes and values (i. e. multiple attributes may be discovered).
+        """
+
+        def refineCharacter(name):
+            """Refines character name from stop-words and distortions."""
+            strippedName = name.strip()
+            nameOnly = re.sub(self.ETC_PATTERN, u'', strippedName)
+            # TODO: extract canonical name (even ME-specific?).
+            canonicalName = nameOnly
+            return canonicalName
+
+        if re.match(u'жанры?', key, re.UNICODE):
+            genres = filter(bool, map(unicode.strip, re.split(u'[,;/]', value)))
+            return {'genres': genres}
+        elif key == u'статус':
+            isInProgress = value == u'в процессе'
+            return {'isInProgress': isInProgress}
+        elif key == u'персонажи':
+            participants = map(refineCharacter, re.split(u'[,;]', value))
+            characters = []
+            pairings = []
+            for participant in participants:
+                if u'/' in participant:
+                    pairings.append(participant)
+                else:
+                    characters.append(participant)
+            return {
+                'characters': characters,
+                'pairings': pairings
+            }
+        elif key == u'предупреждение':
+            return {'warning': value}
+        elif re.match(self.ANNOTATION_PATTERN, key):
+            if not value.endswith(u'.'):
+                value += u'.'
+            # Capitalize would make value[1:] lowercase, which we don't want.
+            value = value[:1].upper() + value[1:]
+            return {'summary': value}
+        else:
+            _logger.info(u"Unrecognized attribute `%s' ignored.", key)
+            return {}
+
+    def _getTextElement(self):
+        """Locate chapter body text element on the page."""
+        if not self._textElement:
+            self._textElement = self.__collectTextElements()
+        return self._textElement
+
+    def __collectTextElements(self):
+        """Return all elements containing parts of chapter text (which may be
+        <p>aragraphs, <div>isions or plain text nodes) under a single root."""
+        starter = self._document.find('div', {'id': u'article'})
+        if starter is None:
+            # FIXME: This will occur if the method is called more than once.
+            # The reason is elements appended to `root' are removed from the document.
+            # BS 4.4 implements cloning via `copy.copy()', but supporting it for BS 4.3
+            # would be error-prone (due to relying on BS internals) and is not needed.
+            if self._textElement:
+                _logger.debug(u"You may not call this function more than once!")
+            raise ParsingError(u'Failed to locate text.')
+        collection = [starter]
+        for element in starter.nextSiblingGenerator():
+            if element is None:
+                break
+            if isinstance(element, bs4.Tag) and element.name == 'tr':
+                break
+            collection.append(element)
+        root = bs4.Tag(name='td')
+        for element in collection:
+            root.append(element)
+
+        if self._configuration['excludeEditorSignature']:
+            root = self._excludeEditorSignature(root)
+
+        return root
+
+    def _getSiblingChapterUrl(self, selector):
+        """Locate a link to a sibling chapter, either previous or next one, and return its URL."""
+        block = self._document \
+            .find('td', {'class': 'eDetails1'}) \
+            .find('div', selector)
+        if not block:
+            return
+        link = block.find('a')
+        if not link:
+            return
+        return link['href']
+
+    # Editor signature always starts with something like this.
+    SIGNED_PATTERN = re.compile(u'отредактирова(?:но|ла?)[:.\s]', re.IGNORECASE + re.UNICODE)
+
+    def _excludeEditorSignature(self, root):
+        """Exclude editor signature from within `root' element."""
+        for textNode in root.findAll(text=True):
+            if re.match(self.SIGNED_PATTERN, textNode.string):
+                editorLink = textNode.findNext('a')
+                if editorLink:
+                    editorLink.extract()
+                # Seldom editor link has inner formatting, which is sibling DOM-wise.
+                editorName = textNode.findNext('i')
+                if editorName:
+                    editorName.extract()
+                textNode.extract()
+                # We could try removing container element, but there is a risk
+                # of removing text ending with it.  Better play safe here.
+                break
+        return root
+
+
+def _getLargestCommonPrefix(*args):
+    """Returns largest common prefix of all unicode arguments, ignoring case.
+    :rtype : unicode
+    """
+    from itertools import takewhile, izip
+    toLower = lambda xs: map(lambda x: x.lower(), xs)
+    allSame = lambda xs: len(set(toLower(xs))) == 1
+    return u''.join([i[0] for i in takewhile(allSame, izip(*args))])

fanficfare/adapters/adapter_mediaminerorg.py

@@ -22,7 +22,6 @@ import re
 import urllib
 import urllib2
 
-from .. import BeautifulSoup as bs
 from ..htmlcleanup import stripHTML
 from .. import exceptions as exceptions
 
@@ -42,7 +41,12 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
         # get storyId from url--url validation guarantees query correct
         m = re.match(self.getSiteURLPattern(),url)
         if m:
-            self.story.setMetadata('storyId',m.group('id'))
+            if m.group('id'):
+                self.story.setMetadata('storyId',m.group('id'))
+            elif m.group('id2'):
+                self.story.setMetadata('storyId',m.group('id2'))
+            elif m.group('id3'):
+                self.story.setMetadata('storyId',m.group('id2'))
             
             # normalized story URL.
             self._setURL('http://' + self.getSiteDomain() + '/fanfic/view_st.php/'+self.story.getMetadata('storyId'))
@@ -62,8 +66,17 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
     def getSiteURLPattern(self):
         ##  http://www.mediaminer.org/fanfic/view_st.php/76882
         ##  http://www.mediaminer.org/fanfic/view_ch.php/167618/594087#fic_c
+        ##  http://www.mediaminer.org/fanfic/view_ch.php?submit=View+Chapter&id=105816&cid=357151
+        ##  http://www.mediaminer.org/fanfic/view_ch.php?cid=612153&submit=View+Chapter&id=171668
         return re.escape("http://"+self.getSiteDomain())+\
-            "/fanfic/view_(st|ch)\.php/"+r"(?P<id>\d+)(/\d+(#fic_c)?)?$"
+            r"/fanfic/view_(st|ch)\.php"+\
+            r"(/(?P<id>\d+)(/\d+(#fic_c)?)?/?|"+\
+            r"\?((submit=View(\+| )Chapter|id=(?P<id2>\d+)|cid=\d+)&?)+)"
+
+    # Override stripURLParameters so the id parameter won't get stripped
+    @classmethod
+    def stripURLParameters(cls, url):
+        return url
 
     def extractChapterUrlsAndMetadata(self):
 
@@ -71,7 +84,7 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
         logger.debug("URL: "+url)
 
         try:
-            data = self._fetchUrl(url)
+            data = self._fetchUrl(url+'/') # trailing / gets 'chapter list' page even for one-shots.
         except urllib2.HTTPError, e:
             if e.code == 404:
                 raise exceptions.StoryDoesNotExist(self.url)
@@ -79,7 +92,7 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
                 raise e
 
         # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
+        soup = self.make_soup(data)
 
         # [ A - All Readers ], strip '[' ']'
         ## Above title because we remove the smtxt font to get title.
@@ -106,18 +119,12 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
         title = soup.find('td',{'class':'ffh'})
         for font in title.findAll('font'):
             font.extract() # removes 'font' tags from inside the td.        
-        if title.has_key('colspan'):
+        if title.has_attr('colspan'):
             titlet = stripHTML(title)
         else:
             ## No colspan, it's part chapter title--even if it's a one-shot.
             titlet = ':'.join(stripHTML(title).split(':')[:-1]) # strip trailing 'Chapter X' or chapter title
         self.story.setMetadata('title',titlet)
-        ## The story title is difficult to reliably parse from the
-        ## story pages.  Getting it from the author page is, but costs
-        ## another fetch.
-        # authsoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
-        # titlea = authsoup.find('a',{'href':'/fanfic/view_st.php/'+self.story.getMetadata('storyId')})
-        # self.story.setMetadata('title',titlea.text)
 
         # save date from first for later.
         firstdate=None
@@ -137,7 +144,9 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
                 # save date from first for later.
                 if not firstdate:
                     firstdate = m.group(3)
-                self.chapterUrls.append((chapter,'http://'+self.host+'/fanfic/view_ch.php/'+self.story.getMetadata('storyId')+'/'+option['value']))
+                # http://www.mediaminer.org/fanfic/view_ch.php?cid=376587&submit=View+Chapter&id=105816
+                # self.chapterUrls.append((chapter,'http://'+self.host+'/fanfic/view_ch.php/'+self.story.getMetadata('storyId')+'/'+option['value']))
+                self.chapterUrls.append((chapter,'http://'+self.host+'/fanfic/view_ch.php?submit=View Chapter&id='+self.story.getMetadata('storyId')+'&cid='+option['value']))
         self.story.setMetadata('numChapters',len(self.chapterUrls))
 
         # category
@@ -193,38 +202,37 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
         logger.debug('Getting chapter text from: %s' % url)
 
         data=self._fetchUrl(url)
-        soup = bs.BeautifulStoneSoup(data,
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        soup = self.make_soup(data)
 
-        anchor = soup.find('a',{'name':'fic_c'})
+        header = soup.find('div',{'class':'post-meta clearfix '})
+        # print("data:%s"%data)
 
-        if None == anchor:
+        chapter=self.make_soup('<div class="story"></div>').find('div')
+
+        if None == header:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
 
         ## find divs with align=left, those are paragraphs in newer stories.
-        divlist = anchor.findAllNext('div',{'align':'left'})
+        divlist = header.findAllNext('div',{'align':'left'})
         if divlist:
             for div in divlist:
                 div.name='p' # convert to <p> mediaminer uses div with
                              # a margin for paragraphs.
-                anchor.append(div) # cheat!  stuff all the content
-                                   # divs into anchor just as a
-                                   # holder.
+                chapter.append(div)
                 del div['style']
                 del div['align']
-            anchor.name='div'
-            return self.utf8FromSoup(url,anchor)
+            return self.utf8FromSoup(url,chapter)
         
         else:
             logger.debug('Using kludgey text find for older mediaminer story.')
             ## Some older mediaminer stories are unparsable with BeautifulSoup.
             ## Really nasty formatting.  Sooo... Cheat!  Parse it ourselves a bit first.
             ## Story stuff falls between:
-            data = "<div id='HERE'>" + data[data.find('<a name="fic_c">'):] +"</div>"
-            soup = bs.BeautifulStoneSoup(data,
-                                         selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+            data = "<div id='HERE'>" + data[data.find('<div class="adWrap">'):data.find('<div class="addthis_sharing_toolbox">')] +"</div>"
+            soup = self.make_soup(data)
             for tag in soup.findAll('td',{'class':'ffh'}) + \
                     soup.findAll('div',{'class':'acl'}) + \
+                    soup.findAll('div',{'class':'adWrap'}) + \
                     soup.findAll('div',{'class':'footer smtxt'}) + \
                     soup.findAll('table',{'class':'tbbrdr'}):
                 tag.extract() # remove tag from soup.

fanficfare/adapters/adapter_nhamagicalworldsus.py

@@ -16,222 +16,24 @@
 #
 
 # Software: eFiction
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return NHAMagicalWorldsUsAdapter
+from base_efiction_adapter import BaseEfictionAdapter
 
 # Class name has to be unique.  Our convention is camel case the
 # sitename with Adapter at the end.  www is skipped.
-class NHAMagicalWorldsUsAdapter(BaseSiteAdapter):
+class NHAMagicalWorldsUsAdapter(BaseEfictionAdapter):
 
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        
-		
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','nha')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = " %d/%m/%y"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
+    @staticmethod
     def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
         return 'nha.magical-worlds.us'
 
     @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-        url = self.url
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-            
-        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
-        if m != None:
-            if self.is_adult or self.getConfig("is_adult"):
-                # We tried the default and still got a warning, so
-                # let's pull the warning number from the 'continue'
-                # link and reload data.
-                addurl = m.group(1)
-                # correct stupid & error in url.
-                addurl = addurl.replace("&","&")
-                url = self.url+'&index=1'+addurl
-                logger.debug("URL 2nd try: "+url)
-
-                try:
-                    data = self._fetchUrl(url)
-                except urllib2.HTTPError, e:
-                    if e.code == 404:
-                        raise exceptions.StoryDoesNotExist(self.url)
-                    else:
-                        raise e    
-            else:
-                raise exceptions.AdultCheckRequired(self.url)
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-            
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
-		
-        try:
-            # in case link points somewhere other than the first chapter
-            a = soup.findAll('option')[1]['value']
-            self.story.setMetadata('storyId',a.split('=',)[1])
-            url = 'http://'+self.host+'/'+a
-            soup = bs.BeautifulSoup(self._fetchUrl(url))
-        except:
-            pass
-		
-        for info in asoup.findAll('table', {'width' : '100%', 'bordercolor' : re.compile(r'#')}):
-            a = info.find('a')
-            if 'viewstory.php?sid='+self.story.getMetadata('storyId') == a['href'] or \
-                    ('viewstory.php?sid='+self.story.getMetadata('storyId')+'&') in a['href']:
-                self.story.setMetadata('title',stripHTML(a))
-                break
-		
-
-        # Find the chapters:
-        chapters=soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+'&chapter=\d+$'))
-        if len(chapters) == 0:
-            self.chapterUrls.append((self.story.getMetadata('title'),url))
-        else:
-            for chapter in chapters:
-                # just in case there's tags, like <i> in chapter titles.
-                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d):
-            try:
-                return d.name
-            except:
-                return ""
-				
-        cats = info.findAll('a',href=re.compile('categories.php'))
-        for cat in cats:
-            self.story.addToList('category',cat.string)
-					
-        a = info.find('a', href=re.compile(r'viewuser.php'))
-        val = a.nextSibling
-        svalue = ""
-        while not defaultGetattr(val) == 'br':
-            val = val.nextSibling
-        val = val.nextSibling
-        while not defaultGetattr(val) == 'br':
-            svalue += unicode(val)
-            val = val.nextSibling
-        self.setDescription(url,svalue)
+    def getSiteAbbrev(self):
+        return 'nha'
         
-        #does not provide convenient way to get word count
-        labels = info.findAll('i')
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = stripHTML(labelspan)
-
-            if 'Rating' in label:
-                self.story.setMetadata('rating', value.split(' -')[0])
-
-            if 'Genres' in label:
-                genres = value.string.split(', ')
-                for genre in genres:
-                    if 'None' not in genre:
-                        self.story.addToList('genre',genre.split(' -')[0])
-            
-            if 'Characters' in label:
-                chars = value.string.split(', ')
-                for char in chars:
-                    if 'None' not in char:
-                        self.story.addToList('characters',char.split(' -')[0])
-
-            if 'Warnings' in label:
-                warnings = value.string.split(', ')
-                for warning in warnings:
-                    if 'None' not in warning:
-                        self.story.addToList('warnings',warning.split(' -')[0])
-
-            if 'Completed' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(value.split(' -')[0], self.dateformat))
-            
-            if 'Updated' in label:
-                # there's a stray [ at the end.
-                #value = value[0:-1]
-                self.story.setMetadata('dateUpdated', makeDate(value.split(' -')[0], self.dateformat))
-
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-		
-        data = self._fetchUrl(url)
-
-        soup = bs.BeautifulSoup(data, selfClosingTags=('br','hr','span','center')) # some chapters seem to be hanging up on those tags, so it is safer to close them
-        
-        story = soup.find('div', {"id" : "story"})
-
-        if None == story:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    @classmethod
+    def getDateFormat(self):
+        return "%d/%m/%y" 
+                           
+def getClass():
+    return NHAMagicalWorldsUsAdapter
     
-        return self.utf8FromSoup(url,story)

fanficfare/adapters/adapter_ninelivesdarksolaceorg.py

@@ -0,0 +1,38 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2015 Fanficdownloader team, 2015 FanFicFare team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Software: eFiction
+import re
+from base_efiction_adapter import BaseEfictionAdapter
+
+class NineLivesDarkSolaceAdapter(BaseEfictionAdapter):
+
+    @staticmethod
+    def getSiteDomain():
+        return 'ninelives.dark-solace.org'
+
+    @classmethod
+    def getSiteAbbrev(self):
+        return '9lvs'
+        
+    @classmethod
+    def getDateFormat(self):
+        return "%B %d, %Y" 
+                           
+def getClass():
+    return NineLivesDarkSolaceAdapter
+    

fanficfare/adapters/adapter_ponyfictionarchivenet.py

@@ -171,7 +171,8 @@ class PonyFictionArchiveNetAdapter(BaseSiteAdapter):
             self.story.addToList('warnings',warning.string)
         
         status = soup.find('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
-        self.story.setMetadata('status',status.string)
+        if status: # apparently this site can have stories with neither In-Progress or Complete.
+            self.story.setMetadata('status',status.string)
         
         section = soup.findAll('span', {'class' : 'General'})[1]            
         

fanficfare/adapters/adapter_portkeyorg.py

@@ -258,6 +258,7 @@ class PortkeyOrgAdapter(BaseSiteAdapter): # XXX
         data = self._fetchUrl(url)
 
         data = data.replace("HTML>","div>")
+        data = data.replace("html>","div>")
         
         soup = bs.BeautifulSoup(data)
 

fanficfare/adapters/adapter_questionablequestingcom.py

@@ -0,0 +1,35 @@
+#  -*- coding: utf-8 -*-
+
+# Copyright 2015 FanFicFare team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from base_xenforoforum_adapter import BaseXenForoForumAdapter
+
+def getClass():
+    return QuestionablequestingComAdapter
+
+class QuestionablequestingComAdapter(BaseXenForoForumAdapter):
+
+    def __init__(self, config, url):
+        BaseXenForoForumAdapter.__init__(self, config, url)
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','qq')
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'questionablequesting.com'
+

fanficfare/adapters/adapter_spikeluvercom.py

@@ -106,7 +106,9 @@ class SpikeluverComAdapter(BaseSiteAdapter):
 
         listbox_tag = soup.find('div', {'class': 'listbox'})
         for span_tag in listbox_tag('span'):
-            key = span_tag.string.strip(' :')
+            key = span_tag.string
+            if key:
+                key = key.strip(' :')
             try:
                 value = stripHTML(span_tag.nextSibling)
             # This can happen with some fancy markup in the summary. Just
@@ -135,8 +137,10 @@ class SpikeluverComAdapter(BaseSiteAdapter):
                         contents.append(sibling)
 
                 # Remove the preceding break line tag and other crud
-                contents.pop()
-                contents.pop()
+                if contents:
+                    contents.pop()
+                if contents:
+                    contents.pop()
                 self.story.setMetadata('description', ''.join(contents))
 
             elif key == 'Rated':

fanficfare/adapters/adapter_storiesonlinenet.py

@@ -21,7 +21,7 @@ logger = logging.getLogger(__name__)
 import re
 import urllib2
 
-from .. import BeautifulSoup as bs
+#from .. import BeautifulSoup as bs
 from ..htmlcleanup import stripHTML
 from .. import exceptions as exceptions
 
@@ -40,22 +40,22 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
         self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
         self.password = ""
         self.is_adult=False
-        
+
         # get storyId from url
         self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2].split(':')[0])
         if 'storyInfo' in self.story.getMetadata('storyId'):
             self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        
+
         # normalized story URL.
         self._setURL('http://' + self.getSiteDomain() + '/s/'+self.story.getMetadata('storyId'))
-        
+
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','strol')
 
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
         self.dateformat = "%Y-%m-%d"
-            
+
     @staticmethod # must be @staticmethod, don't remove it.
     def getSiteDomain():
         # The site domain.  Does have www here, if it uses it.
@@ -78,7 +78,7 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
                 or "Access to unlinked chapters requires" in data:
             self.needToLogin = True
         return self.needToLogin
-        
+
     def performLogin(self, url):
         params = {}
 
@@ -91,13 +91,13 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
         params['rememberMe'] = '1'
         params['page'] = 'http://'+self.getSiteDomain()+'/'
         params['submit'] = 'Login'
-    
+
         loginUrl = 'http://' + self.getSiteDomain() + '/login.php'
         logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['theusername']))
-    
+
         d = self._fetchUrl(loginUrl, params,usecache=False)
-    
+
         if "My Account" not in d : #Member Account
             logger.info("Failed to login to URL %s as %s" % (loginUrl,
                                                               params['theusername']))
@@ -114,7 +114,7 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
         return True
 
     ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
+    def doExtractChapterUrlsAndMetadata(self, get_cover=True):
 
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
@@ -137,22 +137,22 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
             # need to log in for this one.
             self.performLogin(url)
             data = self._fetchUrl(url+":i",usecache=False)
-        
+
         if "Access denied. This story has not been validated by the adminstrators of this site." in data:
             raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
         elif "Error! The story you're trying to access is being filtered by your choice of contents filtering." in data:
             raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Error! The story you're trying to access is being filtered by your choice of contents filtering.")
-            
+
         # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
+        soup = self.make_soup(data)
         #print data
 
         # Now go hunting for all the meta data and the chapter list.
-        
+
         ## Title
         a = soup.find('h1')
         self.story.setMetadata('title',stripHTML(a))
-        
+
         notice = soup.find('div', {'class' : 'notice'})
         if notice:
             self.story.setMetadata('notice',unicode(notice))
@@ -178,7 +178,7 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
         page=0
         i=0
         while i == 0:
-            asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getList('authorUrl')[0]+"/"+str(page)))
+            asoup = self.make_soup(self._fetchUrl(self.story.getList('authorUrl')[0]+"/"+str(page)))
 
             a = asoup.findAll('td', {'class' : 'lc2'})
             for lc2 in a:
@@ -196,8 +196,8 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
 
         score = lc2.findNext('th', {'class' : 'num'}).text
         if score != '-':
-            self.story.setMetadata('score', score)        
-        
+            self.story.setMetadata('score', score)
+
         lc4 = lc2.findNext('td', {'class' : 'lc4'})
         desc = lc4.contents[0]
 
@@ -212,7 +212,7 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
                 self.story.setMetadata('seriesUrl',seriesUrl)
                 series_name = stripHTML(a)
                 logger.debug("Series name= %s" % series_name)
-                series_soup = bs.BeautifulSoup(self._fetchUrl(seriesUrl))
+                series_soup = self.make_soup(self._fetchUrl(seriesUrl))
                 if series_soup:
                     logger.debug("Retrieving Series - looking for name")
                     series_name = series_soup.find('span', {'id' : 'ptitle'}).text.partition(' — ')[0]
@@ -221,7 +221,7 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
                 desc = lc4.contents[2]
                 # Check if series is in a universe
                 universe_url = self.story.getList('authorUrl')[0]  + "&type=uni"
-                universes_soup = bs.BeautifulSoup(self._fetchUrl(universe_url) )
+                universes_soup = self.make_soup(self._fetchUrl(universe_url) )
                 logger.debug("Universe url='{0}'".format(universe_url))
                 if universes_soup:
                     universes = universes_soup.findAll('div', {'class' : 'ser-box'})
@@ -255,11 +255,11 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
                 universe_name = stripHTML(a)
                 universeUrl = 'http://'+self.host+a['href']
                 logger.debug("Retrieving Universe - about to get page - universeUrl='{0}".format(universeUrl))
-                universe_soup = bs.BeautifulSoup(self._fetchUrl(universeUrl))
+                universe_soup = self.make_soup(self._fetchUrl(universeUrl))
                 logger.debug("Retrieving Universe - have page")
                 if universe_soup:
                     logger.debug("Retrieving Universe - looking for name")
-                    universe_name = universe_soup.find('h1', {'id' : 'ptitle'}).text.partition(' —')[0]
+                    universe_name = universe_soup.find('h1', {'id' : 'ptitle'}).text.partition('—')[0]
                     logger.debug("Universes name: '{0}'".format(universe_name))
 
                 self.story.setMetadata('universeUrl',universeUrl)
@@ -274,7 +274,7 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
             pass
 
         self.setDescription('http://'+self.host+'/s/'+self.story.getMetadata('storyId'),desc)
-            
+
         for b in lc4.findAll('b'):
             #logger.debug('Getting metadata: "%s"' % b)
             label = b.text
@@ -284,24 +284,38 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
             else:
                 value = b.nextSibling
             #logger.debug('label: "%s", value: "%s"' % (label, value))
-            
+
             if 'Sex' in label:
                 self.story.setMetadata('rating', value)
-                
+
             if 'Tags' in label or 'Codes' in label:
                 for code in re.split(r'\s*,\s*', value.strip()):
                      self.story.addToList('sitetags',code)
-                    
+
             if 'Posted' in label:
                 self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
                 self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-                
+
             if 'Concluded' in label:
                 self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-                
+
             if 'Updated' in label:
                 self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-#                
+
+        # Some books have a cover in the index page.
+        # Samples are:
+        #     http://storiesonline.net/s/11999
+        #     http://storiesonline.net/s/10823
+        if get_cover:
+            logger.debug("Looking for the cover image...")
+            cover_url = ""
+            img = soup.find('img')
+            if img:
+                cover_url=img['src']
+            logger.debug("cover_url: %s"%cover_url)
+            if cover_url:
+                self.setCoverImage(url,cover_url)
+
         status = lc4.find('span', {'class' : 'ab'})
         if  status != None:
             self.story.setMetadata('status', 'In-Progress')
@@ -312,111 +326,146 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
         else:
             self.story.setMetadata('status', 'Completed')
 
-            
     # grab the text for an individual chapter.
     def getChapterText(self, url):
 
         logger.debug('Getting chapter text from: %s' % url)
 
-        soup = bs.BeautifulSoup(self._fetchUrl(url),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.find('div', {'id' : 'story'})
-        
+        soup = self.make_soup(self._fetchUrl(url))
+
+        chaptertag = soup.find('div', {'id' : 'story'})
+        if not chaptertag:
+            logger.debug("div id=story not found, try article")
+            chaptertag = soup.find('article', {'id' : 'story'})
+
         # some big chapters are split over several pages
-        pager = div.find('span', {'class' : 'pager'})
+        pager = chaptertag.find('span', {'class' : 'pager'})
+
+        self.cleanPage(chaptertag)
+
         if pager != None:
-            a = pager.previousSibling
-            while a != None:
-                logger.debug("before pager: {0}".format(a))
-                b = a.previousSibling
-                a.extract()
-                a = b
 
             urls=pager.findAll('a')
             urls=urls[:len(urls)-1]
+            logger.debug("pager urls:%s"%urls)
             pager.extract()
-            div.contents = div.contents[2:]
-#            logger.debug(div)
-                        
+            chaptertag.contents = chaptertag.contents[2:]
+
             for ur in urls:
-                soup = bs.BeautifulSoup(self._fetchUrl("http://"+self.getSiteDomain()+ur['href']),
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-                div1 = soup.find('div', {'id' : 'story'})
-                
-                # Find the "Continues" marker on the current page and remove everything after that. 
-                continues = div.find('span', {'class' : 'conTag'})
-                if continues != None:
-                    while continues != None:
-#                        logger.debug("removing end: {0}".format(continues))
-                        b = continues.nextSibling
-                        continues.extract()
-                        continues = b
+                soup = self.make_soup(self._fetchUrl("http://"+self.getSiteDomain()+ur['href']))
 
-                # Find the "Continued" marker and delete everything before that
-                continued = div1.find('span', {'class' : 'conTag'})
-                if continued != None:
-                    a = continued.previousSibling
-                    while a != None:
-#                        logger.debug("before conTag: {0}".format(a))
-                        b = a.previousSibling
-                        a.extract()
-                        a = b
-                # Remove the pager from the end if this is the last page
-                endPager = div1.find('span', {'class' : 'pager'})
-                if endPager != None:
-                    b = endPager.nextSibling
-                    while endPager != None:
-                        logger.debug("removing end: {0}".format(endPager))
-                        b = endPager.nextSibling
-                        endPager.extract()
-                        endPager = b
-                    div1.contents = div1.contents[:len(div1) - 2]
-#                logger.debug("after removing pager: {0}".format(div1))
-                for tag in div1.contents[2:]:
-                    div.append(tag)
+                pagetag = soup.find('div', {'id' : 'story'})
+                if not pagetag:
+                    logger.debug("div id=story not found, try article")
+                    pagetag = soup.find('article', {'id' : 'story'})
 
-        # If it is a chapter, there are dates at the start for when it was posted or modified. These plus 
-        # everything before them can be discarded. 
-        postedDates = div.findAll('div', {'class' : 'date'})
+                self.cleanPage(pagetag)
+
+                for tag in pagetag.contents[2:]:
+                    chaptertag.append(tag)
+
+
+        if None == chaptertag:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+
+        return self.utf8FromSoup(url,chaptertag)
+
+    def cleanPage(self,pagetag):
+        "Consolidate 'page' clean up code so it can be called."
+
+        # some big chapters are split over several pages
+        # remove FIRST pager and everything before it.
+        tag = pagetag.find('span', {'class' : 'pager'})
+        while tag != None:
+            # logger.debug("remove before pager: {0}".format(tag))
+            prev = tag.previousSibling
+            tag.extract()
+            tag = prev
+
+        # Find the "Continues" marker on the current page and
+        # remove everything after that.  This is actually
+        # effecting the *previous* 'page'.  EXCEPT!--they are
+        # putting a 'conTag' at the *top* now, too.  So this
+        # was nuking every page but the first and last.  Now
+        # only if 'Continues'
+        for contag in pagetag.findAll('span', {'class' : 'conTag'}):
+            # remove everything after continues...
+            if 'Continuation' in contag.text:
+                tag = contag
+                while tag != None:
+                    # logger.debug("remove before Continuation: {0}".format(tag))
+                    prev = tag.previousSibling
+                    tag.extract()
+                    tag = prev
+            elif 'Continues' in contag.text:
+                tag = contag
+                while tag != None:
+                    # logger.debug("remove after Continues: {0}".format(tag))
+                    nxt = tag.nextSibling
+                    tag.extract()
+                    tag = nxt
+
+        # some big chapters are split over several pages
+        # remove LAST pager and everything before it.
+        # Only needed on last page.
+        tag = pagetag.find('span', {'class' : 'pager'})
+        while tag != None:
+            # logger.debug("remove after pager: {0}".format(tag))
+            nxt = tag.nextSibling
+            tag.extract()
+            tag = nxt
+
+        # If it is a chapter, there are dates at the start for when it was posted or modified. These plus
+        # everything before them can be discarded.
+        postedDates = pagetag.findAll('div', {'class' : 'date'})
         if postedDates:
             a = postedDates[0].previousSibling
             while a != None:
-#                logger.debug("before dates: {0}".format(a))
+                # logger.debug("before dates: {0}".format(a))
                 b = a.previousSibling
                 a.extract()
                 a = b
-            for a in div.findAll('div', {'class' : 'date'}):
+            for a in pagetag.findAll('div', {'class' : 'date'}):
                 a.extract()
 
         # For single chapter stories, there is a copyright statement. Remove this and everything
         # before it.
-        copy = div.find('h4', {'class': 'copy'})
+        copy = pagetag.find('h4', {'class': 'copy'})
         while copy != None:
-#            logger.debug("before copyright: {0}".format(copy))
+            # logger.debug("before copyright: {0}".format(copy))
             b = copy.previousSibling
             copy.extract()
             copy = b
 
         # For a story or the last chapter, remove voting form and the in library box
-        a = div.find('div', {'id' : 'vote-form'})
+        a = pagetag.find('div', {'id' : 'vote-form'})
         if a != None:
             a.extract()
-        a = div.find('div', {'id' : 'b-man-div'})
+        a = pagetag.find('div', {'id' : 'top-header'})
+        if a != None:
+            a.extract()
+        a = pagetag.find('div', {'id' : 'b-man-div'})
         if a != None:
             a.extract()
 
-        # Kill the "The End" header and everything after it.
-        a = div.find(['h2', 'h3'], {'class' : 'end'})
-        logger.debug("Chapter end= '{0}'".format(a))
+        # Kill the vote form and everything after it.
+        a = pagetag.find('div', {'class' : 'vform'})
+        # logger.debug("Chapter end= '{0}'".format(a))
         while a != None:
             b = a.nextSibling
             a.extract()
             a=b
 
+        # Kill the vote form and everything after it.
+        a = pagetag.find('h3', {'class' : 'end'})
+        # logger.debug("Chapter end= '{0}'".format(a))
+        while a != None:
+            b = a.nextSibling
+            a.extract()
+            a=b
+
+        foot = pagetag.find('footer')
+        if foot != None:
+            foot.extract()
+
 
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)

fanficfare/adapters/adapter_tgstorytimecom.py

@@ -0,0 +1,36 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2015 FanFicFare team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Software: eFiction
+from base_efiction_adapter import BaseEfictionAdapter
+
+class TGStorytimeComAdapter(BaseEfictionAdapter):
+
+    @staticmethod
+    def getSiteDomain():
+        return 'tgstorytime.com'
+
+    @classmethod
+    def getSiteAbbrev(self):
+        return 'tgstory'
+
+    @classmethod
+    def getDateFormat(self):
+        return "%m/%d/%y"
+
+def getClass():
+    return TGStorytimeComAdapter

fanficfare/adapters/base_adapter.py

@@ -104,6 +104,8 @@ class BaseSiteAdapter(Configurable):
         self.chapterFirst = None
         self.chapterLast = None
         self.oldchapters = None
+        self.oldchaptersmap = None
+        self.oldchaptersdata = None
         self.oldimgs = None
         self.oldcover = None # (data of existing cover html, data of existing cover image)
         self.calibrebookmark = None
@@ -140,28 +142,6 @@ class BaseSiteAdapter(Configurable):
         '''
         self.get_cookiejar().load(filename, ignore_discard=True, ignore_expires=True)
         
-    # def save_cookiejar(self,filename):
-    #     '''
-    #     Assumed to be a FileCookieJar if self.cookiejar set.
-    #     Takes file *name*.
-    #     '''
-    #     self.get_cookiejar().save(filename, ignore_discard=True, ignore_expires=True)
-
-    # def save_pagecache(self,filename):
-    #     '''
-    #     Writes pickle of pagecache to file *name*
-    #     '''
-    #     with open(filename, 'wb') as f:
-    #         pickle.dump(self.get_pagecache(),
-    #                     f,protocol=pickle.HIGHEST_PROTOCOL)
-        
-    # def load_pagecache(self,filename):
-    #     '''
-    #     Reads pickle of pagecache from file *name*
-    #     '''
-    #     with open(filename, 'rb') as f:
-    #         self.set_pagecache(pickle.load(f))
-
     def get_pagecache(self):
         return self.pagecache
     
@@ -185,9 +165,9 @@ class BaseSiteAdapter(Configurable):
         else:
             return None
 
-    def _set_to_pagecache(self,cachekey,data):
+    def _set_to_pagecache(self,cachekey,data,redirectedurl):
         if self.use_pagecache():
-            self.get_pagecache()[cachekey] = data
+            self.get_pagecache()[cachekey] = (data,redirectedurl)
 
     def use_pagecache(self):
         '''
@@ -257,7 +237,8 @@ class BaseSiteAdapter(Configurable):
         cachekey=self._get_cachekey(url, parameters, headers)
         if usecache and self._has_cachekey(cachekey):
             logger.debug("#####################################\npagecache HIT: %s"%cachekey)
-            return self._get_from_pagecache(cachekey)
+            data,redirecturl = self._get_from_pagecache(cachekey)
+            return data
         
         logger.debug("#####################################\npagecache MISS: %s"%cachekey)
         self.do_sleep(extrasleep)
@@ -272,13 +253,23 @@ class BaseSiteAdapter(Configurable):
                          data=urllib.urlencode(parameters),
                          headers=headers)
         data = self._decode(self.opener.open(req,None,float(self.getConfig('connect_timeout',30.0))).read())
-        self._set_to_pagecache(cachekey,data)
+        self._set_to_pagecache(cachekey,data,url)
         return data
 
     def _fetchUrlRaw(self, url,
                      parameters=None,
                      extrasleep=None,
                      usecache=True):
+        
+        return self._fetchUrlRawOpened(url,
+                                       parameters,
+                                       extrasleep,
+                                       usecache)[0]
+        
+    def _fetchUrlRawOpened(self, url,
+                           parameters=None,
+                           extrasleep=None,
+                           usecache=True):
         '''
         When should cache be cleared or not used? logins...
         
@@ -289,16 +280,25 @@ class BaseSiteAdapter(Configurable):
         cachekey=self._get_cachekey(url, parameters)
         if usecache and self._has_cachekey(cachekey):
             logger.debug("#####################################\npagecache HIT: %s"%cachekey)
-            return self._get_from_pagecache(cachekey)
+            data,redirecturl = self._get_from_pagecache(cachekey)
+            class FakeOpened:
+                def __init__(self,data,url):
+                    self.data=data
+                    self.url=url
+                def geturl(self): return self.url
+                def read(self): return self.data
+            return (data,FakeOpened(data,redirecturl))
         
         logger.debug("#####################################\npagecache MISS: %s"%cachekey)
         self.do_sleep(extrasleep)
         if parameters != None:
-            data = self.opener.open(url.replace(' ','%20'),urllib.urlencode(parameters),float(self.getConfig('connect_timeout',30.0))).read()
+            opened = self.opener.open(url.replace(' ','%20'),urllib.urlencode(parameters),float(self.getConfig('connect_timeout',30.0)))
         else:
-            data = self.opener.open(url.replace(' ','%20'),None,float(self.getConfig('connect_timeout',30.0))).read()
-        self._set_to_pagecache(cachekey,data)
-        return data
+            opened = self.opener.open(url.replace(' ','%20'),None,float(self.getConfig('connect_timeout',30.0)))
+        data = opened.read()
+        self._set_to_pagecache(cachekey,data,opened.url)
+        
+        return (data,opened)
 
     def set_sleep(self,val):
         logger.debug("\n===========\n set sleep time %s\n==========="%val)
@@ -312,20 +312,30 @@ class BaseSiteAdapter(Configurable):
         elif self.getConfig('slow_down_sleep_time'):
             time.sleep(float(self.getConfig('slow_down_sleep_time')))
         
-    # parameters is a dict()
     def _fetchUrl(self, url,
                   parameters=None,
                   usecache=True,
                   extrasleep=None):
+        return self._fetchUrlOpened(url,
+                                    parameters,
+                                    usecache,
+                                    extrasleep)[0]
+
+    # parameters is a dict()
+    def _fetchUrlOpened(self, url,
+                        parameters=None,
+                        usecache=True,
+                        extrasleep=None):
 
         excpt=None
         for sleeptime in [0, 0.5, 4, 9]:
             time.sleep(sleeptime)	
             try:
-                return self._decode(self._fetchUrlRaw(url,
+                (data,opened)=self._fetchUrlRawOpened(url,
                                                       parameters=parameters,
                                                       usecache=usecache,
-                                                      extrasleep=extrasleep))
+                                                      extrasleep=extrasleep)
+                return (self._decode(data),opened)
             except u2.HTTPError, he:
                 excpt=he
                 if he.code == 404:
@@ -345,6 +355,7 @@ class BaseSiteAdapter(Configurable):
             self.chapterFirst=int(first)-1
         if last:
             self.chapterLast=int(last)-1
+        self.story.set_chapters_range(first,last)
     
     # Does the download the first time it's called.
     def getStory(self):
@@ -352,21 +363,40 @@ class BaseSiteAdapter(Configurable):
             self.getStoryMetadataOnly(get_cover=True)
 
             for index, (title,url) in enumerate(self.chapterUrls):
+                newchap = False
                 if (self.chapterFirst!=None and index < self.chapterFirst) or \
                         (self.chapterLast!=None and index > self.chapterLast):
                     self.story.addChapter(url,
                                           removeEntities(title),
                                           None)
                 else:
-                    if self.oldchapters and index < len(self.oldchapters):
+                    data = None
+                    if self.oldchaptersmap:
+                        if url in self.oldchaptersmap:
+                            data = self.utf8FromSoup(None,
+                                                     self.oldchaptersmap[url],
+                                                     partial(cachedfetch,self._fetchUrlRaw,self.oldimgs))
+                    elif self.oldchapters and index < len(self.oldchapters):
                         data = self.utf8FromSoup(None,
                                                  self.oldchapters[index],
                                                  partial(cachedfetch,self._fetchUrlRaw,self.oldimgs))
-                    else:
+
+                    # if already marked new -- ie, origtitle and title don't match
+                    # logger.debug("self.oldchaptersdata[url]:%s"%(self.oldchaptersdata[url]))
+                    newchap = (self.oldchaptersdata is not None and
+                               url in self.oldchaptersdata and (
+                            self.oldchaptersdata[url]['chapterorigtitle'] !=
+                            self.oldchaptersdata[url]['chaptertitle']) )
+                    
+                    if not data:
                         data = self.getChapterText(url)
+                        # if had to fetch and has existing chapters
+                        newchap = bool(self.oldchapters or self.oldchaptersmap)
+                        
                     self.story.addChapter(url,
                                           removeEntities(title),
-                                          removeEntities(data))
+                                          removeEntities(data),
+                                          newchap)
             self.storyDone = True
             
             # include image, but no cover from story, add default_cover_image cover.
@@ -379,10 +409,12 @@ class BaseSiteAdapter(Configurable):
                                                                self.getConfig('allow_unsafe_filename')),
                                      self._fetchUrlRaw,
                                      cover=True)
+                self.story.setMetadata('cover_image','default')
 
             # no new cover, set old cover, if there is one.
             if not self.story.cover and self.oldcover:
                 self.story.oldcover = self.oldcover
+                self.story.setMetadata('cover_image','old')
                 
             # cheesy way to carry calibre bookmark file forward across update.
             if self.calibrebookmark:
@@ -397,11 +429,24 @@ class BaseSiteAdapter(Configurable):
             self.doExtractChapterUrlsAndMetadata(get_cover=get_cover)
             
             if not self.story.getMetadataRaw('dateUpdated'):
-                self.story.setMetadata('dateUpdated',self.story.getMetadataRaw('datePublished'))
+                if self.story.getMetadataRaw('datePublished'):
+                    self.story.setMetadata('dateUpdated',self.story.getMetadataRaw('datePublished'))
+                else:
+                    self.story.setMetadata('dateUpdated',self.story.getMetadataRaw('dateCreated'))                
 
             self.metadataDone = True
         return self.story
 
+    def setStoryMetadata(self,metahtml):
+        if metahtml:
+            self.story.load_html_metadata(metahtml)
+            self.metadataDone = True
+            if not self.story.getMetadataRaw('dateUpdated'):
+                if self.story.getMetadataRaw('datePublished'):
+                    self.story.setMetadata('dateUpdated',self.story.getMetadataRaw('datePublished'))
+                else:
+                    self.story.setMetadata('dateUpdated',self.story.getMetadataRaw('dateCreated'))                
+    
     def hookForUpdates(self,chaptercount):
         "Usually not needed."
         return chaptercount
@@ -418,6 +463,11 @@ class BaseSiteAdapter(Configurable):
         "Only needs to be overriden if != site domain."
         return cls.getSiteDomain()
     
+    @classmethod
+    def getConfigSections(cls):
+        "Only needs to be overriden if has additional ini sections."
+        return [cls.getConfigSection()]
+    
     @classmethod
     def stripURLParameters(cls,url):
         "Only needs to be overriden if URL contains more than one parameter"
@@ -467,6 +517,13 @@ class BaseSiteAdapter(Configurable):
 
     def setDescription(self,url,svalue):
         #print("\n\nsvalue:\n%s\n"%svalue)
+        strval = u"%s"%svalue # works for either soup or string
+        if self.hasConfig('description_limit'):
+            limit = int(self.getConfig('description_limit'))
+            if limit and len(strval) > limit:
+                svalue = strval[:limit]
+
+        #print(u"[[[[[\n\n%s\n\n]]]]]]]]"%svalue) # works for either soup or string
         if self.getConfig('keep_summary_html'):
             if isinstance(svalue,basestring):
                 # bs4/html5lib add html, header and body tags, which
@@ -479,8 +536,10 @@ class BaseSiteAdapter(Configurable):
 
     def setCoverImage(self,storyurl,imgurl):
         if self.getConfig('include_images'):
-            self.story.addImgUrl(storyurl,imgurl,self._fetchUrlRaw,cover=True,
-                                 coverexclusion=self.getConfig('cover_exclusion_regexp'))
+            return self.story.addImgUrl(storyurl,imgurl,self._fetchUrlRaw,cover=True,
+                                        coverexclusion=self.getConfig('cover_exclusion_regexp'))
+        else:
+            return (None,None)
 
     # bs3 & bs4 are different here.
     # will move to a bs3 vs bs4 block if there's lots of changes.

fanficfare/adapters/base_efiction_adapter.py

@@ -70,6 +70,11 @@ class BaseEfictionAdapter(BaseSiteAdapter):
         self.triedAcceptWarnings = False
         self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
 
+    @classmethod
+    def getConfigSections(cls):
+        "Only needs to be overriden if has additional ini sections."
+        return ['base_efiction',cls.getConfigSection()]
+    
     @classmethod
     def getAcceptDomains(cls):
         return [cls.getSiteDomain(),'www.' + cls.getSiteDomain()]
@@ -89,7 +94,6 @@ class BaseEfictionAdapter(BaseSiteAdapter):
         """
         return ["Windows-1252", "utf8"]
 
-
     @classmethod
     def getPathToArchive(cls):
         """
@@ -265,7 +269,6 @@ class BaseEfictionAdapter(BaseSiteAdapter):
                 else:
                     super(NameOfMyAdapter, self).handleMetadata(key, value)
         """
-        # logger.debug("metadata: '%s' == '%s'" % (key, value))
         if value == 'None':
             return
         elif key == 'Summary':
@@ -288,7 +291,7 @@ class BaseEfictionAdapter(BaseSiteAdapter):
                 self.story.addToList('challenge', val)
         elif key == 'Chapters':
             self.story.setMetadata('numChapters', int(value))
-        elif key == 'Rating':
+        elif key == 'Rating' or key == 'Rated':
             self.story.setMetadata('rating', value)
         elif key == 'Word count':
             self.story.setMetadata('numWords', value)
@@ -325,6 +328,8 @@ class BaseEfictionAdapter(BaseSiteAdapter):
 
         ## Handle warnings and login checks
         errorDiv = soup.find("div", "errortext")
+        if errorDiv is None:
+            errorDiv = soup.find("div", "errormsg") # sometimes different class.
         while errorDiv is not None:
             if self.getMessageRegisteredUsersOnly() in errorDiv.prettify():
                 if not self.triedLoggingIn:

fanficfare/adapters/base_xenforoforum_adapter.py

@@ -0,0 +1,229 @@
+#  -*- coding: utf-8 -*-
+
+# Copyright 2015 FanFicFare team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+logger = logging.getLogger(__name__)
+
+class BaseXenForoForumAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["utf8",
+                       "Windows-1252"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+							   
+							   
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])        
+        
+        # get storyId from url--url validation guarantees query correct
+        m = re.match(self.getSiteURLPattern(),url)
+        if m:
+            self.story.setMetadata('storyId',m.group('id'))
+
+            # normalized story URL.
+            self._setURL(self.getURLPrefix() + '/'+m.group('tp')+'/'+self.story.getMetadata('storyId')+'/')
+        else:
+            raise exceptions.InvalidStoryURL(url,
+                                             self.getSiteDomain(),
+                                             self.getSiteExampleURLs())
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','fsb')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%b %d, %Y at %I:%M %p"
+            
+    @classmethod
+    def getConfigSections(cls):
+        "Only needs to be overriden if has additional ini sections."
+        return ['base_xenforoforum',cls.getConfigSection()]
+    
+    @classmethod
+    def getURLPrefix(cls):
+        # The site domain.  Does have www here, if it uses it.
+        return 'https://' + cls.getSiteDomain() 
+
+    @classmethod
+    def getSiteExampleURLs(cls):
+        return cls.getURLPrefix()+"/threads/some-story-name.123456/"
+
+    def getSiteURLPattern(self):
+        return r"https?://"+re.escape(self.getSiteDomain())+r"/(?P<tp>threads|posts)/(.+\.)?(?P<id>\d+)/?"
+        
+    def use_pagecache(self):
+        '''
+        adapters that will work with the page cache need to implement
+        this and change it to True.
+        '''
+        return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        useurl = self.url
+        logger.info("url: "+useurl)
+
+        try:
+            (data,opened) = self._fetchUrlOpened(useurl)
+            useurl = opened.geturl()
+            logger.info("use useurl: "+useurl)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = self.make_soup(data)
+
+        a = soup.find('h3',{'class':'userText'}).find('a')
+        self.story.addToList('authorId',a['href'].split('/')[1])
+        self.story.addToList('authorUrl',self.getURLPrefix()+'/'+a['href'])
+        self.story.addToList('author',a.text)
+
+        h1 = soup.find('div',{'class':'titleBar'}).h1
+        self.story.setMetadata('title',stripHTML(h1))
+        
+        if '#' in useurl:
+            anchorid = useurl.split('#')[1]
+            soup = soup.find('li',id=anchorid)
+        else:
+            # try threadmarks if no '#' in , require at least 2.
+            threadmarksa = soup.find('a',{'class':'threadmarksTrigger'})
+            if threadmarksa:
+                soupmarks = self.make_soup(self._fetchUrl(self.getURLPrefix()+'/'+threadmarksa['href']))
+                markas = soupmarks.find('ol',{'class':'overlayScroll'}).find_all('a')
+                if len(markas) > 1:
+                    for (atag,url,name) in [ (x,x['href'],stripHTML(x)) for x in markas ]:
+                        date = self.make_date(atag.find_next_sibling('div',{'class':'extra'}))
+                        if not self.story.getMetadataRaw('datePublished') or date < self.story.getMetadataRaw('datePublished'):
+                            self.story.setMetadata('datePublished', date)
+                        if not self.story.getMetadataRaw('dateUpdated') or date > self.story.getMetadataRaw('dateUpdated'):
+                            self.story.setMetadata('dateUpdated', date)
+                            
+                        self.chapterUrls.append((name,self.getURLPrefix()+'/'+url))
+                        
+            soup = soup.find('li',{'class':'message'}) # limit first post for date stuff below. ('#' posts above)
+                
+        # Now go hunting for the 'chapter list'.
+        bq = soup.find('blockquote') # assume first posting contains TOC urls.
+        
+        bq.name='div'
+
+        for iframe in bq.find_all('iframe'):
+            iframe.extract() # calibre book reader & editor don't like iframes to youtube.
+
+        for qdiv in bq.find_all('div',{'class':'quoteExpand'}):
+            qdiv.extract() # Remove <div class="quoteExpand">click to expand</div>
+            
+        self.setDescription(useurl,bq)
+
+        # otherwise, use first post links--include first post since
+        # that's often also the first chapter.
+        if not self.chapterUrls:
+            self.chapterUrls.append(("First Post",useurl))
+            for (url,name) in [ (x['href'],stripHTML(x)) for x in bq.find_all('a') ]:
+                logger.debug("found chapurl:%s"%url)
+                if not url.startswith('http'):
+                    url = self.getURLPrefix()+'/'+url
+    
+                if ( url.startswith(self.getURLPrefix()) or
+                     url.startswith('http://'+self.getSiteDomain()) or
+                     url.startswith('https://'+self.getSiteDomain()) ) and ('/posts/' in url or '/threads/' in url):
+                    # brute force way to deal with SB's http->https change when hardcoded http urls.
+                    url = url.replace('http://'+self.getSiteDomain(),self.getURLPrefix())
+                    logger.debug("used chapurl:%s"%(url))
+                    self.chapterUrls.append((name,url))
+                    if url == useurl and 'First Post' == self.chapterUrls[0][0]:
+                        # remove "First Post" if included in list.
+                        logger.debug("delete dup 'First Post' chapter: %s %s"%self.chapterUrls[0])
+                        del self.chapterUrls[0]
+                        
+            # Didn't use threadmarks, so take created/updated dates
+            # from the 'first' posting created and updated.
+            date = self.make_date(soup.find('a',{'class':'datePermalink'}))
+            if date:
+                self.story.setMetadata('datePublished', date)
+                self.story.setMetadata('dateUpdated', date) # updated overwritten below if found.
+        
+            date = self.make_date(soup.find('div',{'class':'editDate'}))
+            if date:
+                self.story.setMetadata('dateUpdated', date) 
+            
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+    def make_date(self,parenttag): # forums use a BS thing where dates
+                                  # can appear different if recent.
+        datestr=None
+        try:
+            datetag = parenttag.find('span',{'class':'DateTime'})
+            if datetag:
+                datestr = datetag['title']
+            else:
+                datetag = parenttag.find('abbr',{'class':'DateTime'})
+                if datetag:
+                    datestr="%s at %s"%(datetag['data-datestring'],datetag['data-timestring'])
+            # Apr 24, 2015 at 4:39 AM
+            # May 1, 2015 at 5:47 AM
+            datestr = re.sub(r' (\d[^\d])',r' 0\1',datestr) # add leading 0 for single digit day & hours.
+            return makeDate(datestr, self.dateformat)
+        except:
+            logger.debug('No date found in %s'%parenttag)
+            return None
+        
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+        logger.debug('Getting chapter text from: %s' % url)
+
+        origurl = url
+        (data,opened) = self._fetchUrlOpened(url)
+        url = opened.geturl()
+        if '#' in origurl and '#' not in url:
+            url = url + origurl[origurl.index('#'):]
+        logger.debug("chapter URL redirected to: %s"%url)
+
+        soup = self.make_soup(data)
+
+        if '#' in url:
+            anchorid = url.split('#')[1]
+            soup = soup.find('li',id=anchorid)
+        bq = soup.find('blockquote')
+
+        bq.name='div'
+
+        for iframe in bq.find_all('iframe'):
+            iframe.extract() # calibre book reader & editor don't like iframes to youtube.
+
+        for qdiv in bq.find_all('div',{'class':'quoteExpand'}):
+            qdiv.extract() # Remove <div class="quoteExpand">click to expand</div>
+
+        return self.utf8FromSoup(url,bq)

fanficfare/cli.py

@@ -16,7 +16,8 @@
 #
 
 from optparse import OptionParser
-from os.path import expanduser, isfile, join, dirname
+from os.path import expanduser, join, dirname
+from os import access, R_OK
 from subprocess import call
 import ConfigParser
 import getpass
@@ -40,12 +41,14 @@ try:
     # running under calibre
     from calibre_plugins.fanfictiondownloader_plugin.fanficfare import adapters, writers, exceptions
     from calibre_plugins.fanfictiondownloader_plugin.fanficfare.configurable import Configuration
-    from calibre_plugins.fanfictiondownloader_plugin.fanficfare.epubutils import get_dcsource_chaptercount, get_update_data
+    from calibre_plugins.fanfictiondownloader_plugin.fanficfare.epubutils import (
+        get_dcsource_chaptercount, get_update_data, reset_orig_chapters_epub)
     from calibre_plugins.fanfictiondownloader_plugin.fanficfare.geturls import get_urls_from_page
 except ImportError:
     from fanficfare import adapters, writers, exceptions
     from fanficfare.configurable import Configuration
-    from fanficfare.epubutils import get_dcsource_chaptercount, get_update_data
+    from fanficfare.epubutils import (
+        get_dcsource_chaptercount, get_update_data, reset_orig_chapters_epub)
     from fanficfare.geturls import get_urls_from_page
 
 
@@ -86,12 +89,19 @@ def main(argv=None, parser=None, passed_defaultsini=None, passed_personalini=Non
     parser.add_option('-u', '--update-epub',
                       action='store_true', dest='update',
                       help='Update an existing epub with new chapters, give epub filename instead of storyurl.', )
+    parser.add_option('--unnew',
+                      action='store_true', dest='unnew',
+                      help='Remove (new) chapter marks left by mark_new_chapters setting.', )
     parser.add_option('--update-cover',
                       action='store_true', dest='updatecover',
                       help='Update cover in an existing epub, otherwise existing cover (if any) is used on update.  Only valid with --update-epub.', )
     parser.add_option('--force',
                       action='store_true', dest='force',
                       help='Force overwrite of an existing epub, download and overwrite all chapters.', )
+    parser.add_option('-i', '--infile',
+                      help='Give a filename to read for URLs (and/or existing EPUB files with -u for updates).',
+                      dest='infile', default=None,
+                      metavar='INFILE')
     parser.add_option('-l', '--list',
                       action='store_true', dest='list',
                       help='Get list of valid story URLs from page given.', )
@@ -111,7 +121,7 @@ def main(argv=None, parser=None, passed_defaultsini=None, passed_personalini=Non
         logger = logging.getLogger('fanficfare')
         logger.setLevel(logging.INFO)
 
-    if not options.siteslist and len(args) != 1:
+    if not (options.siteslist or options.infile) and len(args) != 1:
         parser.error('incorrect number of arguments')
 
     if options.siteslist:
@@ -124,26 +134,71 @@ def main(argv=None, parser=None, passed_defaultsini=None, passed_personalini=Non
     if options.update and options.format != 'epub':
         parser.error('-u/--update-epub only works with epub')
 
+    if options.unnew and options.format != 'epub':
+        parser.error('--unnew only works with epub')
+
+    # for passing in a file list
+    if options.infile:
+        urls=[]
+        with open(options.infile,"r") as infile:
+            #print "File exists and is readable"
+          
+            #fileurls = [line.strip() for line in infile]
+            for url in infile:
+                url = url[:url.find('#')].strip()
+                if len(url) > 0:
+                    #print "URL: (%s)"%url
+                    urls.append(url)
+    else:
+        urls = args
+
+    if len(urls) > 1:
+        for url in urls:
+            try:
+                do_download(url,
+                            options,
+                            passed_defaultsini,
+                            passed_personalini)
+            except Exception, e:
+                print "URL(%s) Failed: Exception (%s). Run URL individually for more detail."%(url,e)
+    else:
+        do_download(urls[0],
+                    options,
+                    passed_defaultsini,
+                    passed_personalini)
+
+# make rest a function and loop on it.
+def do_download(arg,
+                options,
+                passed_defaultsini,
+                passed_personalini):
+                
     # Attempt to update an existing epub.
     chaptercount = None
     output_filename = None
+
+    if options.unnew:
+        # remove mark_new_chapters marks
+        reset_orig_chapters_epub(arg,arg)
+        return
+    
     if options.update:
         try:
-            url, chaptercount = get_dcsource_chaptercount(args[0])
+            url, chaptercount = get_dcsource_chaptercount(arg)
             if not url:
                 print 'No story URL found in epub to update.'
                 return
-            print 'Updating %s, URL: %s' % (args[0], url)
-            output_filename = args[0]
+            print 'Updating %s, URL: %s' % (arg, url)
+            output_filename = arg
         except Exception:
             # if there's an error reading the update file, maybe it's a URL?
             # we'll look for an existing outputfile down below.
-            url = args[0]
+            url = arg
     else:
-        url = args[0]
-
+        url = arg
+        
     try:
-        configuration = Configuration(adapters.getConfigSectionFor(url), options.format)
+        configuration = Configuration(adapters.getConfigSectionsFor(url), options.format)
     except exceptions.UnknownSite, e:
         if options.list or options.normalize:
             # list for page doesn't have to be a supported site.
@@ -159,25 +214,18 @@ def main(argv=None, parser=None, passed_defaultsini=None, passed_personalini=Non
     if passed_defaultsini:
         configuration.readfp(passed_defaultsini)
 
-    if isfile(join(dirname(__file__), 'defaults.ini')):
-        conflist.append(join(dirname(__file__), 'defaults.ini'))
-        
-    if isfile(join(homepath, 'defaults.ini')):
-        conflist.append(join(homepath, 'defaults.ini'))
-    if isfile(join(homepath2, 'defaults.ini')):
-        conflist.append(join(homepath2, 'defaults.ini'))
-    if isfile('defaults.ini'):
-        conflist.append('defaults.ini')
+    # don't need to check existance for our selves.
+    conflist.append(join(dirname(__file__), 'defaults.ini'))
+    conflist.append(join(homepath, 'defaults.ini'))
+    conflist.append(join(homepath2, 'defaults.ini'))
+    conflist.append('defaults.ini')
 
     if passed_personalini:
         configuration.readfp(passed_personalini)
 
-    if isfile(join(homepath, 'personal.ini')):
-        conflist.append(join(homepath, 'personal.ini'))
-    if isfile(join(homepath2, 'personal.ini')):
-        conflist.append(join(homepath2, 'personal.ini'))
-    if isfile('personal.ini'):
-        conflist.append('personal.ini')
+    conflist.append(join(homepath, 'personal.ini'))
+    conflist.append(join(homepath2, 'personal.ini'))
+    conflist.append('personal.ini')
 
     if options.configfile:
         conflist.extend(options.configfile)
@@ -210,7 +258,7 @@ def main(argv=None, parser=None, passed_defaultsini=None, passed_personalini=Non
             configuration.set('overrides', var, val)
 
     if options.list or options.normalize:
-        retlist = get_urls_from_page(args[0], configuration, normalize=options.normalize)
+        retlist = get_urls_from_page(arg, configuration, normalize=options.normalize)
         print '\n'.join(retlist)
         return
 
@@ -277,7 +325,15 @@ def main(argv=None, parser=None, passed_defaultsini=None, passed_personalini=Non
                 # update now handled by pre-populating the old
                 # images and chapters in the adapter rather than
                 # merging epubs.
-                url, chaptercount, adapter.oldchapters, adapter.oldimgs, adapter.oldcover, adapter.calibrebookmark, adapter.logfile = get_update_data(output_filename)
+                (url,
+                 chaptercount,
+                 adapter.oldchapters,
+                 adapter.oldimgs,
+                 adapter.oldcover,
+                 adapter.calibrebookmark,
+                 adapter.logfile,
+                 adapter.oldchaptersmap,
+                 adapter.oldchaptersdata) = (get_update_data(output_filename))[0:9]
 
                 print 'Do update - epub(%d) vs url(%d)' % (chaptercount, urlchaptercount)
 

fanficfare/configurable.py

@@ -17,6 +17,7 @@
 
 import ConfigParser, re
 import exceptions
+import codecs
 from ConfigParser import DEFAULTSECT, MissingSectionHeaderError, ParsingError
 
 # All of the writers(epub,html,txt) and adapters(ffnet,twlt,etc)
@@ -41,15 +42,48 @@ def re_compile(regex,line):
         return re.compile(regex)
     except Exception, e: 
         raise exceptions.RegularExpresssionFailed(e,regex,line)
-    
+
+# fall back labels.
+titleLabels = {
+    'category':'Category',
+    'genre':'Genre',
+    'language':'Language',
+    'status':'Status',
+    'series':'Series',
+    'characters':'Characters',
+    'ships':'Relationships',
+    'datePublished':'Published',
+    'dateUpdated':'Updated',
+    'dateCreated':'Packaged',
+    'rating':'Rating',
+    'warnings':'Warnings',
+    'numChapters':'Chapters',
+    'numWords':'Words',
+    'site':'Site',
+    'storyId':'Story ID',
+    'authorId':'Author ID',
+    'extratags':'Extra Tags',
+    'title':'Title',
+    'storyUrl':'Story URL',
+    'description':'Summary',
+    'author':'Author',
+    'authorUrl':'Author URL',
+    'formatname':'File Format',
+    'formatext':'File Extension',
+    'siteabbrev':'Site Abbrev',
+    'version':'Downloader Version'
+    }
+
 formatsections = ['html','txt','epub','mobi']
 othersections = ['defaults','overrides']
 
 def get_valid_sections():
-    sites = adapters.getConfigSections()
+    sites = adapters.getConfigSections() 
     sitesections = list(othersections)
     for section in sites:
         sitesections.append(section)
+        # also allows [www.base_efiction] and [www.base_forum]. Not
+        # likely to matter.
         if section.startswith('www.'):
             # add w/o www if has www
             sitesections.append(section[4:])
@@ -99,6 +133,7 @@ def get_valid_set_options():
                'replace_hr':(None,None,boollist),
                'sort_ships':(None,None,boollist),
                'strip_chapter_numbers':(None,None,boollist),
+               'mark_new_chapters':(None,None,boollist),
                'titlepage_use_table':(None,None,boollist),
                
                'use_ssl_unverified_context':(None,None,boollist),
@@ -111,22 +146,18 @@ def get_valid_set_options():
                
                'fix_fimf_blockquotes':(['fimfiction.net'],None,boollist),
                'fail_on_password':(['fimfiction.net'],None,boollist),
-               'do_update_hook':(['fimfiction.net'],None,boollist),
+               'do_update_hook':(['fimfiction.net',
+                                  'archiveofourown.org'],None,boollist),
 
                'force_login':(['phoenixsong.net'],None,boollist),
                'non_breaking_spaces':(['fictionmania.tv'],None,boollist),
                'universe_as_series':(['storiesonline.net'],None,boollist),
                'strip_text_links':(['bloodshedverse.com'],None,boollist),
 
-               # eFiction Base
-               'bulk_load':(['fannation.shades-of-moonlight.com',
-                             'fhsarchive.com',
-                             'lotrfanfiction.com',
-                             'themaplebookshelf.com',
-                             'devianthearts.com',
-                             'thehookupzone.net',
-                             'libraryofmoria.com',
-                             ],None,boollist),
+               # eFiction Base adapters allow bulk_load
+               # kept forgetting to add them, so now it's automatic.
+               'bulk_load':(adapters.get_bulk_load_sites(),
+                            None,boollist),
                
                'include_logpage':(None,['epub'],boollist+['smart']),
                
@@ -136,6 +167,7 @@ def get_valid_set_options():
                'grayscale_images':(None,['epub','html'],boollist),
                'no_image_processing':(None,['epub','html'],boollist),
                }
+
     return dict(valdict)
 
 def get_valid_scalar_entries():
@@ -163,6 +195,7 @@ def get_valid_scalar_entries():
                  'seriesHTML',
                  'langcode',
                  'output_css',
+                 'cover_image',
                  ])
 
 def get_valid_entries():
@@ -173,6 +206,7 @@ def get_valid_keywords():
     return list(['(in|ex)clude_metadata_(pre|post)',
                  'add_chapter_numbers',
                  'add_genre_when_multi_category',
+                 'adult_ratings',
                  'allow_unsafe_filename',
                  'always_overwrite',
                  'anthology_tags',
@@ -181,8 +215,12 @@ def get_valid_keywords():
                  'bulk_load',
                  'chapter_end',
                  'chapter_start',
-                 'chapter_title_add_pattern',
                  'chapter_title_strip_pattern',
+                 'chapter_title_def_pattern',
+                 'chapter_title_add_pattern',
+                 'chapter_title_new_pattern',
+                 'chapter_title_addnew_pattern',
+                 'mark_new_chapters',
                  'check_next_chapter',
                  'skip_author_cover',
                  'collect_series',
@@ -195,8 +233,10 @@ def get_valid_keywords():
                  'datePublished_format',
                  'dateUpdated_format',
                  'default_cover_image',
+                 'description_limit',
                  'do_update_hook',
                  'exclude_notes',
+                 'exclude_editor_signature',
                  'extra_logpage_entries',
                  'extra_subject_tags',
                  'extra_titlepage_entries',
@@ -248,6 +288,7 @@ def get_valid_keywords():
                  'output_filename_safepattern',
                  'password',
                  'post_process_cmd',
+                 'rating_titles',
                  'remove_transparency',
                  'replace_br_with_p',
                  'replace_hr',
@@ -302,13 +343,20 @@ def make_generate_cover_settings(param):
 
 class Configuration(ConfigParser.SafeConfigParser):
 
-    def __init__(self, site, fileform):
+    def __init__(self, sections, fileform):
+        site = sections[-1] # first section is site DN.
         ConfigParser.SafeConfigParser.__init__(self)
 
         self.linenos=dict() # key by section or section,key -> lineno
         
-        self.sectionslist = ['defaults']
+        ## [injected] section has even less priority than [defaults]
+        self.sectionslist = ['defaults','injected']
 
+        ## add other sections (not including site DN) after defaults,
+        ## but before site-specific.
+        for section in sections[:-1]:
+            self.addConfigSection(section)
+        
         if site.startswith("www."):
             sitewith = site
             sitewithout = site.replace("www.","")
@@ -318,8 +366,13 @@ class Configuration(ConfigParser.SafeConfigParser):
         
         self.addConfigSection(sitewith)
         self.addConfigSection(sitewithout)
+        
         if fileform:
             self.addConfigSection(fileform)
+            ## add other sections:fileform (not including site DN)
+            ## after fileform, but before site-specific:fileform.
+            for section in sections[:-1]:
+                self.addConfigSection(section+":"+fileform)
             self.addConfigSection(sitewith+":"+fileform)
             self.addConfigSection(sitewithout+":"+fileform)
         self.addConfigSection("overrides")
@@ -407,6 +460,30 @@ class Configuration(ConfigParser.SafeConfigParser):
         else:
             return self.linenos.get(section,None)
     
+    ## Copied from Python 2.7 library so as to make read utf8.
+    def read(self, filenames):
+        """Read and parse a filename or a list of filenames.
+        Files that cannot be opened are silently ignored; this is
+        designed so that you can specify a list of potential
+        configuration file locations (e.g. current directory, user's
+        home directory, systemwide directory), and all existing
+        configuration files in the list will be read.  A single
+        filename may also be given.
+        Return list of successfully read files.
+        """
+        if isinstance(filenames, basestring):
+            filenames = [filenames]
+        read_ok = []
+        for filename in filenames:
+            try:
+                fp = codecs.open(filename,encoding='utf-8')
+            except IOError:
+                continue
+            self._read(fp, filename)
+            fp.close()
+            read_ok.append(filename)
+        return read_ok
+    
     ## Copied from Python 2.7 library so as to make it save linenos too.
     #
     # Regular expressions for parsing section headers and options.
@@ -609,3 +686,13 @@ class Configurable(object):
 
     def get_config_list(self, sections, key):
         return self.configuration.get_config_list(sections,key)
+
+    def get_label(self, entry):
+        if self.hasConfig(entry+"_label"):
+            label=self.getConfig(entry+"_label")
+        elif entry in titleLabels:
+            label=titleLabels[entry]
+        else:
+            label=entry.title()
+        return label
+

fanficfare/defaults.ini

@@ -85,9 +85,9 @@ authorId_label:Author ID
 ## show up in Calibre as tags.  Also carried into mobi when converted.
 extratags_label:Extra Tags
 ## The version of fanficdownloader
-version_label:FFDL Version
+version_label:Downloader Version
 
-## Date formats used by FFDL.  Published and Update don't have time.
+## Date formats used by FanFicFare.  Published and Update don't have time.
 ## See http://docs.python.org/library/datetime.html#strftime-strptime-behavior
 ## Note that ini format requires % to be escaped as %%.
 dateCreated_format:%%Y-%%m-%%d %%H:%%M:%%S
@@ -205,7 +205,7 @@ connect_timeout:60.0
 ## Note that if metakey == conditionalkey the conditional is ignored.
 ## You can use \s in the replacement to add explicit spaces.  (The config parser
 ## tends to discard trailing spaces.)
-## replace_metadata <entry>_LIST options: FFDL replace_metadata lines
+## replace_metadata <entry>_LIST options: FanFicFare replace_metadata lines
 ## operate on individual list items for list entries. But if you
 ## want to do a replacement on the joined string for the whole list,
 ## you can by using <entry>_LIST. Example, if you added
@@ -268,6 +268,11 @@ keep_summary_html:true
 ## kept.  Some sites(AO3) use <a title="Translation">Foreign language</a>
 #keep_title_attr: false
 
+## If a chapter range was given, use this pattern for the book title.
+## replace_metadata and include/exclude will be applied *after* this.
+## Set to empty value to disable.
+title_chapter_range_pattern:${title} (Ch ${first}-${last})
+
 ## Don't like the numbers at the start of chapter titles on some
 ## sites?  You can use strip_chapter_numbers to strip them off.  Just
 ## want to make them all look the same?  Strip them off, then add them
@@ -300,32 +305,119 @@ chapter_title_strip_pattern:^[0-9]+[\.: -]+
 ## etc
 #chapter_title_strip_pattern:^([0-9]+[\.: -]+)?(Chapter *[0-9]+[\.:, -]*)?
 
-## Uses a python template substitution.  The ${index} is the 'chapter'
-## number and ${title} is the chapter title, after applying
-## chapter_title_strip_pattern.  Those are the only variables available.
-## "The Beginning" => "1. The Beginning" 
+## If true, when updating an epub that already has old chapters, new
+## chapters will be marked in the TOC and chapter header by using
+## chapter_title_new_pattern and chapter_title_addnew_pattern to set the chapter.
+mark_new_chapters:false
+
+## chapter title patterns use python template substitution.  The
+## ${index} is the 'chapter' number and ${title} is the chapter title,
+## after applying chapter_title_strip_pattern.  Those are the only
+## variables available.
+
+## The basic pattern used when not using add_chapter_numbers or
+## mark_new_chapters
+chapter_title_def_pattern:${title}
+
+## Pattern used with add_chapter_numbers, but not mark_new_chapters
 chapter_title_add_pattern:${index}. ${title}
 
+## Pattern used with mark_new_chapters, but not add_chapter_numbers
+## (new) is just text and can be changed.
+chapter_title_new_pattern:(new) ${title}
+
+## Pattern used with add_chapter_numbers and mark_new_chapters
+## (new) is just text and can be changed.
+chapter_title_addnew_pattern:${index}. (new) ${title}
+
 ## Reorder ships so b/a and c/b/a become a/b and a/b/c. Only separates
 ## on '/', so use replace_metadata to change separator first if
 ## needed. Something like: ships=>[ ]*(/|&amp;|&)[ ]*=>/ You can use
 ## ships_LIST to change the / back to something else if you want.
 sort_ships:false
 
-## join_string_<entry> options -- FFDL list entries are comma
+## join_string_<entry> options -- FanFicFare list entries are comma
 ## separated by default. You can use this to change that. For example,
 ## if you want authors separated with ' & ' instead, use
 ## join_string_calibre_author:\s&\s. (\s == space)
 #join_string_author:,\s
 
-## keep_in_order_<entry> options: FFDL sorts list entries by default
+## keep_in_order_<entry> options: FanFicFare sorts list entries by default
 ## (except for author/authorUrl/authorId). But if you want to use an
 ## extra entry derived from author, it ends up sorted. For example, if
 ## you added calibre_author: keep_in_order_calibre_author:true
 #keep_in_order_author:true
 
 ## User-agent
-user_agent:FFDL/2.0
+user_agent:FFF/2.X
+
+## Added for [base_xenforoforum], but can be used with other sites,
+## too.  Limit the 'description' to the first X *characters*
+## collected.  Character count includes HTML tags, so it can be
+## non-intuitive.
+#description_limit:1000
+
+[base_efiction]
+## At the time of writing, eFiction Base adapters allow downloading
+## the whole story in bulk using the 'Print' feature. If 'bulk_load'
+## is set to 'true', both metadata and chapters can be loaded in one
+## step
+bulk_load:true
+
+[base_xenforoforum]
+## Currently only forums.spacebattles.com and forums.sufficientvelocity.com
+
+cover_exclusion_regexp:/clear.png
+
+## I saw lots of chapters name simply '1.1' etc during testing.
+strip_chapter_numbers:false
+
+## Copy title to tagsfromtitle for parsing tags.
+add_to_extra_valid_entries:,tagsfromtitle
+
+## '.NOREPL' tells the system to *not* apply title's
+## in/exclude/replace_metadata -- Only works on include_in_ lines.
+include_in_tagsfromtitle:title.NOREPL
+
+tagsfromtitle_label:Tags from Title
+
+## might want to do this, maybe not.  Will often include category, but
+## also often include non-category stuff.
+# include_in_category:tagsfromtitle 
+
+add_to_include_metadata_pre:
+# only keep tagsfromtitle with ( or [ in.
+ tagsfromtitle=~[\[\(]
+ 
+add_to_replace_metadata:
+# remove anything outside () or []
+ tagsfromtitle=>^.*?([\(\[]([^\]\)]+)[\)\]]).*?$=>\1
+# remove () []
+ tagsfromtitle=>[\(\)\[\]]=>
+# change (spaces)slash(or semicolon)(spaces) to comma
+ tagsfromtitle=> *[/;] *=>,
+ tagsfromtitle=> [xX] =>,
+ 
+# remove [] or () blocks and leading/trailing spaces/dashes/colons
+ title=>[-: ]*[\(\[]([^\]\)]+)[\)\]][-: ]*=>
+# remove 'Thread' and the next word, usually "Thread 2", "Thread
+# four", "Thread iv", etc
+ title,tagsfromtitle=>[-: ]*[Tt]hread [^ ]+[-: ]*=>
+
+add_to_extra_titlepage_entries:,tagsfromtitle
+
+## '.SPLIT' tells the system to split by ','
+add_to_include_subject_tags:,tagsfromtitle.SPLIT
+
+## base_xenforoforum reads Published and Updated datetimes from
+## Threadmarks if used, or from the posted & updated times of the
+## 'first' post if no threadmarks.
+datePublished_format:%%Y-%%m-%%d %%H:%%M:%%S
+dateUpdated_format:%%Y-%%m-%%d %%H:%%M:%%S
+
+## Only take the first X characters of the 'first' post to use as
+## the description.
+description_limit:500
 
 ## Each output format has a section that overrides [defaults]
 [html]
@@ -341,7 +433,7 @@ user_agent:FFDL/2.0
 #output_filename: books/${author}/${title}/${title}-${siteabbrev}_${authorId}_${storyId}${formatext}
 #zip_output: false
 
-## This switch prevents FFDL from doing any processing on the images.
+## This switch prevents FanFicFare from doing any processing on the images.
 ## Usually they would be converted to jpg, resized and optionally made
 ## grayscale.
 no_image_processing: true
@@ -503,6 +595,11 @@ convert_images_to: jpg
 ## Remove transparency and fill with background_color if true.
 remove_transparency: true
 
+## This switch prevents FanFicFare from doing any processing on the images.
+## Usually they would be converted to jpg, resized and optionally made
+## grayscale.
+#no_image_processing: false
+
 ## if the <img> tag doesn't have a div or a p around it, nook gets
 ## confused and displays it on every page after that under the text
 ## for the rest of the chapter.  I doubt adding a div around the img
@@ -573,10 +670,12 @@ extratags: FanFiction,Testing,HTML
 #is_adult:true
 
 ## archiveofourown.org stories allow chapters to be added out of
-## order.  So the newest chapter may not be the last one.  FFDL update
+## order.  So the newest chapter may not be the last one.  FanFicFare update
 ## doesn't like that.  If do_update_hook is uncommented and set true,
 ## the adapter will discard all existing chapters from the newest one
 ## on when updating to enforce accurate chapters.
+## Starting July 2015, FFF stores chapter URLs in the chapter files.
+## Stories downloaded after that shouldn't need this setting anymore.
 #do_update_hook:false
 
 ## AO3 adapter defines a few extra metadata entries.
@@ -592,6 +691,18 @@ hits_label:Hits
 collections_label:Collections
 bookmarks_label:Bookmarks
 
+## AO3 doesn't have anything it calls 'genre'.  The adapter used to be
+## hardcoded to include the site specific metadata freeformtags &
+## ao3categories in the standard metadata field genre.  By making it
+## configurable, users can change it.
+include_in_genre: freeformtags, ao3categories
+
+## AO3 uses the word 'category' differently than most sites.  The
+## adapter used to be hardcoded to include the site specific metadata
+## fandom in the standard metadata field category.  By making it
+## configurable, users can change it.
+include_in_category:fandoms
+
 ## freeformtags was previously typo'ed as freefromtags.  This way,
 ## freefromtags will still work for people who've used it.
 include_in_freefromtags:freeformtags
@@ -603,7 +714,7 @@ include_in_freefromtags:freeformtags
 #extra_subject_tags:fandoms,freeformtags,ao3categories
 
 ## AO3 chapters can include several different types of notes.  We've
-## traditional included them all in the chapter text, but this allows
+## traditionally included them all in the chapter text, but this allows
 ## you to customize which you include.  Copy this parameter to your
 ## personal.ini and list the ones you don't want.
 #exclude_notes:authorheadnotes,chaptersummary,chapterheadnotes,chapterfootnotes,authorfootnotes
@@ -753,11 +864,6 @@ extraships:Spike/Buffy
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
-## Virtually all eFiction-based sites allow downloading the whole story in
-## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
-## metadata and chapters can be loaded in one step
-bulk_load:true
-
 [dramione.org]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -782,7 +888,7 @@ cover_exclusion_regexp:/images/.*?ribbon.gif
 
 ## Some adapters collect additional meta information beyond the
 ## standard ones.  They need to be defined in extra_valid_entries to
-## tell the rest of the FFDL system about them.  They can be used in
+## tell the rest of the FanFicFare system about them.  They can be used in
 ## include_subject_tags, titlepage_entries, extra_titlepage_entries,
 ## logpage_entries, extra_logpage_entries, and include_in_* config
 ## items.  You can also add additional entries here to build up
@@ -920,11 +1026,6 @@ extraships:Harry Potter/Hermione Granger
 #username:YourName
 #password:yourpassword
 
-## Virtually all eFiction-based sites allow downloading the whole story in
-## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
-## metadata and chapters can be loaded in one step
-bulk_load:true
-
 extra_valid_entries: readings,romance
 extra_titlepage_entries: readings,romance
 readings_label: Readings
@@ -1014,6 +1115,12 @@ extra_valid_entries:size
 # don't show twitter icon.
 cover_exclusion_regexp:/res/css/bir.png
 
+[forums.spacebattles.com]
+## see [base_xenforoforum]
+
+[forums.sufficientvelocity.com]
+## see [base_xenforoforum]
+
 [grangerenchanted.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -1060,6 +1167,18 @@ extracategories:In Death
 ## cover image.  This lets you exclude them.
 cover_exclusion_regexp:/public/style_emoticons/.*
 
+[it-could-happen.net]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
+## RPF == http://en.wikipedia.org/wiki/Real_person_fiction
+## Site dedicated to these categories/characters/ships
+extracategories:Glee RPF
+extracharacters:Darren Criss, Chris Colfer
+extraships:Darren Criss/Chris Colfer
+
 [ksarchive.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:Star Trek
@@ -1079,11 +1198,6 @@ eroticatags_label:Erotica Tags
 extra_titlepage_entries: eroticatags
 
 [lotrfanfiction.com]
-## Virtually all eFiction-based sites allow downloading the whole story in
-## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
-## metadata and chapters can be loaded in one step
-bulk_load:true
-
 extra_valid_entries: readings
 readings_label: Readings
 
@@ -1137,6 +1251,12 @@ extracategories:NCIS
 extracategories:Buffy: The Vampire Slayer
 extracharacters:Willow
 
+[ninelives.dark-solace.org]
+## Site dedicated to these categories/characters/ships
+extracategories:The Walking Dead
+extracharacters:Carol,Daryl
+extraships:Carol/Daryl
+
 [nocturnal-light.net]
 ## Extra metadata that this adapter knows about.  See [dramione.org]
 ## for examples of how to use them.
@@ -1325,6 +1445,14 @@ universe_as_series: true
 ## cover image.  This lets you exclude them.
 cover_exclusion_regexp:/css/bir.png
 
+[tgstorytime.com]
+## Site dedicated to these categories/characters/ships
+extracategories:Transgender
+
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.
+#is_adult:true
+
 [thehexfiles.net]
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter
@@ -1344,11 +1472,6 @@ extraships:Harry Potter/Draco Malfoy
 ## personal.ini, not defaults.ini.
 #is_adult:true
 
-## Virtually all eFiction-based sites allow downloading the whole story in
-## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
-## metadata and chapters can be loaded in one step
-bulk_load:true
-
 ## Site dedicated to these categories/characters/ships
 extracategories:Criminal Minds
 
@@ -1358,11 +1481,6 @@ extracategories:Criminal Minds
 ## personal.ini, not defaults.ini.
 #is_adult:true
 
-## Virtually all eFiction-based sites allow downloading the whole story in
-## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
-## metadata and chapters can be loaded in one step
-bulk_load:true
-
 extra_valid_entries: readings,challenge
 extra_titlepage_entries: readings,challenge
 challenge_label: Challenge
@@ -1431,6 +1549,8 @@ extra_titlepage_entries:readings,awards
 awards_label:Awards
 readings_label:Readings
 
+## some sites include images that we don't ever want becoming the
+## cover image.  This lets you exclude them.
 cover_exclusion_regexp:art/.*Awards.jpg
 
 [voracity2.e-fic.com]
@@ -1533,7 +1653,7 @@ extra_valid_entries:reviews,favs,follows
 ## to be romantic pairings.
 ships_label:Pairings
 
-## Date formats used by FFDL.  Published and Update don't usually have
+## Date formats used by FanFicFare.  Published and Update don't usually have
 ## time, but they do now on ffnet.
 ## See http://docs.python.org/library/datetime.html#strftime-strptime-behavior
 ## Note that ini format requires % to be escaped as %%.
@@ -1590,7 +1710,7 @@ cover_exclusion_regexp:(/imageu/|d_60_90\.jpg)
 ## reducing this sleep.
 slow_down_sleep_time:4
 
-## Date formats used by FFDL.  Published and Update don't usually have
+## Date formats used by FanFicFare.  Published and Update don't usually have
 ## time, but they do now on ffnet.
 ## See http://docs.python.org/library/datetime.html#strftime-strptime-behavior
 ## Note that ini format requires % to be escaped as %%.
@@ -1610,7 +1730,7 @@ dateUpdated_format:%%Y-%%m-%%d %%H:%%M:%%S
 #fail_on_password: false
 
 ## fimfiction.net stories allow chapters to be added out of order.  So
-## the newest chapter may not be the last one.  FFDL update doesn't
+## the newest chapter may not be the last one.  FanFicFare update doesn't
 ## like that.  If do_update_hook is uncommented and set true, the
 ## adapter will discard all existing chapters from the newest one on
 ## when updating to enforce accurate chapters.
@@ -1621,6 +1741,10 @@ dateUpdated_format:%%Y-%%m-%%d %%H:%%M:%%S
 ## to on, but can be switched off if it is found to cause problems.
 fix_fimf_blockquotes:true
 
+## some sites include images that we don't ever want becoming the
+## cover image.  This lets you exclude them.
+cover_exclusion_regexp:/images/emoticons/
+
 ## Site dedicated to these categories/characters/ships
 extracategories:My Little Pony: Friendship is Magic
 
@@ -1704,10 +1828,27 @@ extraships:InuYasha/Kagome
 ## Site dedicated to these categories/characters/ships
 extracategories:Lord of the Rings
 
-## Virtually all eFiction-based sites allow downloading the whole story in
-## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
-## metadata and chapters can be loaded in one step
-bulk_load:true
+[www.masseffect2.in]
+## Site dedicated to this fandom.
+extracategories: Mass Effect
+
+## Ucoz.com engine, upon which MassEffect2.in is based, imposes an unspecified limit on request frequency.
+## Reports vary from `5 requests per second' to `2 requests per second for more than 10 per minute'.
+## With default settings, a several-hours IP ban may follow, so set it higher.
+slow_down_sleep_time: 2
+
+## Whether to exclude editor signature from the bottom of chapter text.
+exclude_editor_signature: false
+
+## Stories on the site almost never have cover image, and for the stories which do,
+## this may be adjusted in `personal.ini' before downloading.
+never_make_cover: true
+
+## Titles for ratings identified by 1- or 2-letter codes from `ERATING system'
+## (`система Р.Е.Й.Т.И.Н.Г.').  MassEffect2.in and some other sites adopted it,
+## but changed titles and update them occasionally.
+rating_titles: R=RESTRICTED (16+), E=EXEMPT (18+), I=ART HOUSE, T=To every, A=IN=Иной мир, Nn=Новый мир, G=О\, Господи!
+adult_ratings: E,R
 
 [www.mediaminer.org]
 
@@ -1743,7 +1884,7 @@ extraships:Nick Stokes/Greg Sanders
 
 ## phoenixsong.net, oddly, can have high rated chapters (login
 ## required) in the middle of a lower rated story.  Use this to force
-## FFDL to always login to phoenixsong.net so those stories download
+## FanFicFare to always login to phoenixsong.net so those stories download
 ## correctly.  If you have a login, this is recommended.
 #force_login:true
 

fanficfare/epubutils.py

@@ -1,18 +1,17 @@
-#!/usr/bin/env python
-# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
-from __future__ import (unicode_literals, division, absolute_import,
-                        print_function)
+# -*- coding: utf-8 -*-
 
 __license__   = 'GPL v3'
-__copyright__ = '2014, Jim Miller'
+__copyright__ = '2015, Jim Miller'
 __docformat__ = 'restructuredtext en'
 
 import logging
 logger = logging.getLogger(__name__)
 
 import re, os, traceback
-from zipfile import ZipFile
+from collections import defaultdict
+from zipfile import ZipFile, ZIP_STORED, ZIP_DEFLATED
 from xml.dom.minidom import parseString
+from StringIO import StringIO
 
 import bs4 as bs
 
@@ -90,11 +89,13 @@ def get_update_data(inputio,
 
     filecount = 0
     soups = [] # list of xhmtl blocks
+    urlsoups = {} # map of xhtml blocks by url
     images = {} # dict() longdesc->data
+    datamaps = defaultdict(dict) # map of data maps by url
     if getfilecount:
         # spin through the manifest--only place there are item tags.
         for item in contentdom.getElementsByTagName("item"):
-            # First, count the 'chapter' files.  FFDL uses file0000.xhtml,
+            # First, count the 'chapter' files.  FFF uses file0000.xhtml,
             # but can also update epubs downloaded from Twisting the
             # Hellmouth, which uses chapter0.html.
             if( item.getAttribute("media-type") == "application/xhtml+xml" ):
@@ -124,20 +125,45 @@ def get_update_data(inputio,
                                 logger.warn("Image %s not found!\n(originally:%s)"%(newsrc,longdesc))
                                 logger.warn("Exception: %s"%(unicode(e)))
                                 traceback.print_exc()
-                        soup = soup.find('body')
+                        bodysoup = soup.find('body')
                         # ffdl epubs have chapter title h3
-                        h3 = soup.find('h3')
+                        h3 = bodysoup.find('h3')
                         if h3:
                             h3.extract()
                         # TtH epubs have chapter title h2
-                        h2 = soup.find('h2')
+                        h2 = bodysoup.find('h2')
                         if h2:
                             h2.extract()
 
-                        for skip in soup.findAll(attrs={'class':'skip_on_ffdl_update'}):
+                        for skip in bodysoup.findAll(attrs={'class':'skip_on_ffdl_update'}):
                             skip.extract()
+
+                        ## <meta name="chapterurl" content="${url}"></meta>
+                        #print("look for meta chapurl")
+                        currenturl = None
+                        chapurl = soup.find('meta',{'name':'chapterurl'})
+                        if chapurl:
+                            if chapurl['content'] not in urlsoups: # keep first found if more than one.
+                            #print("Found chapurl['content']:%s"%chapurl['content'])
+                                currenturl = chapurl['content']
+                                urlsoups[chapurl['content']] = bodysoup
+                        else:
+                            # for older pre-meta.  Only temp.
+                            chapa = bodysoup.find('a',{'class':'chapterurl'})
+                            if chapa and chapa['href'] not in urlsoups: # keep first found if more than one.
+                                urlsoups[chapa['href']] = bodysoup
+                                currenturl = chapa['href']
+                                chapa.extract()
+
+                        chapterorigtitle = soup.find('meta',{'name':'chapterorigtitle'})
+                        if chapterorigtitle:
+                            datamaps[currenturl]['chapterorigtitle'] = chapterorigtitle['content']
                             
-                        soups.append(soup)
+                        chaptertitle = soup.find('meta',{'name':'chaptertitle'})
+                        if chaptertitle:
+                            datamaps[currenturl]['chaptertitle'] = chaptertitle['content']
+                            
+                        soups.append(bodysoup)
                         
                     filecount+=1
 
@@ -148,7 +174,8 @@ def get_update_data(inputio,
                     
     #for k in images.keys():
         #print("\tlongdesc:%s\n\tData len:%s\n"%(k,len(images[k])))
-    return (source,filecount,soups,images,oldcover,calibrebookmark,logfile)
+    # print("datamaps:%s"%datamaps)
+    return (source,filecount,soups,images,oldcover,calibrebookmark,logfile,urlsoups,datamaps)
 
 def get_path_part(n):
     relpath = os.path.dirname(n)
@@ -175,7 +202,7 @@ def get_story_url_from_html(inputio,_is_good_url=None):
             
     # spin through the manifest--only place there are item tags.
     for item in contentdom.getElementsByTagName("item"):
-        # First, count the 'chapter' files.  FFDL uses file0000.xhtml,
+        # First, count the 'chapter' files.  FFF uses file0000.xhtml,
         # but can also update epubs downloaded from Twisting the
         # Hellmouth, which uses chapter0.html.
         #print("---- item:%s"%item)
@@ -192,3 +219,105 @@ def get_story_url_from_html(inputio,_is_good_url=None):
                 if _is_good_url == None or _is_good_url(ahref):
                     return ahref
     return None
+
+def reset_orig_chapters_epub(inputio,outfile):
+    inputepub = ZipFile(inputio, 'r') # works equally well with a path or a blob
+
+    ## build zip in memory in case updating in place(CLI).
+    zipio = StringIO()
+    
+    ## Write mimetype file, must be first and uncompressed.
+    ## Older versions of python(2.4/5) don't allow you to specify
+    ## compression by individual file.
+    ## Overwrite if existing output file.
+    outputepub = ZipFile(zipio, 'w', compression=ZIP_STORED)
+    outputepub.debug = 3
+    outputepub.writestr("mimetype", "application/epub+zip")
+    outputepub.close()
+
+    ## Re-open file for content.
+    outputepub = ZipFile(zipio, "a", compression=ZIP_DEFLATED)
+    outputepub.debug = 3
+
+    changed = False
+    
+    tocncxdom = parseString(inputepub.read('toc.ncx'))
+    ## spin through file contents.
+    for zf in inputepub.namelist():
+        if zf not in ['mimetype','toc.ncx'] :
+            entrychanged = False
+            data = inputepub.read(zf)
+            # if isinstance(data,unicode):
+            #     logger.debug("\n\n\ndata is unicode\n\n\n")
+            if re.match(r'.*/file\d+\.xhtml',zf):
+                data = data.decode('utf-8')
+                soup = bs.BeautifulSoup(data,"html5lib")
+                
+                chapterorigtitle = None
+                tag = soup.find('meta',{'name':'chapterorigtitle'})
+                if tag:
+                    chapterorigtitle = tag['content']
+
+                # toctitle is separate for add_chapter_numbers:toconly users.
+                chaptertoctitle = None
+                tag = soup.find('meta',{'name':'chaptertoctitle'})
+                if tag:
+                    chaptertoctitle = tag['content']
+                elif chapterorigtitle:
+                    chaptertoctitle = chapterorigtitle
+                    
+                chaptertitle = None
+                tag = soup.find('meta',{'name':'chaptertitle'})
+                if tag:
+                    chaptertitle = tag['content']
+
+                if chaptertitle and chapterorigtitle and chapterorigtitle != chaptertitle:
+                    origdata = data
+                    # print("\n%s\n%s\n"%(chapterorigtitle,chaptertitle))
+                    data = data.replace(u'<meta name="chaptertitle" content="'+chaptertitle+u'"></meta>',
+                                        u'<meta name="chaptertitle" content="'+chapterorigtitle+u'"></meta>')
+                    data = data.replace(u'<title>'+chaptertitle+u'</title>',u'<title>'+chapterorigtitle+u'</title>')
+                    data = data.replace(u'<h3>'+chaptertitle+u'</h3>',u'<h3>'+chapterorigtitle+u'</h3>')
+
+                    entrychanged = ( origdata != data )
+                    changed = changed or entrychanged
+                    
+                    if entrychanged:
+                        ## go after the TOC entry, too.
+                        # <navPoint id="file0005" playOrder="6">
+                        #   <navLabel>
+                        #     <text>5. (new) Chapter 4</text>
+                        #   </navLabel>
+                        #   <content src="OEBPS/file0005.xhtml"/>
+                        # </navPoint>
+                        for contenttag in tocncxdom.getElementsByTagName("content"):
+                            if contenttag.getAttribute('src') == zf:
+                                texttag = contenttag.parentNode.getElementsByTagName('navLabel')[0].getElementsByTagName('text')[0]
+                                texttag.childNodes[0].replaceWholeText(chaptertoctitle)
+                                # logger.debug("text label:%s"%texttag.toxml())
+                                continue
+                    
+                outputepub.writestr(zf,data.encode('utf-8'))
+            else:
+                # possibly binary data, thus no .encode().
+                outputepub.writestr(zf,data)
+
+    outputepub.writestr('toc.ncx',tocncxdom.toxml(encoding='utf-8'))
+    outputepub.close()
+    # declares all the files created by Windows.  otherwise, when
+    # it runs in appengine, windows unzips the files as 000 perms.
+    for zf in outputepub.filelist:
+        zf.create_system = 0
+        
+    # only *actually* write if changed.
+    if changed:
+        if isinstance(outfile,basestring):
+            with open(outfile,"wb") as outputio:
+                outputio.write(zipio.getvalue())
+        else:
+            outfile.write(zipio.getvalue())
+
+    inputepub.close()
+    zipio.close()
+            
+    return changed

fanficfare/exceptions.py

@@ -24,6 +24,13 @@ class FailedToDownload(Exception):
     def __str__(self):
         return self.error
 
+class RejectImage(Exception):
+    def __init__(self,error):
+        self.error=error
+
+    def __str__(self):
+        return self.error
+
 class InvalidStoryURL(Exception):
     def __init__(self,url,domain,example):
         self.url=url

fanficfare/geturls.py

@@ -15,12 +15,15 @@
 # limitations under the License.
 #
 
-import re
-import urlparse
-import urllib2 as u2
-
-import imaplib
+import collections
 import email
+import imaplib
+import re
+import urllib2 as u2
+import urlparse
+
+import logging
+logger = logging.getLogger(__name__)
 
 from BeautifulSoup import BeautifulSoup 
 from gziphttp import GZipProcessor
@@ -74,28 +77,25 @@ def get_urls_from_page(url,configuration=None,normalize=False):
     return get_urls_from_html(data,url,configuration,normalize,restrictsearch)
 
 def get_urls_from_html(data,url=None,configuration=None,normalize=False,restrictsearch=None):
+    urls = collections.OrderedDict()
 
-    normalized = [] # normalized url
-    retlist = [] # orig urls.
-    
     if not configuration:
         configuration = Configuration("test1.com","EPUB")
 
     soup = BeautifulSoup(data)
     if restrictsearch:
         soup = soup.find(*restrictsearch)
-        #print("restrict search:%s"%soup)
+        #logger.debug("restrict search:%s"%soup)
     
     for a in soup.findAll('a'):
         if a.has_key('href'):
-            #print("a['href']:%s"%a['href'])
+            #logger.debug("a['href']:%s"%a['href'])
             href = form_url(url,a['href'])
-            #print("1 urlhref:%s"%href)
+            #logger.debug("1 urlhref:%s"%href)
             # this (should) catch normal story links, some javascript
             # 'are you old enough' links, and 'Report This' links.
-            # The 'normalized' set prevents duplicates.
             if 'story.php' in a['href']:
-                #print("trying:%s"%a['href'])
+                #logger.debug("trying:%s"%a['href'])
                 m = re.search(r"(?P<sid>(view)?story\.php\?(sid|psid|no|story|stid)=\d+)",a['href'])
                 if m != None:
                     href = form_url(a['href'] if '//' in a['href'] else url,
@@ -103,34 +103,32 @@ def get_urls_from_html(data,url=None,configuration=None,normalize=False,restrict
                     
             try:
                 href = href.replace('&index=1','')
-                #print("2 urlhref:%s"%href)
+                #logger.debug("2 urlhref:%s"%href)
                 adapter = adapters.getAdapter(configuration,href)
-                #print("found adapter")
-                if adapter.story.getMetadata('storyUrl') not in normalized:
-                    normalized.append(adapter.story.getMetadata('storyUrl'))
-                    retlist.append(href)
+                #logger.debug("found adapter")
+                if adapter.story.getMetadata('storyUrl') not in urls:
+                    urls[adapter.story.getMetadata('storyUrl')] = [href]
+                else:
+                    urls[adapter.story.getMetadata('storyUrl')].append(href)
             except Exception, e:
-                #print e
+                #logger.debug e
                 pass
 
-    if normalize:
-        return normalized
-    else:
-        return retlist
+    # Simply return the longest URL with the assumption that it contains the
+    # most user readable metadata, if not normalized
+    return urls.keys() if normalize else [max(value, key=len) for key, value in urls.items()]
+
 
 def get_urls_from_text(data,configuration=None,normalize=False):
-
-    normalized = [] # normalized url
-    retlist = [] # orig urls.
+    urls = collections.OrderedDict()
     data=unicode(data)
-    
+
     if not configuration:
         configuration = Configuration("test1.com","EPUB")
     
     for href in re.findall('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+', data):
         # this (should) catch normal story links, some javascript
         # 'are you old enough' links, and 'Report This' links.
-        # The 'normalized' set prevents duplicates.
         if 'story.php' in href:
             m = re.search(r"(?P<sid>(view)?story\.php\?(sid|psid|no|story|stid)=\d+)",href)
             if m != None:
@@ -138,16 +136,17 @@ def get_urls_from_text(data,configuration=None,normalize=False):
         try:
             href = href.replace('&index=1','')
             adapter = adapters.getAdapter(configuration,href)
-            if adapter.story.getMetadata('storyUrl') not in normalized:
-                normalized.append(adapter.story.getMetadata('storyUrl'))
-                retlist.append(href)
+            if adapter.story.getMetadata('storyUrl') not in urls:
+                urls[adapter.story.getMetadata('storyUrl')] = [href]
+            else:
+                urls[adapter.story.getMetadata('storyUrl')].append(href)
         except:
             pass
 
-    if normalize:
-        return normalized
-    else:
-        return retlist
+    # Simply return the longest URL with the assumption that it contains the
+    # most user readable metadata, if not normalized
+    return urls.keys() if normalize else [max(value, key=len) for key, value in urls.items()]
+
 
 def form_url(parenturl,url):
      url = url.strip() # ran across an image with a space in the
@@ -177,7 +176,8 @@ def form_url(parenturl,url):
      return returl
        
 def get_urls_from_imap(srv,user,passwd,folder,markread=True):
-    
+
+    logger.debug("get_urls_from_imap srv:(%s)"%srv)
     mail = imaplib.IMAP4_SSL(srv)
     mail.login(user, passwd)
     mail.list()
@@ -186,8 +186,8 @@ def get_urls_from_imap(srv,user,passwd,folder,markread=True):
     
     result, data = mail.uid('search', None, "UNSEEN")
     
-    #print("result:%s"%result)
-    #print("data:%s"%data)
+    #logger.debug("result:%s"%result)
+    #logger.debug("data:%s"%data)
     urls=set()
     
     #latest_email_uid = data[0].split()[-1]
@@ -195,8 +195,8 @@ def get_urls_from_imap(srv,user,passwd,folder,markread=True):
 
         result, data = mail.uid('fetch', email_uid, '(BODY.PEEK[])') #RFC822
     
-        #print("result:%s"%result)
-        #print("data:%s"%data)
+        #logger.debug("result:%s"%result)
+        #logger.debug("data:%s"%data)
     
         raw_email = data[0][1]
         
@@ -205,28 +205,28 @@ def get_urls_from_imap(srv,user,passwd,folder,markread=True):
     
         email_message = email.message_from_string(raw_email)
      
-        #print "To:%s"%email_message['To']
-        #print "From:%s"%email_message['From']
-        #print "Subject:%s"%email_message['Subject']
+        #logger.debug "To:%s"%email_message['To']
+        #logger.debug "From:%s"%email_message['From']
+        #logger.debug "Subject:%s"%email_message['Subject']
     
-    #    print("payload:%s"%email_message.get_payload())
+    #    logger.debug("payload:%s"%email_message.get_payload())
 
         urllist=[]
         for part in email_message.walk():
             try:
-            #print("part mime:%s"%part.get_content_type())
+            #logger.debug("part mime:%s"%part.get_content_type())
                 if part.get_content_type() == 'text/plain':
                     urllist.extend(get_urls_from_text(part.get_payload(decode=True)))
                 if part.get_content_type() == 'text/html':
                     urllist.extend(get_urls_from_html(part.get_payload(decode=True)))
             except Exception as e:
-                print("Failed to read email content: %s"%e)
-        #print "urls:%s"%get_urls_from_text(get_first_text_block(email_message))
+                logger.error("Failed to read email content: %s"%e)
+        #logger.debug "urls:%s"%get_urls_from_text(get_first_text_block(email_message))
 
         if urllist and markread:
             #obj.store(data[0].replace(' ',','),'+FLAGS','\Seen')
             r,d = mail.uid('store',email_uid,'+FLAGS','(\\SEEN)')
-            #print("seen result:%s->%s"%(email_uid,r))
+            #logger.debug("seen result:%s->%s"%(email_uid,r))
                 
         [ urls.add(x) for x in urllist ]
     

fanficfare/htmlcleanup.py

@@ -81,10 +81,10 @@ def removeEntities(text):
     
     try:
         t = text.decode('utf-8')
-    except UnicodeEncodeError, e:
+    except (UnicodeEncodeError,UnicodeDecodeError), e:
         try:
             t = text.encode ('ascii', 'xmlcharrefreplace') 
-        except UnicodeEncodeError, e:
+        except (UnicodeEncodeError,UnicodeDecodeError), e:
             t = text
     text = t 
     # replace numeric versions of [&<>] with named versions,

fanficfare/story.py

@@ -16,18 +16,25 @@
 #
 
 import os, re
+from collections import namedtuple
 import urlparse
 import string
+import json
+import datetime
 from math import floor
 from functools import partial
 import logging
 logger = logging.getLogger(__name__)
 import urlparse as up
 
+import bs4
+
 import exceptions
 from htmlcleanup import conditionalRemoveEntities, removeAllEntities
 from configurable import Configurable, re_compile
 
+Chapter = namedtuple('Chapter', 'url title html origtitle toctitle new')
+
 SPACE_REPLACE=u'\s'
 SPLIT_META=u'\,'
 
@@ -44,6 +51,8 @@ imagetypes = {
 
 try:
     from calibre.utils.magick import Image
+    from StringIO import StringIO
+    from gif import GifInfo, CHECK_IS_ANIMATED
     convtype = {'jpg':'JPG', 'png':'PNG'}
 
     def convert_image(url,data,sizes,grayscale,
@@ -55,6 +64,10 @@ try:
         owidth, oheight = img.size
         nwidth, nheight = sizes
         scaled, nwidth, nheight = fit_image(owidth, oheight, nwidth, nheight)
+
+        if normalize_format_name(img.format)=="gif" and GifInfo(StringIO(data),CHECK_IS_ANIMATED).frameCount > 1:
+            raise exceptions.RejectImage("Animated gifs come out purely--not going to use it.")
+
         if scaled:
             img.size = (nwidth, nheight)
             export = True
@@ -224,7 +237,7 @@ langs = {
     "Devanagari":"hi",
 
     ## These are from/for AO3:
-    
+
     u'العربية':'ar',
     u'беларуская':'be',
     u'Български език':'bg',
@@ -325,7 +338,7 @@ class InExMatch:
         else:
             retval = self.match == value
             #print(">>>>>>>>>>>>>%s==%s r: %s,%s=%s"%(self.match,value,self.negate,retval, self.negate != retval))
-            
+
         return self.negate != retval
 
     def __str__(self):
@@ -338,7 +351,7 @@ class InExMatch:
         else:
             s='='
         return u'InExMatch(%s %s%s %s)'%(self.keys,f,s,self.match)
-    
+
 ## metakey[,metakey]=~pattern
 ## metakey[,metakey]==string
 ## *for* part lines.  Effect only when trailing conditional key=~regexp matches
@@ -358,7 +371,7 @@ def set_in_ex_clude(setting):
             match = InExMatch(line)
             dest.append([match,condmatch])
     return dest
-              
+
 ## Two or three part lines.  Two part effect everything.
 ## Three part effect only those key(s) lists.
 ## pattern=>replacement
@@ -402,7 +415,9 @@ class Story(Configurable):
         except:
             self.metadata = {'version':'4.4'}
         self.in_ex_cludes = {}
-        self.chapters = [] # chapters will be tuples of (title,html)
+        self.chapters = [] # chapters will be namedtuple of Chapter(url,title,html,etc)
+        self.chapter_first = None
+        self.chapter_last = None
         self.imgurls = []
         self.imgtuples = []
 
@@ -431,9 +446,13 @@ class Story(Configurable):
                 iel = []
                 self.in_ex_cludes[ie] = set_in_ex_clude(ies)
 
-    def join_list(self, key, vallist):
-        return self.getConfig("join_string_"+key,u", ").replace(SPACE_REPLACE,' ').join(map(unicode, vallist))
+    def set_chapters_range(self,first=None,last=None):
+        self.chapter_first=first
+        self.chapter_last=last
                 
+    def join_list(self, key, vallist):
+        return self.getConfig("join_string_"+key,u", ").replace(SPACE_REPLACE,' ').join(map(unicode, [ x for x in vallist if x is not None ]))
+
     def setMetadata(self, key, value, condremoveentities=True):
 
         # keep as list type, but set as only value.
@@ -445,20 +464,20 @@ class Story(Configurable):
                 self.metadata[key]=conditionalRemoveEntities(value)
             else:
                 self.metadata[key]=value
-                
+
         if key == "language":
             try:
                 # getMetadata not just self.metadata[] to do replace_metadata.
                 self.setMetadata('langcode',langs[self.getMetadata(key)])
             except:
                 self.setMetadata('langcode','en')
-                
+
         if key == 'dateUpdated' and value:
             # Last Update tags for Bill.
-            self.addToList('lastupdate',value.strftime("Last Update Year/Month: %Y/%m"))
+            self.addToList('lastupdate',value.strftime("Last Update Year/Month: %Y/%m"),clear=True)
             self.addToList('lastupdate',value.strftime("Last Update: %Y/%m/%d"))
 
-        
+
     def do_in_ex_clude(self,which,value,key):
         if value and which in self.in_ex_cludes:
             include = 'include' in which
@@ -487,7 +506,7 @@ class Story(Configurable):
             if include and keyfound and not found:
                 value = None
         return value
-        
+
 
     def doReplacements(self,value,key,return_list=False,seen_list=[]):
         value = self.do_in_ex_clude('include_metadata_pre',value,key)
@@ -526,7 +545,7 @@ class Story(Configurable):
                         # print("replacement,value:%s,%s->%s"%(replacement,value,regexp.sub(replacement,value)))
                         value = regexp.sub(replacement,value)
                         retlist = [value]
-                    
+
         for val in retlist:
             retlist = map(partial(self.do_in_ex_clude,'include_metadata_post',key=key),retlist)
             retlist = map(partial(self.do_in_ex_clude,'exclude_metadata_post',key=key),retlist)
@@ -538,6 +557,69 @@ class Story(Configurable):
         else:
             return self.join_list(key,retlist)
 
+    # for saving an html-ified copy of metadata.
+    def dump_html_metadata(self):
+        lines=[]
+        for k,v in sorted(self.metadata.iteritems()):
+            classes=['metadata']
+            if isinstance(v, (datetime.date, datetime.datetime, datetime.time)):
+                classes.append("datetime")
+                val = v.isoformat()
+            elif isinstance(v,list):
+                classes.append("list")
+                if '' in v:
+                    v.remove('')
+                if None in v:
+                    v.remove(None)
+                #logger.debug("k:%s v:%s"%(k,v))
+                # force ints/floats to strings.
+                val = "<ul>\n<li>%s</li>\n</ul>" % "</li>\n<li>".join([ "%s"%x for x in v ])
+            elif isinstance(v, (int)):
+                classes.append("int")
+                val = v
+            else:
+                val = v
+
+            # don't include items passed in for calibre cols, etc.
+            if not k.startswith('calibre_') and k not in ['output_css']:
+                lines.append("<p><span class='label'>%s</span>: <div class='%s' id='%s'>%s</div><p>\n"%(
+                        self.get_label(k),
+                        " ".join(classes),
+                        k,val))
+        return "\n".join(lines)
+
+    # for loading an html-ified copy of metadata.
+    def load_html_metadata(self,data):
+        soup = bs4.BeautifulSoup(data,'html5lib')
+        for tag in soup.find_all('div','metadata'):
+            val = None
+            if 'datetime' in tag['class']:
+                v = tag.string
+                try:
+                    val = datetime.datetime.strptime(v, '%Y-%m-%dT%H:%M:%S.%f')
+                except ValueError:
+                    try:
+                        val = datetime.datetime.strptime(v, '%Y-%m-%dT%H:%M:%S')
+                    except ValueError:
+                        try:
+                            val = datetime.datetime.strptime(v, '%Y-%m-%d')
+                        except ValueError:
+                            pass
+            elif 'list' in tag['class']:
+                val = []
+                for i in tag.find_all('li'):
+                    val.append(i.string)
+            elif 'int' in tag['class']:
+                val = int(tag.string)
+            else:
+                val = unicode("\n".join([ unicode(c) for c in tag.contents ]))
+                
+            #logger.debug("key(%s)=val(%s)"%(tag['id'],val))
+            if val:
+                self.metadata[tag['id']]=val
+                
+        # self.metadata = json.loads(s, object_hook=datetime_decoder)
+        
     def getMetadataRaw(self,key):
         if self.isValidMetaEntry(key) and self.metadata.has_key(key):
             return self.metadata[key]
@@ -568,6 +650,14 @@ class Story(Configurable):
                 if key in ("datePublished","dateUpdated"):
                     value = value.strftime(self.getConfig(key+"_format","%Y-%m-%d"))
 
+                if key == "title" and (self.chapter_first or self.chapter_last) and self.getConfig("title_chapter_range_pattern"):
+                    first = self.chapter_first or "1"
+                    last = self.chapter_last or self.getMetadata("numChapters")
+                    templ = string.Template(self.getConfig("title_chapter_range_pattern"))
+                    value = templ.substitute({'title':value,
+                                              'first':commaGroups(first),
+                                              'last':commaGroups(last)})
+
             if doreplacements:
                 value=self.doReplacements(value,key)
             if removeallentities and value != None:
@@ -610,7 +700,7 @@ class Story(Configurable):
                                                     self.getMetadata('author', removeallentities, doreplacements)))
 
         self.extendList("extratags",self.getConfigList("extratags"))
-            
+
         if self.getMetadataRaw('seriesUrl'):
             self.setMetadata('seriesHTML',linkhtml%('series',self.getMetadata('seriesUrl', removeallentities, doreplacements),
                                                     self.getMetadata('series', removeallentities, doreplacements)))
@@ -665,9 +755,6 @@ class Story(Configurable):
         if not value in self.metadata[listname]:
             self.metadata[listname].append(value)
 
-        if listname == 'category' and self.getConfig('add_genre_when_multi_category') and len(self.metadata[listname]) > 1:
-            self.addToList('genre',self.getConfig('add_genre_when_multi_category'))
-
     def isList(self,listname):
         'Everything set with an include_in_* is considered a list.'
         return self.isListType(listname) or \
@@ -687,8 +774,12 @@ class Story(Configurable):
         # includelist prevents infinite recursion of include_in_'s
         if self.hasConfig("include_in_"+listname) and listname not in includelist:
             for k in self.getConfigList("include_in_"+listname):
+                ldorepl = doreplacements
+                if k.endswith('.NOREPL'):
+                    k = k[:-len('.NOREPL')]
+                    ldorepl = False
                 retlist.extend(self.getList(k,removeallentities=False,
-                                            doreplacements=doreplacements,includelist=includelist+[listname]))
+                                            doreplacements=ldorepl,includelist=includelist+[listname]))
         else:
 
             if not self.isList(listname):
@@ -696,6 +787,8 @@ class Story(Configurable):
                                             doreplacements=doreplacements)]
             else:
                 retlist = self.getMetadataRaw(listname)
+                if retlist is None:
+                    retlist = []
 
         if retlist:
             if doreplacements:
@@ -703,12 +796,19 @@ class Story(Configurable):
                 for val in retlist:
                     newretlist.extend(self.doReplacements(val,listname,return_list=True))
                 retlist = newretlist
-                
+
             if removeallentities:
                 retlist = map(removeAllEntities,retlist)
-                
+
             retlist = filter( lambda x : x!=None and x!='' ,retlist)
 
+        if listname == 'genre' and self.getConfig('add_genre_when_multi_category') and len(self.getList('category',
+                                                                                                        removeallentities=False,
+                                                                                                        # to avoid inf loops if genre/cat substs
+                                                                                                        doreplacements=False
+                                                                                                        )) > 1:
+            retlist.append(self.getConfig('add_genre_when_multi_category'))
+
         # reorder ships so b/a and c/b/a become a/b and a/b/c.  Only on '/',
         # use replace_metadata to change separator first if needed.
         # ships=>[ ]*(/|&amp;|&)[ ]*=>/
@@ -734,7 +834,16 @@ class Story(Configurable):
 
         # metadata all go into dc:subject tags, but only if they are configured.
         for (name,value) in self.getAllMetadata(removeallentities=removeallentities,keeplists=True).iteritems():
-            if name in tags_list:
+            if name+'.SPLIT' in tags_list:
+                flist=[]
+                if isinstance(value,list):
+                    for tag in value:
+                        flist.extend(tag.split(','))
+                else:
+                    flist.extend(value)
+                for tag in flist:
+                    subjectset.add(tag)
+            elif name in tags_list:
                 if isinstance(value,list):
                     for tag in value:
                         subjectset.add(tag)
@@ -748,24 +857,63 @@ class Story(Configurable):
 
         return list(subjectset | set(self.getConfigList("extratags")))
 
-    def addChapter(self, url, title, html):
+    def addChapter(self, url, title, html, newchap=False):
+        # logger.debug("addChapter(%s,%s)"%(url,newchap))
         if self.getConfig('strip_chapter_numbers') and \
                 self.getConfig('chapter_title_strip_pattern'):
             title = re.sub(self.getConfig('chapter_title_strip_pattern'),"",title)
-        self.chapters.append( (url,title,html) )
+        self.chapters.append( Chapter(url,title,html,title,title,newchap) )
 
     def getChapters(self,fortoc=False):
-        "Chapters will be tuples of (title,html)"
+        "Chapters will be Chapter namedtuples"
         retval = []
-        ## only add numbers if more than one chapter.
-        if len(self.chapters) > 1 and \
-                (self.getConfig('add_chapter_numbers') == "true" \
-                     or (self.getConfig('add_chapter_numbers') == "toconly" and fortoc)) \
-                     and self.getConfig('chapter_title_add_pattern'):
-            for index, (url,title,html) in enumerate(self.chapters):
-                retval.append( (url,
-                                string.Template(self.getConfig('chapter_title_add_pattern')).substitute({'index':index+1,'title':title}),
-                                html) )
+
+        ## only add numbers if more than one chapter.  Ditto (new) marks.
+        if len(self.chapters) > 1:
+            addnums = ( self.getConfig('add_chapter_numbers') == "true" 
+                        or (self.getConfig('add_chapter_numbers') == "toconly" and fortoc) )
+            
+            marknew = self.getConfig('mark_new_chapters')=='true'
+
+            defpattern = self.getConfig('chapter_title_def_pattern','${title}') # default val in case of missing defaults.ini
+            if addnums and marknew:
+                pattern = self.getConfig('chapter_title_add_pattern')
+                newpattern = self.getConfig('chapter_title_addnew_pattern')
+            elif addnums:
+                pattern = self.getConfig('chapter_title_add_pattern')
+                newpattern = pattern
+            elif marknew:
+                pattern = defpattern
+                newpattern = self.getConfig('chapter_title_new_pattern')
+            else:
+                pattern = defpattern
+                newpattern = pattern
+
+            if self.getConfig('add_chapter_numbers') in ["true","toconly"]:
+                tocpattern = self.getConfig('chapter_title_add_pattern')
+            else:
+                tocpattern = defpattern
+
+            # logger.debug("Patterns: (%s)(%s)"%(pattern,newpattern))
+            templ = string.Template(pattern)
+            newtempl = string.Template(newpattern)
+            toctempl = string.Template(tocpattern)            
+            
+            for index, chap in enumerate(self.chapters):
+                if chap.new:
+                    usetempl = newtempl
+                else:
+                    usetempl = templ
+                # logger.debug("chap.url, chap.new: (%s)(%s)"%(chap.url,chap.new))
+                retval.append( Chapter(chap.url,
+                                       # 'new'
+                                       usetempl.substitute({'index':index+1,'title':chap.title}),
+                                       chap.html,
+                                       # 'orig'
+                                       templ.substitute({'index':index+1,'title':chap.title}),
+                                       # 'toc'
+                                       toctempl.substitute({'index':index+1,'title':chap.title}),
+                                       chap.new) )
         else:
             retval = self.chapters
 
@@ -832,6 +980,7 @@ class Story(Configurable):
                 #print("\n===========\nparsedUrl.path:%s\ntoppath:%s\nimgurl:%s\n\n"%(parsedUrl.path,toppath,imgurl))
 
         # apply coverexclusion to explicit covers, too.  Primarily for ffnet imageu.
+        #print("[[[[[\n\n %s %s \n\n]]]]]]]"%(imgurl,coverexclusion))
         if cover and coverexclusion and re.search(coverexclusion,imgurl):
             return (None,None)
 
@@ -874,6 +1023,7 @@ class Story(Configurable):
                 self.imgurls.insert(0,imgurl)
                 newsrc = "images/cover.%s"%ext
                 self.cover=newsrc
+                self.setMetadata('cover_image','specific')
                 self.imgtuples.insert(0,{'newsrc':newsrc,'mime':mime,'data':data})
             else:
                 self.imgurls.append(imgurl)
@@ -888,6 +1038,7 @@ class Story(Configurable):
                         not (coverexclusion and re.search(coverexclusion,imgurl)):
                     newsrc = "images/cover.%s"%ext
                     self.cover=newsrc
+                    self.setMetadata('cover_image','first')
                     self.imgtuples.append({'newsrc':newsrc,'mime':mime,'data':data})
                     self.imgurls.append(imgurl)
 
@@ -921,4 +1072,3 @@ def commaGroups(s):
         groups.append(s[-3:])
         s = s[:-3]
     return s + ','.join(reversed(groups))
-

fanficfare/writers/base_writer.py

@@ -45,36 +45,6 @@ class BaseStoryWriter(Configurable):
         self.adapter = adapter
         self.story = adapter.getStoryMetadataOnly() # only cache the metadata initially.
         
-        # fall back labels.
-        self.titleLabels = {
-            'category':'Category',
-            'genre':'Genre',
-            'language':'Language',
-            'status':'Status',
-            'series':'Series',
-            'characters':'Characters',
-            'ships':'Relationships',
-            'datePublished':'Published',
-            'dateUpdated':'Updated',
-            'dateCreated':'Packaged',
-            'rating':'Rating',
-            'warnings':'Warnings',
-            'numChapters':'Chapters',
-            'numWords':'Words',
-            'site':'Site',
-            'storyId':'Story ID',
-            'authorId':'Author ID',
-            'extratags':'Extra Tags',
-            'title':'Title',
-            'storyUrl':'Story URL',
-            'description':'Summary',
-            'author':'Author',
-            'authorUrl':'Author URL',
-            'formatname':'File Format',
-            'formatext':'File Extension',
-            'siteabbrev':'Site Abbrev',
-            'version':'FFDL Version'
-            }
         self.story.setMetadata('formatname',self.getFormatName())
         self.story.setMetadata('formatext',self.getFormatExt())
 
@@ -135,15 +105,16 @@ class BaseStoryWriter(Configurable):
                             TEMPLATE=WIDE_ENTRY
                         else:
                             TEMPLATE=ENTRY
-                            
-                        if self.hasConfig(entry+"_label"):
-                            label=self.getConfig(entry+"_label")
-                        elif entry in self.titleLabels:
-                            logger.debug("Using fallback label for %s_label"%entry)
-                            label=self.titleLabels[entry]
-                        else:
-                            label="%s"%entry.title()
-                            logger.debug("No known label for %s, fallback to '%s'"%(entry,label))
+
+                        label=self.get_label(entry)
+                        # if self.hasConfig(entry+"_label"):
+                        #     label=self.getConfig(entry+"_label")
+                        # elif entry in self.titleLabels:
+                        #     logger.debug("Using fallback label for %s_label"%entry)
+                        #     label=self.titleLabels[entry]
+                        # else:
+                        #     label="%s"%entry.title()
+                        #     logger.debug("No known label for %s, fallback to '%s'"%(entry,label))
 
                         # If the label for the title entry is empty, use the
                         # 'no title' option if there is one.
@@ -177,12 +148,12 @@ class BaseStoryWriter(Configurable):
             
             self._write(out,START.substitute(self.story.getAllMetadata()))
 
-            for index, (url,title,html) in enumerate(self.story.getChapters(fortoc=True)):
-                if html:
-                    self._write(out,ENTRY.substitute({'chapter':title,
+            for index, chap in enumerate(self.story.getChapters(fortoc=True)):
+                if chap.html:
+                    self._write(out,ENTRY.substitute({'chapter':chap.title,
                                                       'number':index+1,
                                                       'index':"%04d"%(index+1),
-                                                      'url':url}))
+                                                      'url':chap.url}))
 
             self._write(out,END.substitute(self.story.getAllMetadata()))
 

fanficfare/writers/writer_epub.py

@@ -28,7 +28,7 @@ import re
 from xml.dom.minidom import parse, parseString, getDOMImplementation
 
 from base_writer import *
-from ..htmlcleanup import stripHTML
+from ..htmlcleanup import stripHTML,removeEntities
 
 logger = logging.getLogger(__name__)
 
@@ -133,6 +133,10 @@ ${value}<br />
 <head>
 <title>${chapter}</title>
 <link href="stylesheet.css" type="text/css" charset="UTF-8" rel="stylesheet"/>
+<meta name="chapterurl" content="${url}"></meta>
+<meta name="chapterorigtitle" content="${origchapter}"></meta>
+<meta name="chaptertoctitle" content="${tocchapter}"></meta>
+<meta name="chaptertitle" content="${chapter}"></meta>
 </head>
 <body>
 <h3>${chapter}</h3>
@@ -258,14 +262,15 @@ div { margin: 0pt; padding: 0pt; }
             if self.isValidMetaEntry(entry):
                 val = self.story.getMetadata(entry)
                 if val and ( entry not in oldvalues or val != oldvalues[entry] ):
-                    if self.hasConfig(entry+"_label"):
-                        label=self.getConfig(entry+"_label")
-                    elif entry in self.titleLabels:
-                        logger.debug("Using fallback label for %s_label"%entry)
-                        label=self.titleLabels[entry]
-                    else:
-                        label="%s"%entry.title()
-                        logger.debug("No known label for %s, fallback to '%s'"%(entry,label))
+                    label=self.get_label(entry)
+                    # if self.hasConfig(entry+"_label"):
+                    #     label=self.getConfig(entry+"_label")
+                    # elif entry in self.titleLabels:
+                    #     logger.debug("Using fallback label for %s_label"%entry)
+                    #     label=self.titleLabels[entry]
+                    # else:
+                    #     label="%s"%entry.title()
+                    #     logger.debug("No known label for %s, fallback to '%s'"%(entry,label))
 
                     retval = retval + ENTRY.substitute({'id':entry,
                                                         'label':label,
@@ -501,13 +506,13 @@ div { margin: 0pt; padding: 0pt; }
             items.append(("log_page","OEBPS/log_page.xhtml","application/xhtml+xml","Update Log"))
             itemrefs.append("log_page")
             
-        for index, (url,title,html) in enumerate(self.story.getChapters(fortoc=True)):
-            if html:
+        for index, chap in enumerate(self.story.getChapters(fortoc=True)):
+            if chap.html:
                 i=index+1
                 items.append(("file%04d"%i,
                               "OEBPS/file%04d.xhtml"%i,
                               "application/xhtml+xml",
-                              title))
+                              chap.title))
                 itemrefs.append("file%04d"%i)
 
         manifest = contentdom.createElement("manifest")
@@ -649,19 +654,21 @@ div { margin: 0pt; padding: 0pt; }
         else:
             CHAPTER_END = self.EPUB_CHAPTER_END
         
-        for index, (url,title,html) in enumerate(self.story.getChapters()):
-            if html:
-                logger.debug('Writing chapter text for: %s' % title)
-                vals={'url':url, 'chapter':title, 'index':"%04d"%(index+1), 'number':index+1}
-                fullhtml = CHAPTER_START.substitute(vals) + html + CHAPTER_END.substitute(vals)
+        for index, chap in enumerate(self.story.getChapters()): # (url,title,html)
+            if chap.html:
+                logger.debug('Writing chapter text for: %s' % chap.title)
+                vals={'url':removeEntities(chap.url),
+                      'chapter':chap.title,
+                      'origchapter':chap.origtitle,
+                      'tocchapter':chap.toctitle,
+                      'index':"%04d"%(index+1),
+                      'number':index+1}
+                fullhtml = CHAPTER_START.substitute(vals) + \
+                    chap.html + CHAPTER_END.substitute(vals)
                 # ffnet(& maybe others) gives the whole chapter text
                 # as one line.  This causes problems for nook(at
                 # least) when the chapter size starts getting big
                 # (200k+)
-                #fullhtml = fullhtml.replace('</p>','</p>\n').replace('<br />','<br />\n')
-                # The replaces above added tons of extra newlines
-                # during *each* epub update.  The regexp version adds
-                # only one and removes any extra.
                 fullhtml = re.sub(r'(</p>|<br ?/>)\n*',r'\1\n',fullhtml)
 
                 outputepub.writestr("OEBPS/file%04d.xhtml"%(index+1),fullhtml.encode('utf-8'))

fanficfare/writers/writer_html.py

@@ -128,12 +128,12 @@ ${output_css}
         else:
             CHAPTER_END = self.HTML_CHAPTER_END
         
-        for index, (url,title,html) in enumerate(self.story.getChapters()):
-            if html:
-                logging.debug('Writing chapter text for: %s' % title)
-                vals={'url':url, 'chapter':title, 'index':"%04d"%(index+1), 'number':index+1}
+        for index, chap in enumerate(self.story.getChapters()):
+            if chap.html:
+                logging.debug('Writing chapter text for: %s' % chap.title)
+                vals={'url':chap.url, 'chapter':chap.title, 'index':"%04d"%(index+1), 'number':index+1}
                 self._write(out,CHAPTER_START.substitute(vals))
-                self._write(out,html)
+                self._write(out,chap.html)
                 self._write(out,CHAPTER_END.substitute(vals))
 
         self._write(out,FILE_END.substitute(self.story.getAllMetadata()))

fanficfare/writers/writer_mobi.py

@@ -161,11 +161,11 @@ ${value}<br />
         else:
             CHAPTER_END = self.MOBI_CHAPTER_END
         
-        for index, (url,title,html) in enumerate(self.story.getChapters()):
-            if html:
-                logger.debug('Writing chapter text for: %s' % title)
-                vals={'url':url, 'chapter':title, 'index':"%04d"%(index+1), 'number':index+1}
-                fullhtml = CHAPTER_START.substitute(vals) + html + CHAPTER_END.substitute(vals)
+        for index, chap in enumerate(self.story.getChapters()):
+            if chap.html:
+                logger.debug('Writing chapter text for: %s' % chap.title)
+                vals={'url':chap.url, 'chapter':chap.title, 'index':"%04d"%(index+1), 'number':index+1}
+                fullhtml = CHAPTER_START.substitute(vals) + chap.html + CHAPTER_END.substitute(vals)
                 # ffnet(& maybe others) gives the whole chapter text
                 # as one line.  This causes problems for nook(at
                 # least) when the chapter size starts getting big

fanficfare/writers/writer_txt.py

@@ -154,12 +154,12 @@ End file.
         else:
             CHAPTER_END = self.TEXT_CHAPTER_END
         
-        for index, (url, title,html) in enumerate(self.story.getChapters()):
-            if html:
-                logging.debug('Writing chapter text for: %s' % title)
-                vals={'url':url, 'chapter':title, 'index':"%04d"%(index+1), 'number':index+1}
+        for index, chap in enumerate(self.story.getChapters()):
+            if chap.html:
+                logging.debug('Writing chapter text for: %s' % chap.title)
+                vals={'url':chap.url, 'chapter':chap.title, 'index':"%04d"%(index+1), 'number':index+1}
                 self._write(out,self.lineends(self.wraplines(removeAllEntities(CHAPTER_START.substitute(vals)))))
-                self._write(out,self.lineends(html2text(html,wrap_width=self.wrap_width)))
+                self._write(out,self.lineends(html2text(chap.html,wrap_width=self.wrap_width)))
                 self._write(out,self.lineends(self.wraplines(removeAllEntities(CHAPTER_END.substitute(vals)))))
 
         self._write(out,self.lineends(self.wraplines(FILE_END.substitute(self.story.getAllMetadata()))))

included_dependencies/gif.py

@@ -0,0 +1,283 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+A pure Python GIF metadata extractor.
+Supports adjustable detail to fine-tune performance.
+
+Example code and full epydoc docstrings included.
+
+Uses:
+ - Identifying whether a GIF is static or animated.
+ - Extracting the dimensions, pixel aspect ratio, number of frames, loop count,
+    global palette or palette size, and background color.
+ - Extracting comments and other plaintext.
+ - Testing for various structural errors.
+
+TODO:
+ - Provide basic support for XMP Metadata extraction
+   - http://en.wikipedia.org/wiki/Extensible_Metadata_Platform#Location_in_file_types
+   - http://www.matthewflickinger.com/lab/whatsinagif/bits_and_bytes.asp
+   - Generate test GIF with http://code.google.com/p/python-xmp-toolkit/
+
+Changelog:
+ - 0.2.2: Audited the code and made some corrections.
+ - 0.2.1: 40% speed improvement (went from 15 to 9 seconds for 1000 images)
+ - 0.2.0: Feature-complete
+ - 0.1.0: Initial release
+"""
+
+__appname__ = "gif.py"
+__author__  = "Stephan Sokolow (deitarion/SSokolow)"
+__version__ = "0.2.2"
+__license__ = "PSF License 2.4 or higher (The Python License)"
+
+#{ Check Types (enum, numerical ordering is significant)
+CHECK_IS_GIF_FILE   = 0  #: Just check for a valid GIF header.
+CHECK_IS_ANIMATED   = 1  #: Check whether the file has more than one frame.
+CHECK_COUNT_FRAMES  = 2  #: Count the number of frames in the file.
+CHECK_PARSE_PALETTE = 3  #: Parse the palette and resolve the background color.
+CHECK_READ_COMMENTS = 4  #: Load comments (can sometimes be large) into the L{GifInfo} object.
+CHECK_READ_ALL_TEXT = 5  #: Also load the contents of Plain Text extension blocks.
+
+CHECK_ALL = CHECK_READ_ALL_TEXT #: alias to allow for future modifications
+
+#{  Warning Codes (bitfield)
+WARN_NONE = 0        #: No warnings
+WARN_BAD_IMG = 1     #: Corruption (of the [sub]block size field(s)) or truncation detected in an image block
+WARN_BAD_EXT = 2     #: Corruption (of the [sub]block size field(s)) or truncation detected in an extension block
+WARN_BAD_SIZE = 4    #: An image block specifies dimensions exceeding the global canvas size
+WARN_BAD_BGCOLOR = 8 #: The background color index specified is greater than the palette size
+WARN_EOF  = 16       #: File is missing it's trailer. (Corrupt elsewhere, truncated, or breaking spec by using EOF as the terminator.)
+WARN_TRUNC  = 32     #: File is definitively either truncated or corrupt. (An EOF was encountered part-way through a structure.)
+WARN_LOOP_POS = 64   #: Netscape Application Extension block (animation-control) was present but not first in the file.
+#}
+
+import struct
+
+#{ Structures used by GifInfo
+gifHeaderStruct = struct.Struct('<xxxxxxHHBBB')  #: File header
+gifImageStruct = struct.Struct('<HHHHB')         #: Image block header
+gifExtenStruct = struct.Struct('<BB')            #: Top-level extension block header
+gifNetscapeStruct = struct.Struct('<BBH')        #: NETSCAPE Loop-control sub-block
+gifColorTripleStruct = struct.Struct('<BBB')     #: RGB palette element
+gifPlaintextStruct = struct.Struct('<BHHHHBBBB') #: Plain Text block header
+#}
+
+class BadHeaderException(Exception):
+    """Raised when no valid GIF header is found"""
+
+class GifInfo(object):
+    """A class for loading and storing metadata from GIF files.
+
+    Accepts paths and file-like objects.
+
+    When using L{CHECK_ALL}, this can also be used to walk past a valid
+    GIF file in an un-delimited byte stream in order to identify the point at
+    which the following file starts. (It doesn't C{fh.seek(0)} or C{fh.close()} after
+    use)
+    """
+    warnFlags = WARN_NONE  #: A bit field of C{WARN_*} flags set by L{__init__}
+    checkLevel = CHECK_ALL #: Default C{CHECK_*} level used by L{__init__}
+
+    #{ Pre-defined "unset" values for GIF Metadata
+    path = None        #: The path to the file, if one was passed to L{__init__}
+    version = None     #: C{87a} or C{89a}
+    width = None
+    height = None
+    loopCount = None
+    pixelAspect = None
+    paletteSize = None #: Always calculated if a global palette is present
+    palette = None     #: The palette as a list of integer RGB tuples. Requires L{CHECK_PARSE_PALETTE}.
+    bgColor = None     #: Global background color as an RGB tuple. Requires L{CHECK_PARSE_PALETTE}.
+    comments = None    #: Text in Comment (0xFE) extension blocks as a list of strings. Requires L{CHECK_READ_COMMENTS}
+    otherText = None   #: Text in "Plain Text" (0x01) extension blocks as a list of strings. Requires L{CHECK_READ_ALL_TEXT}
+    frameCount = 0
+    #}
+
+    def __init__(self, fh, checkLevel=checkLevel):
+        """
+        @param fh: A path or file-like object for a GIF file.
+        @param checkLevel: A C{CHECK_*} constant.
+
+        @raises BadHeaderException: The given file lacks a valid GIF header.
+        @raises IOError: The underlying C{open()} system call failed.
+        """
+        self.checkLevel = checkLevel
+        if isinstance(fh, basestring):
+            self.path = fh
+            fh = open(fh, 'rb')
+
+        header = fh.read(gifHeaderStruct.size)
+        if len(header) < gifHeaderStruct.size:
+            raise BadHeaderException("File is too small to be a GIF")
+
+        self.version = header[3:6]
+        if header[0:3] != 'GIF' or self.version not in ['87a', '89a']:
+            raise BadHeaderException("File does not have a recognizable GIF header")
+        elif self.checkLevel <= CHECK_IS_GIF_FILE:
+            return
+
+        self.width, self.height, GCTF_Byte, bgColor, self.pixelAspect = gifHeaderStruct.unpack(header)
+
+        if self.pixelAspect:
+            self.pixelAspect = (self.pixelAspect + 15) / 64.0
+
+        rawPalette = self._getPalette(fh, GCTF_Byte)
+        self.paletteSize = int(len(rawPalette) / 3)
+        if self.checkLevel >= CHECK_PARSE_PALETTE:
+            self.palette = []
+            for pos in range(0, self.paletteSize):
+                self.palette.append(gifColorTripleStruct.unpack_from(rawPalette, pos * 3))
+
+        if self.paletteSize and bgColor > self.paletteSize:
+            self.warnFlags = self.warnFlags | WARN_BAD_BGCOLOR
+        elif self.palette:
+            self.bgColor = self.palette[bgColor]
+
+        # Iterate blocks
+        self.firstBlock = True
+        blocktype = self._read(fh, 1)
+        while not blocktype == chr(0x3B) and not self.warnFlags & WARN_EOF:
+            self._blockHandlers.get(blocktype, lambda x, y:'')(self, fh)
+            if self.checkLevel <= CHECK_IS_ANIMATED and self.frameCount > 1:
+                return
+
+            self.firstBlock = False
+            blocktype = self._read(fh, 1)
+
+        del self.firstBlock
+
+    def _handleImageBlock(self, fh):
+        """"""
+        self.frameCount += 1
+        try:
+            x, y, w, h, LCTF_Byte = gifImageStruct.unpack(self._read(fh, gifImageStruct.size))
+        except:
+            self.warnFlags = self.warnFlags | WARN_EOF | WARN_TRUNC
+            return
+
+        if x + w > self.width or y + h > self.height:
+            self.warnFlags = self.warnFlags | WARN_BAD_SIZE
+
+        self._getPalette(fh, LCTF_Byte) # Skip the local color table if present
+        fh.read(1)                      # Skip the LZW minimum code size.
+
+        # Skip content and test for the block terminator
+        if not self._skipSubBlocks(fh): # For example, if it's a zero-length string like EOF would return.
+            self.warnFlags = self.warnFlags | WARN_BAD_IMG
+
+    def _handleGenericExtensionBlock(self, fh):
+        """@todo: Rewrite this so extension block types have method handlers."""
+        try:
+            extType, blkSize = gifExtenStruct.unpack(self._read(fh, gifExtenStruct.size))
+        except:
+            self.warnFlags = self.warnFlags | WARN_EOF | WARN_TRUNC
+            return
+        startOffset = fh.tell()
+
+        if extType == 0x01 and self.checkLevel >= CHECK_READ_ALL_TEXT: # Plain Text Block
+            self._read(fh, gifPlaintextStruct.size)
+            self.otherText = self.otherText or []
+            blkSize = self._read(fh, 1)
+            while blkSize and blkSize != '\x00':
+                self.otherText.append(self._read(fh, ord(blkSize)))
+                blkSize = self._read(fh, 1)
+        elif extType == 0xFE and self.checkLevel >= CHECK_READ_COMMENTS: # Comment Block
+            self.comments = self.comments or []
+            blkSize = self._read(fh, 1)
+            while blkSize and blkSize != '\x00':
+                self.comments.append(self._read(fh, ord(blkSize)))
+                blkSize = self._read(fh, 1)
+        elif extType == 0xFF: # Application Block
+            if blkSize == 0x0B and self._read(fh, blkSize) == "NETSCAPE2.0":
+                try:
+                    a, b, self.loopCount = gifNetscapeStruct.unpack(self._read(fh, gifNetscapeStruct.size))
+                except:
+                    self.warnFlags = self.warnFlags | WARN_EOF | WARN_TRUNC
+                    return
+
+                if a != 3 and b != 1:
+                    self.warnFlags = self.warnFlags | WARN_BAD_EXT
+
+                if not self.firstBlock:
+                    self.warnFlags = self.warnFlags | WARN_LOOP_POS
+        else:
+            fh.seek( startOffset + blkSize ) # Skip the contents
+
+        # Test for the block terminator
+        if not self._skipSubBlocks(fh):
+            self.warnFlags = self.warnFlags | WARN_BAD_EXT
+
+    def _getPalette(self, handle, bitfield):
+        """Using the size value from C{bitfield},
+        load the palette at C{handle}'s current file pointer position."""
+        if bitfield & int("10000000", 2):
+            nBits = bitfield & int("00000111", 2)
+            tableSize = 3 * 2**( nBits + 1 )
+            return handle.read(tableSize)
+        else:
+            return ''
+
+    def _read(self, handle, size):
+        """Attempt to read the specified number of bytes. Set L{WARN_EOF} if
+        fewer are received."""
+        content = handle.read(size)
+        if len(content) < size:
+            self.warnFlags = self.warnFlags | WARN_EOF
+        return content
+
+    def _skipSubBlocks(self, handle):
+        """Skip sub-blocks beginning at the current file pointer position
+        using fseek."""
+        offset = handle.tell()
+        blkSize = handle.read(1)
+        while blkSize and blkSize != '\x00':
+            offset += ord(blkSize) + 1
+            handle.seek(offset)
+            blkSize = handle.read(1)
+        return blkSize
+
+    _blockHandlers = {
+            chr(0x2C) : _handleImageBlock,
+            chr(0x21) : _handleGenericExtensionBlock,
+        }
+
+def gif_is_animated(path):
+    """A simple convenience function for testing whether a GIF is animated.
+    @rtype: C{bool}
+    """
+    return GifInfo(file(path,'rb'), CHECK_IS_ANIMATED).frameCount > 1
+
+if __name__ == '__main__':
+    from optparse import OptionParser
+    parser = OptionParser(description=__doc__.split('\n\n')[0],
+            version="%%prog v%s" % __version__, usage="%prog <path> ...")
+
+    opts, args = parser.parse_args()
+
+    if args:
+        for fpath in args:
+            try:
+                info = GifInfo(fpath, CHECK_COUNT_FRAMES)
+                warnFlags = (
+                        (info.warnFlags & WARN_BAD_IMG     and 'I' or ' ') +
+                        (info.warnFlags & WARN_BAD_EXT     and 'X' or ' ') +
+                        (info.warnFlags & WARN_BAD_SIZE    and 'C' or ' ') +
+                        (info.warnFlags & WARN_BAD_BGCOLOR and 'B' or ' ') +
+                        (info.warnFlags & WARN_EOF         and 'E' or ' ') +
+                        (info.warnFlags & WARN_TRUNC       and 'T' or ' ') +
+                        (info.warnFlags & WARN_LOOP_POS    and 'L' or ' ')
+                    )
+                print "[%s](%3s Frames): %s" % (warnFlags, info.frameCount, info.path)
+            except BadHeaderException, err:
+                print "%s: %s" % (str(err), fpath)
+        print "\nWarning Flags:"
+        print " I = Image Chunk Corruption/Truncation"
+        print " X = Extension Chunk Corruption/Truncation"
+        print " C = Image Chunk Dimensions Exceed Global Canvas"
+        print " B = Bad Background Color (Index Exceeds Palette Size)"
+        print " E = Unexpected EOF Encountered (Missing Image Terminator)"
+        print " T = EOF Encountered Within A Block Header (Corrupt or Truncated File)"
+        print " L = Loop-control block misplaced within the file"
+        print
+        print "Note: A nearly-threefold speed-up can be had by using CHECK_IS_ANIMATED rather than CHECK_COUNT_FRAMES"

included_dependencies/html5lib/__init__.py

@@ -20,4 +20,4 @@ from .serializer import serialize
 
 __all__ = ["HTMLParser", "parse", "parseFragment", "getTreeBuilder",
            "getTreeWalker", "serialize"]
-__version__ = "0.999"
+__version__ = "0.99999"

included_dependencies/html5lib/constants.py

@@ -1,292 +1,290 @@
 from __future__ import absolute_import, division, unicode_literals
 
 import string
-import gettext
-_ = gettext.gettext
 
 EOF = None
 
 E = {
     "null-character":
-        _("Null character in input stream, replaced with U+FFFD."),
+        "Null character in input stream, replaced with U+FFFD.",
     "invalid-codepoint":
-        _("Invalid codepoint in stream."),
+        "Invalid codepoint in stream.",
     "incorrectly-placed-solidus":
-        _("Solidus (/) incorrectly placed in tag."),
+        "Solidus (/) incorrectly placed in tag.",
     "incorrect-cr-newline-entity":
-        _("Incorrect CR newline entity, replaced with LF."),
+        "Incorrect CR newline entity, replaced with LF.",
     "illegal-windows-1252-entity":
-        _("Entity used with illegal number (windows-1252 reference)."),
+        "Entity used with illegal number (windows-1252 reference).",
     "cant-convert-numeric-entity":
-        _("Numeric entity couldn't be converted to character "
-          "(codepoint U+%(charAsInt)08x)."),
+        "Numeric entity couldn't be converted to character "
+        "(codepoint U+%(charAsInt)08x).",
     "illegal-codepoint-for-numeric-entity":
-        _("Numeric entity represents an illegal codepoint: "
-          "U+%(charAsInt)08x."),
+        "Numeric entity represents an illegal codepoint: "
+        "U+%(charAsInt)08x.",
     "numeric-entity-without-semicolon":
-        _("Numeric entity didn't end with ';'."),
+        "Numeric entity didn't end with ';'.",
     "expected-numeric-entity-but-got-eof":
-        _("Numeric entity expected. Got end of file instead."),
+        "Numeric entity expected. Got end of file instead.",
     "expected-numeric-entity":
-        _("Numeric entity expected but none found."),
+        "Numeric entity expected but none found.",
     "named-entity-without-semicolon":
-        _("Named entity didn't end with ';'."),
+        "Named entity didn't end with ';'.",
     "expected-named-entity":
-        _("Named entity expected. Got none."),
+        "Named entity expected. Got none.",
     "attributes-in-end-tag":
-        _("End tag contains unexpected attributes."),
+        "End tag contains unexpected attributes.",
     'self-closing-flag-on-end-tag':
-        _("End tag contains unexpected self-closing flag."),
+        "End tag contains unexpected self-closing flag.",
     "expected-tag-name-but-got-right-bracket":
-        _("Expected tag name. Got '>' instead."),
+        "Expected tag name. Got '>' instead.",
     "expected-tag-name-but-got-question-mark":
-        _("Expected tag name. Got '?' instead. (HTML doesn't "
-          "support processing instructions.)"),
+        "Expected tag name. Got '?' instead. (HTML doesn't "
+        "support processing instructions.)",
     "expected-tag-name":
-        _("Expected tag name. Got something else instead"),
+        "Expected tag name. Got something else instead",
     "expected-closing-tag-but-got-right-bracket":
-        _("Expected closing tag. Got '>' instead. Ignoring '</>'."),
+        "Expected closing tag. Got '>' instead. Ignoring '</>'.",
     "expected-closing-tag-but-got-eof":
-        _("Expected closing tag. Unexpected end of file."),
+        "Expected closing tag. Unexpected end of file.",
     "expected-closing-tag-but-got-char":
-        _("Expected closing tag. Unexpected character '%(data)s' found."),
+        "Expected closing tag. Unexpected character '%(data)s' found.",
     "eof-in-tag-name":
-        _("Unexpected end of file in the tag name."),
+        "Unexpected end of file in the tag name.",
     "expected-attribute-name-but-got-eof":
-        _("Unexpected end of file. Expected attribute name instead."),
+        "Unexpected end of file. Expected attribute name instead.",
     "eof-in-attribute-name":
-        _("Unexpected end of file in attribute name."),
+        "Unexpected end of file in attribute name.",
     "invalid-character-in-attribute-name":
-        _("Invalid character in attribute name"),
+        "Invalid character in attribute name",
     "duplicate-attribute":
-        _("Dropped duplicate attribute on tag."),
+        "Dropped duplicate attribute on tag.",
     "expected-end-of-tag-name-but-got-eof":
-        _("Unexpected end of file. Expected = or end of tag."),
+        "Unexpected end of file. Expected = or end of tag.",
     "expected-attribute-value-but-got-eof":
-        _("Unexpected end of file. Expected attribute value."),
+        "Unexpected end of file. Expected attribute value.",
     "expected-attribute-value-but-got-right-bracket":
-        _("Expected attribute value. Got '>' instead."),
+        "Expected attribute value. Got '>' instead.",
     'equals-in-unquoted-attribute-value':
-        _("Unexpected = in unquoted attribute"),
+        "Unexpected = in unquoted attribute",
     'unexpected-character-in-unquoted-attribute-value':
-        _("Unexpected character in unquoted attribute"),
+        "Unexpected character in unquoted attribute",
     "invalid-character-after-attribute-name":
-        _("Unexpected character after attribute name."),
+        "Unexpected character after attribute name.",
     "unexpected-character-after-attribute-value":
-        _("Unexpected character after attribute value."),
+        "Unexpected character after attribute value.",
     "eof-in-attribute-value-double-quote":
-        _("Unexpected end of file in attribute value (\")."),
+        "Unexpected end of file in attribute value (\").",
     "eof-in-attribute-value-single-quote":
-        _("Unexpected end of file in attribute value (')."),
+        "Unexpected end of file in attribute value (').",
     "eof-in-attribute-value-no-quotes":
-        _("Unexpected end of file in attribute value."),
+        "Unexpected end of file in attribute value.",
     "unexpected-EOF-after-solidus-in-tag":
-        _("Unexpected end of file in tag. Expected >"),
+        "Unexpected end of file in tag. Expected >",
     "unexpected-character-after-solidus-in-tag":
-        _("Unexpected character after / in tag. Expected >"),
+        "Unexpected character after / in tag. Expected >",
     "expected-dashes-or-doctype":
-        _("Expected '--' or 'DOCTYPE'. Not found."),
+        "Expected '--' or 'DOCTYPE'. Not found.",
     "unexpected-bang-after-double-dash-in-comment":
-        _("Unexpected ! after -- in comment"),
+        "Unexpected ! after -- in comment",
     "unexpected-space-after-double-dash-in-comment":
-        _("Unexpected space after -- in comment"),
+        "Unexpected space after -- in comment",
     "incorrect-comment":
-        _("Incorrect comment."),
+        "Incorrect comment.",
     "eof-in-comment":
-        _("Unexpected end of file in comment."),
+        "Unexpected end of file in comment.",
     "eof-in-comment-end-dash":
-        _("Unexpected end of file in comment (-)"),
+        "Unexpected end of file in comment (-)",
     "unexpected-dash-after-double-dash-in-comment":
-        _("Unexpected '-' after '--' found in comment."),
+        "Unexpected '-' after '--' found in comment.",
     "eof-in-comment-double-dash":
-        _("Unexpected end of file in comment (--)."),
+        "Unexpected end of file in comment (--).",
     "eof-in-comment-end-space-state":
-        _("Unexpected end of file in comment."),
+        "Unexpected end of file in comment.",
     "eof-in-comment-end-bang-state":
-        _("Unexpected end of file in comment."),
+        "Unexpected end of file in comment.",
     "unexpected-char-in-comment":
-        _("Unexpected character in comment found."),
+        "Unexpected character in comment found.",
     "need-space-after-doctype":
-        _("No space after literal string 'DOCTYPE'."),
+        "No space after literal string 'DOCTYPE'.",
     "expected-doctype-name-but-got-right-bracket":
-        _("Unexpected > character. Expected DOCTYPE name."),
+        "Unexpected > character. Expected DOCTYPE name.",
     "expected-doctype-name-but-got-eof":
-        _("Unexpected end of file. Expected DOCTYPE name."),
+        "Unexpected end of file. Expected DOCTYPE name.",
     "eof-in-doctype-name":
-        _("Unexpected end of file in DOCTYPE name."),
+        "Unexpected end of file in DOCTYPE name.",
     "eof-in-doctype":
-        _("Unexpected end of file in DOCTYPE."),
+        "Unexpected end of file in DOCTYPE.",
     "expected-space-or-right-bracket-in-doctype":
-        _("Expected space or '>'. Got '%(data)s'"),
+        "Expected space or '>'. Got '%(data)s'",
     "unexpected-end-of-doctype":
-        _("Unexpected end of DOCTYPE."),
+        "Unexpected end of DOCTYPE.",
     "unexpected-char-in-doctype":
-        _("Unexpected character in DOCTYPE."),
+        "Unexpected character in DOCTYPE.",
     "eof-in-innerhtml":
-        _("XXX innerHTML EOF"),
+        "XXX innerHTML EOF",
     "unexpected-doctype":
-        _("Unexpected DOCTYPE. Ignored."),
+        "Unexpected DOCTYPE. Ignored.",
     "non-html-root":
-        _("html needs to be the first start tag."),
+        "html needs to be the first start tag.",
     "expected-doctype-but-got-eof":
-        _("Unexpected End of file. Expected DOCTYPE."),
+        "Unexpected End of file. Expected DOCTYPE.",
     "unknown-doctype":
-        _("Erroneous DOCTYPE."),
+        "Erroneous DOCTYPE.",
     "expected-doctype-but-got-chars":
-        _("Unexpected non-space characters. Expected DOCTYPE."),
+        "Unexpected non-space characters. Expected DOCTYPE.",
     "expected-doctype-but-got-start-tag":
-        _("Unexpected start tag (%(name)s). Expected DOCTYPE."),
+        "Unexpected start tag (%(name)s). Expected DOCTYPE.",
     "expected-doctype-but-got-end-tag":
-        _("Unexpected end tag (%(name)s). Expected DOCTYPE."),
+        "Unexpected end tag (%(name)s). Expected DOCTYPE.",
     "end-tag-after-implied-root":
-        _("Unexpected end tag (%(name)s) after the (implied) root element."),
+        "Unexpected end tag (%(name)s) after the (implied) root element.",
     "expected-named-closing-tag-but-got-eof":
-        _("Unexpected end of file. Expected end tag (%(name)s)."),
+        "Unexpected end of file. Expected end tag (%(name)s).",
     "two-heads-are-not-better-than-one":
-        _("Unexpected start tag head in existing head. Ignored."),
+        "Unexpected start tag head in existing head. Ignored.",
     "unexpected-end-tag":
-        _("Unexpected end tag (%(name)s). Ignored."),
+        "Unexpected end tag (%(name)s). Ignored.",
     "unexpected-start-tag-out-of-my-head":
-        _("Unexpected start tag (%(name)s) that can be in head. Moved."),
+        "Unexpected start tag (%(name)s) that can be in head. Moved.",
     "unexpected-start-tag":
-        _("Unexpected start tag (%(name)s)."),
+        "Unexpected start tag (%(name)s).",
     "missing-end-tag":
-        _("Missing end tag (%(name)s)."),
+        "Missing end tag (%(name)s).",
     "missing-end-tags":
-        _("Missing end tags (%(name)s)."),
+        "Missing end tags (%(name)s).",
     "unexpected-start-tag-implies-end-tag":
-        _("Unexpected start tag (%(startName)s) "
-          "implies end tag (%(endName)s)."),
+        "Unexpected start tag (%(startName)s) "
+        "implies end tag (%(endName)s).",
     "unexpected-start-tag-treated-as":
-        _("Unexpected start tag (%(originalName)s). Treated as %(newName)s."),
+        "Unexpected start tag (%(originalName)s). Treated as %(newName)s.",
     "deprecated-tag":
-        _("Unexpected start tag %(name)s. Don't use it!"),
+        "Unexpected start tag %(name)s. Don't use it!",
     "unexpected-start-tag-ignored":
-        _("Unexpected start tag %(name)s. Ignored."),
+        "Unexpected start tag %(name)s. Ignored.",
     "expected-one-end-tag-but-got-another":
-        _("Unexpected end tag (%(gotName)s). "
-          "Missing end tag (%(expectedName)s)."),
+        "Unexpected end tag (%(gotName)s). "
+        "Missing end tag (%(expectedName)s).",
     "end-tag-too-early":
-        _("End tag (%(name)s) seen too early. Expected other end tag."),
+        "End tag (%(name)s) seen too early. Expected other end tag.",
     "end-tag-too-early-named":
-        _("Unexpected end tag (%(gotName)s). Expected end tag (%(expectedName)s)."),
+        "Unexpected end tag (%(gotName)s). Expected end tag (%(expectedName)s).",
     "end-tag-too-early-ignored":
-        _("End tag (%(name)s) seen too early. Ignored."),
+        "End tag (%(name)s) seen too early. Ignored.",
     "adoption-agency-1.1":
-        _("End tag (%(name)s) violates step 1, "
-          "paragraph 1 of the adoption agency algorithm."),
+        "End tag (%(name)s) violates step 1, "
+        "paragraph 1 of the adoption agency algorithm.",
     "adoption-agency-1.2":
-        _("End tag (%(name)s) violates step 1, "
-          "paragraph 2 of the adoption agency algorithm."),
+        "End tag (%(name)s) violates step 1, "
+        "paragraph 2 of the adoption agency algorithm.",
     "adoption-agency-1.3":
-        _("End tag (%(name)s) violates step 1, "
-          "paragraph 3 of the adoption agency algorithm."),
+        "End tag (%(name)s) violates step 1, "
+        "paragraph 3 of the adoption agency algorithm.",
     "adoption-agency-4.4":
-        _("End tag (%(name)s) violates step 4, "
-          "paragraph 4 of the adoption agency algorithm."),
+        "End tag (%(name)s) violates step 4, "
+        "paragraph 4 of the adoption agency algorithm.",
     "unexpected-end-tag-treated-as":
-        _("Unexpected end tag (%(originalName)s). Treated as %(newName)s."),
+        "Unexpected end tag (%(originalName)s). Treated as %(newName)s.",
     "no-end-tag":
-        _("This element (%(name)s) has no end tag."),
+        "This element (%(name)s) has no end tag.",
     "unexpected-implied-end-tag-in-table":
-        _("Unexpected implied end tag (%(name)s) in the table phase."),
+        "Unexpected implied end tag (%(name)s) in the table phase.",
     "unexpected-implied-end-tag-in-table-body":
-        _("Unexpected implied end tag (%(name)s) in the table body phase."),
+        "Unexpected implied end tag (%(name)s) in the table body phase.",
     "unexpected-char-implies-table-voodoo":
-        _("Unexpected non-space characters in "
-          "table context caused voodoo mode."),
+        "Unexpected non-space characters in "
+        "table context caused voodoo mode.",
     "unexpected-hidden-input-in-table":
-        _("Unexpected input with type hidden in table context."),
+        "Unexpected input with type hidden in table context.",
     "unexpected-form-in-table":
-        _("Unexpected form in table context."),
+        "Unexpected form in table context.",
     "unexpected-start-tag-implies-table-voodoo":
-        _("Unexpected start tag (%(name)s) in "
-          "table context caused voodoo mode."),
+        "Unexpected start tag (%(name)s) in "
+        "table context caused voodoo mode.",
     "unexpected-end-tag-implies-table-voodoo":
-        _("Unexpected end tag (%(name)s) in "
-          "table context caused voodoo mode."),
+        "Unexpected end tag (%(name)s) in "
+        "table context caused voodoo mode.",
     "unexpected-cell-in-table-body":
-        _("Unexpected table cell start tag (%(name)s) "
-          "in the table body phase."),
+        "Unexpected table cell start tag (%(name)s) "
+        "in the table body phase.",
     "unexpected-cell-end-tag":
-        _("Got table cell end tag (%(name)s) "
-          "while required end tags are missing."),
+        "Got table cell end tag (%(name)s) "
+        "while required end tags are missing.",
     "unexpected-end-tag-in-table-body":
-        _("Unexpected end tag (%(name)s) in the table body phase. Ignored."),
+        "Unexpected end tag (%(name)s) in the table body phase. Ignored.",
     "unexpected-implied-end-tag-in-table-row":
-        _("Unexpected implied end tag (%(name)s) in the table row phase."),
+        "Unexpected implied end tag (%(name)s) in the table row phase.",
     "unexpected-end-tag-in-table-row":
-        _("Unexpected end tag (%(name)s) in the table row phase. Ignored."),
+        "Unexpected end tag (%(name)s) in the table row phase. Ignored.",
     "unexpected-select-in-select":
-        _("Unexpected select start tag in the select phase "
-          "treated as select end tag."),
+        "Unexpected select start tag in the select phase "
+        "treated as select end tag.",
     "unexpected-input-in-select":
-        _("Unexpected input start tag in the select phase."),
+        "Unexpected input start tag in the select phase.",
     "unexpected-start-tag-in-select":
-        _("Unexpected start tag token (%(name)s in the select phase. "
-          "Ignored."),
+        "Unexpected start tag token (%(name)s in the select phase. "
+        "Ignored.",
     "unexpected-end-tag-in-select":
-        _("Unexpected end tag (%(name)s) in the select phase. Ignored."),
+        "Unexpected end tag (%(name)s) in the select phase. Ignored.",
     "unexpected-table-element-start-tag-in-select-in-table":
-        _("Unexpected table element start tag (%(name)s) in the select in table phase."),
+        "Unexpected table element start tag (%(name)s) in the select in table phase.",
     "unexpected-table-element-end-tag-in-select-in-table":
-        _("Unexpected table element end tag (%(name)s) in the select in table phase."),
+        "Unexpected table element end tag (%(name)s) in the select in table phase.",
     "unexpected-char-after-body":
-        _("Unexpected non-space characters in the after body phase."),
+        "Unexpected non-space characters in the after body phase.",
     "unexpected-start-tag-after-body":
-        _("Unexpected start tag token (%(name)s)"
-          " in the after body phase."),
+        "Unexpected start tag token (%(name)s)"
+        " in the after body phase.",
     "unexpected-end-tag-after-body":
-        _("Unexpected end tag token (%(name)s)"
-          " in the after body phase."),
+        "Unexpected end tag token (%(name)s)"
+        " in the after body phase.",
     "unexpected-char-in-frameset":
-        _("Unexpected characters in the frameset phase. Characters ignored."),
+        "Unexpected characters in the frameset phase. Characters ignored.",
     "unexpected-start-tag-in-frameset":
-        _("Unexpected start tag token (%(name)s)"
-          " in the frameset phase. Ignored."),
+        "Unexpected start tag token (%(name)s)"
+        " in the frameset phase. Ignored.",
     "unexpected-frameset-in-frameset-innerhtml":
-        _("Unexpected end tag token (frameset) "
-          "in the frameset phase (innerHTML)."),
+        "Unexpected end tag token (frameset) "
+        "in the frameset phase (innerHTML).",
     "unexpected-end-tag-in-frameset":
-        _("Unexpected end tag token (%(name)s)"
-          " in the frameset phase. Ignored."),
+        "Unexpected end tag token (%(name)s)"
+        " in the frameset phase. Ignored.",
     "unexpected-char-after-frameset":
-        _("Unexpected non-space characters in the "
-          "after frameset phase. Ignored."),
+        "Unexpected non-space characters in the "
+        "after frameset phase. Ignored.",
     "unexpected-start-tag-after-frameset":
-        _("Unexpected start tag (%(name)s)"
-          " in the after frameset phase. Ignored."),
+        "Unexpected start tag (%(name)s)"
+        " in the after frameset phase. Ignored.",
     "unexpected-end-tag-after-frameset":
-        _("Unexpected end tag (%(name)s)"
-          " in the after frameset phase. Ignored."),
+        "Unexpected end tag (%(name)s)"
+        " in the after frameset phase. Ignored.",
     "unexpected-end-tag-after-body-innerhtml":
-        _("Unexpected end tag after body(innerHtml)"),
+        "Unexpected end tag after body(innerHtml)",
     "expected-eof-but-got-char":
-        _("Unexpected non-space characters. Expected end of file."),
+        "Unexpected non-space characters. Expected end of file.",
     "expected-eof-but-got-start-tag":
-        _("Unexpected start tag (%(name)s)"
-          ". Expected end of file."),
+        "Unexpected start tag (%(name)s)"
+        ". Expected end of file.",
     "expected-eof-but-got-end-tag":
-        _("Unexpected end tag (%(name)s)"
-          ". Expected end of file."),
+        "Unexpected end tag (%(name)s)"
+        ". Expected end of file.",
     "eof-in-table":
-        _("Unexpected end of file. Expected table content."),
+        "Unexpected end of file. Expected table content.",
     "eof-in-select":
-        _("Unexpected end of file. Expected select content."),
+        "Unexpected end of file. Expected select content.",
     "eof-in-frameset":
-        _("Unexpected end of file. Expected frameset content."),
+        "Unexpected end of file. Expected frameset content.",
     "eof-in-script-in-script":
-        _("Unexpected end of file. Expected script content."),
+        "Unexpected end of file. Expected script content.",
     "eof-in-foreign-lands":
-        _("Unexpected end of file. Expected foreign content"),
+        "Unexpected end of file. Expected foreign content",
     "non-void-element-with-trailing-solidus":
-        _("Trailing solidus not allowed on element %(name)s"),
+        "Trailing solidus not allowed on element %(name)s",
     "unexpected-html-element-in-foreign-content":
-        _("Element %(name)s not allowed in a non-html context"),
+        "Element %(name)s not allowed in a non-html context",
     "unexpected-end-tag-before-html":
-        _("Unexpected end tag (%(name)s) before html."),
+        "Unexpected end tag (%(name)s) before html.",
     "XXX-undefined-error":
-        _("Undefined error (this sucks and should be fixed)"),
+        "Undefined error (this sucks and should be fixed)",
 }
 
 namespaces = {
@@ -298,7 +296,7 @@ namespaces = {
     "xmlns": "http://www.w3.org/2000/xmlns/"
 }
 
-scopingElements = frozenset((
+scopingElements = frozenset([
     (namespaces["html"], "applet"),
     (namespaces["html"], "caption"),
     (namespaces["html"], "html"),
@@ -316,9 +314,9 @@ scopingElements = frozenset((
     (namespaces["svg"], "foreignObject"),
     (namespaces["svg"], "desc"),
     (namespaces["svg"], "title"),
-))
+])
 
-formattingElements = frozenset((
+formattingElements = frozenset([
     (namespaces["html"], "a"),
     (namespaces["html"], "b"),
     (namespaces["html"], "big"),
@@ -333,9 +331,9 @@ formattingElements = frozenset((
     (namespaces["html"], "strong"),
     (namespaces["html"], "tt"),
     (namespaces["html"], "u")
-))
+])
 
-specialElements = frozenset((
+specialElements = frozenset([
     (namespaces["html"], "address"),
     (namespaces["html"], "applet"),
     (namespaces["html"], "area"),
@@ -416,22 +414,22 @@ specialElements = frozenset((
     (namespaces["html"], "wbr"),
     (namespaces["html"], "xmp"),
     (namespaces["svg"], "foreignObject")
-))
+])
 
-htmlIntegrationPointElements = frozenset((
+htmlIntegrationPointElements = frozenset([
     (namespaces["mathml"], "annotaion-xml"),
     (namespaces["svg"], "foreignObject"),
     (namespaces["svg"], "desc"),
     (namespaces["svg"], "title")
-))
+])
 
-mathmlTextIntegrationPointElements = frozenset((
+mathmlTextIntegrationPointElements = frozenset([
     (namespaces["mathml"], "mi"),
     (namespaces["mathml"], "mo"),
     (namespaces["mathml"], "mn"),
     (namespaces["mathml"], "ms"),
     (namespaces["mathml"], "mtext")
-))
+])
 
 adjustForeignAttributes = {
     "xlink:actuate": ("xlink", "actuate", namespaces["xlink"]),
@@ -451,21 +449,21 @@ adjustForeignAttributes = {
 unadjustForeignAttributes = dict([((ns, local), qname) for qname, (prefix, local, ns) in
                                   adjustForeignAttributes.items()])
 
-spaceCharacters = frozenset((
+spaceCharacters = frozenset([
     "\t",
     "\n",
     "\u000C",
     " ",
     "\r"
-))
+])
 
-tableInsertModeElements = frozenset((
+tableInsertModeElements = frozenset([
     "table",
     "tbody",
     "tfoot",
     "thead",
     "tr"
-))
+])
 
 asciiLowercase = frozenset(string.ascii_lowercase)
 asciiUppercase = frozenset(string.ascii_uppercase)
@@ -486,7 +484,7 @@ headingElements = (
     "h6"
 )
 
-voidElements = frozenset((
+voidElements = frozenset([
     "base",
     "command",
     "event-source",
@@ -502,11 +500,11 @@ voidElements = frozenset((
     "input",
     "source",
     "track"
-))
+])
 
-cdataElements = frozenset(('title', 'textarea'))
+cdataElements = frozenset(['title', 'textarea'])
 
-rcdataElements = frozenset((
+rcdataElements = frozenset([
     'style',
     'script',
     'xmp',
@@ -514,27 +512,27 @@ rcdataElements = frozenset((
     'noembed',
     'noframes',
     'noscript'
-))
+])
 
 booleanAttributes = {
-    "": frozenset(("irrelevant",)),
-    "style": frozenset(("scoped",)),
-    "img": frozenset(("ismap",)),
-    "audio": frozenset(("autoplay", "controls")),
-    "video": frozenset(("autoplay", "controls")),
-    "script": frozenset(("defer", "async")),
-    "details": frozenset(("open",)),
-    "datagrid": frozenset(("multiple", "disabled")),
-    "command": frozenset(("hidden", "disabled", "checked", "default")),
-    "hr": frozenset(("noshade")),
-    "menu": frozenset(("autosubmit",)),
-    "fieldset": frozenset(("disabled", "readonly")),
-    "option": frozenset(("disabled", "readonly", "selected")),
-    "optgroup": frozenset(("disabled", "readonly")),
-    "button": frozenset(("disabled", "autofocus")),
-    "input": frozenset(("disabled", "readonly", "required", "autofocus", "checked", "ismap")),
-    "select": frozenset(("disabled", "readonly", "autofocus", "multiple")),
-    "output": frozenset(("disabled", "readonly")),
+    "": frozenset(["irrelevant"]),
+    "style": frozenset(["scoped"]),
+    "img": frozenset(["ismap"]),
+    "audio": frozenset(["autoplay", "controls"]),
+    "video": frozenset(["autoplay", "controls"]),
+    "script": frozenset(["defer", "async"]),
+    "details": frozenset(["open"]),
+    "datagrid": frozenset(["multiple", "disabled"]),
+    "command": frozenset(["hidden", "disabled", "checked", "default"]),
+    "hr": frozenset(["noshade"]),
+    "menu": frozenset(["autosubmit"]),
+    "fieldset": frozenset(["disabled", "readonly"]),
+    "option": frozenset(["disabled", "readonly", "selected"]),
+    "optgroup": frozenset(["disabled", "readonly"]),
+    "button": frozenset(["disabled", "autofocus"]),
+    "input": frozenset(["disabled", "readonly", "required", "autofocus", "checked", "ismap"]),
+    "select": frozenset(["disabled", "readonly", "autofocus", "multiple"]),
+    "output": frozenset(["disabled", "readonly"]),
 }
 
 # entitiesWindows1252 has to be _ordered_ and needs to have an index. It
@@ -574,7 +572,7 @@ entitiesWindows1252 = (
     376     # 0x9F  0x0178  LATIN CAPITAL LETTER Y WITH DIAERESIS
 )
 
-xmlEntities = frozenset(('lt;', 'gt;', 'amp;', 'apos;', 'quot;'))
+xmlEntities = frozenset(['lt;', 'gt;', 'amp;', 'apos;', 'quot;'])
 
 entities = {
     "AElig": "\xc6",
@@ -3088,8 +3086,8 @@ tokenTypes = {
     "ParseError": 7
 }
 
-tagTokenTypes = frozenset((tokenTypes["StartTag"], tokenTypes["EndTag"],
-                           tokenTypes["EmptyTag"]))
+tagTokenTypes = frozenset([tokenTypes["StartTag"], tokenTypes["EndTag"],
+                           tokenTypes["EmptyTag"]])
 
 
 prefixes = dict([(v, k) for k, v in namespaces.items()])

included_dependencies/html5lib/filters/lint.py

@@ -1,8 +1,5 @@
 from __future__ import absolute_import, division, unicode_literals
 
-from gettext import gettext
-_ = gettext
-
 from . import _base
 from ..constants import cdataElements, rcdataElements, voidElements
 
@@ -23,24 +20,24 @@ class Filter(_base.Filter):
             if type in ("StartTag", "EmptyTag"):
                 name = token["name"]
                 if contentModelFlag != "PCDATA":
-                    raise LintError(_("StartTag not in PCDATA content model flag: %(tag)s") % {"tag": name})
+                    raise LintError("StartTag not in PCDATA content model flag: %(tag)s" % {"tag": name})
                 if not isinstance(name, str):
-                    raise LintError(_("Tag name is not a string: %(tag)r") % {"tag": name})
+                    raise LintError("Tag name is not a string: %(tag)r" % {"tag": name})
                 if not name:
-                    raise LintError(_("Empty tag name"))
+                    raise LintError("Empty tag name")
                 if type == "StartTag" and name in voidElements:
-                    raise LintError(_("Void element reported as StartTag token: %(tag)s") % {"tag": name})
+                    raise LintError("Void element reported as StartTag token: %(tag)s" % {"tag": name})
                 elif type == "EmptyTag" and name not in voidElements:
-                    raise LintError(_("Non-void element reported as EmptyTag token: %(tag)s") % {"tag": token["name"]})
+                    raise LintError("Non-void element reported as EmptyTag token: %(tag)s" % {"tag": token["name"]})
                 if type == "StartTag":
                     open_elements.append(name)
                 for name, value in token["data"]:
                     if not isinstance(name, str):
-                        raise LintError(_("Attribute name is not a string: %(name)r") % {"name": name})
+                        raise LintError("Attribute name is not a string: %(name)r" % {"name": name})
                     if not name:
-                        raise LintError(_("Empty attribute name"))
+                        raise LintError("Empty attribute name")
                     if not isinstance(value, str):
-                        raise LintError(_("Attribute value is not a string: %(value)r") % {"value": value})
+                        raise LintError("Attribute value is not a string: %(value)r" % {"value": value})
                 if name in cdataElements:
                     contentModelFlag = "CDATA"
                 elif name in rcdataElements:
@@ -51,43 +48,43 @@ class Filter(_base.Filter):
             elif type == "EndTag":
                 name = token["name"]
                 if not isinstance(name, str):
-                    raise LintError(_("Tag name is not a string: %(tag)r") % {"tag": name})
+                    raise LintError("Tag name is not a string: %(tag)r" % {"tag": name})
                 if not name:
-                    raise LintError(_("Empty tag name"))
+                    raise LintError("Empty tag name")
                 if name in voidElements:
-                    raise LintError(_("Void element reported as EndTag token: %(tag)s") % {"tag": name})
+                    raise LintError("Void element reported as EndTag token: %(tag)s" % {"tag": name})
                 start_name = open_elements.pop()
                 if start_name != name:
-                    raise LintError(_("EndTag (%(end)s) does not match StartTag (%(start)s)") % {"end": name, "start": start_name})
+                    raise LintError("EndTag (%(end)s) does not match StartTag (%(start)s)" % {"end": name, "start": start_name})
                 contentModelFlag = "PCDATA"
 
             elif type == "Comment":
                 if contentModelFlag != "PCDATA":
-                    raise LintError(_("Comment not in PCDATA content model flag"))
+                    raise LintError("Comment not in PCDATA content model flag")
 
             elif type in ("Characters", "SpaceCharacters"):
                 data = token["data"]
                 if not isinstance(data, str):
-                    raise LintError(_("Attribute name is not a string: %(name)r") % {"name": data})
+                    raise LintError("Attribute name is not a string: %(name)r" % {"name": data})
                 if not data:
-                    raise LintError(_("%(type)s token with empty data") % {"type": type})
+                    raise LintError("%(type)s token with empty data" % {"type": type})
                 if type == "SpaceCharacters":
                     data = data.strip(spaceCharacters)
                     if data:
-                        raise LintError(_("Non-space character(s) found in SpaceCharacters token: %(token)r") % {"token": data})
+                        raise LintError("Non-space character(s) found in SpaceCharacters token: %(token)r" % {"token": data})
 
             elif type == "Doctype":
                 name = token["name"]
                 if contentModelFlag != "PCDATA":
-                    raise LintError(_("Doctype not in PCDATA content model flag: %(name)s") % {"name": name})
+                    raise LintError("Doctype not in PCDATA content model flag: %(name)s" % {"name": name})
                 if not isinstance(name, str):
-                    raise LintError(_("Tag name is not a string: %(tag)r") % {"tag": name})
+                    raise LintError("Tag name is not a string: %(tag)r" % {"tag": name})
                 # XXX: what to do with token["data"] ?
 
             elif type in ("ParseError", "SerializeError"):
                 pass
 
             else:
-                raise LintError(_("Unknown token type: %(type)s") % {"type": type})
+                raise LintError("Unknown token type: %(type)s" % {"type": type})
 
             yield token

included_dependencies/html5lib/html5parser.py

@@ -18,6 +18,7 @@ from .constants import cdataElements, rcdataElements
 from .constants import tokenTypes, ReparseException, namespaces
 from .constants import htmlIntegrationPointElements, mathmlTextIntegrationPointElements
 from .constants import adjustForeignAttributes as adjustForeignAttributesMap
+from .constants import E
 
 
 def parse(doc, treebuilder="etree", encoding=None,
@@ -129,6 +130,17 @@ class HTMLParser(object):
 
         self.framesetOK = True
 
+    @property
+    def documentEncoding(self):
+        """The name of the character encoding
+        that was used to decode the input stream,
+        or :obj:`None` if that is not determined yet.
+
+        """
+        if not hasattr(self, 'tokenizer'):
+            return None
+        return self.tokenizer.stream.charEncoding[0]
+
     def isHTMLIntegrationPoint(self, element):
         if (element.name == "annotation-xml" and
                 element.namespace == namespaces["mathml"]):
@@ -245,7 +257,7 @@ class HTMLParser(object):
         # XXX The idea is to make errorcode mandatory.
         self.errors.append((self.tokenizer.stream.position(), errorcode, datavars))
         if self.strict:
-            raise ParseError
+            raise ParseError(E[errorcode] % datavars)
 
     def normalizeToken(self, token):
         """ HTML5 specific normalizations to the token stream """
@@ -868,7 +880,7 @@ def getPhases(debug):
             self.startTagHandler = utils.MethodDispatcher([
                 ("html", self.startTagHtml),
                 (("base", "basefont", "bgsound", "command", "link", "meta",
-                  "noframes", "script", "style", "title"),
+                  "script", "style", "title"),
                  self.startTagProcessInHead),
                 ("body", self.startTagBody),
                 ("frameset", self.startTagFrameset),
@@ -1205,8 +1217,7 @@ def getPhases(debug):
             attributes["name"] = "isindex"
             self.processStartTag(impliedTagToken("input", "StartTag",
                                                  attributes=attributes,
-                                                 selfClosing=
-                                                 token["selfClosing"]))
+                                                 selfClosing=token["selfClosing"]))
             self.processEndTag(impliedTagToken("label"))
             self.processStartTag(impliedTagToken("hr", "StartTag"))
             self.processEndTag(impliedTagToken("form"))

included_dependencies/html5lib/inputstream.py

@@ -28,7 +28,18 @@ asciiLettersBytes = frozenset([item.encode("ascii") for item in asciiLetters])
 asciiUppercaseBytes = frozenset([item.encode("ascii") for item in asciiUppercase])
 spacesAngleBrackets = spaceCharactersBytes | frozenset([b">", b"<"])
 
-invalid_unicode_re = re.compile("[\u0001-\u0008\u000B\u000E-\u001F\u007F-\u009F\uD800-\uDFFF\uFDD0-\uFDEF\uFFFE\uFFFF\U0001FFFE\U0001FFFF\U0002FFFE\U0002FFFF\U0003FFFE\U0003FFFF\U0004FFFE\U0004FFFF\U0005FFFE\U0005FFFF\U0006FFFE\U0006FFFF\U0007FFFE\U0007FFFF\U0008FFFE\U0008FFFF\U0009FFFE\U0009FFFF\U000AFFFE\U000AFFFF\U000BFFFE\U000BFFFF\U000CFFFE\U000CFFFF\U000DFFFE\U000DFFFF\U000EFFFE\U000EFFFF\U000FFFFE\U000FFFFF\U0010FFFE\U0010FFFF]")
+
+invalid_unicode_no_surrogate = "[\u0001-\u0008\u000B\u000E-\u001F\u007F-\u009F\uFDD0-\uFDEF\uFFFE\uFFFF\U0001FFFE\U0001FFFF\U0002FFFE\U0002FFFF\U0003FFFE\U0003FFFF\U0004FFFE\U0004FFFF\U0005FFFE\U0005FFFF\U0006FFFE\U0006FFFF\U0007FFFE\U0007FFFF\U0008FFFE\U0008FFFF\U0009FFFE\U0009FFFF\U000AFFFE\U000AFFFF\U000BFFFE\U000BFFFF\U000CFFFE\U000CFFFF\U000DFFFE\U000DFFFF\U000EFFFE\U000EFFFF\U000FFFFE\U000FFFFF\U0010FFFE\U0010FFFF]"
+
+if utils.supports_lone_surrogates:
+    # Use one extra step of indirection and create surrogates with
+    # unichr. Not using this indirection would introduce an illegal
+    # unicode literal on platforms not supporting such lone
+    # surrogates.
+    invalid_unicode_re = re.compile(invalid_unicode_no_surrogate +
+                                    eval('"\\uD800-\\uDFFF"'))
+else:
+    invalid_unicode_re = re.compile(invalid_unicode_no_surrogate)
 
 non_bmp_invalid_codepoints = set([0x1FFFE, 0x1FFFF, 0x2FFFE, 0x2FFFF, 0x3FFFE,
                                   0x3FFFF, 0x4FFFE, 0x4FFFF, 0x5FFFE, 0x5FFFF,
@@ -164,13 +175,18 @@ class HTMLUnicodeInputStream(object):
 
         """
 
-        # Craziness
-        if len("\U0010FFFF") == 1:
+        if not utils.supports_lone_surrogates:
+            # Such platforms will have already checked for such
+            # surrogate errors, so no need to do this checking.
+            self.reportCharacterErrors = None
+            self.replaceCharactersRegexp = None
+        elif len("\U0010FFFF") == 1:
             self.reportCharacterErrors = self.characterErrorsUCS4
-            self.replaceCharactersRegexp = re.compile("[\uD800-\uDFFF]")
+            self.replaceCharactersRegexp = re.compile(eval('"[\\uD800-\\uDFFF]"'))
         else:
             self.reportCharacterErrors = self.characterErrorsUCS2
-            self.replaceCharactersRegexp = re.compile("([\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?<![\uD800-\uDBFF])[\uDC00-\uDFFF])")
+            self.replaceCharactersRegexp = re.compile(
+                eval('"([\\uD800-\\uDBFF](?![\\uDC00-\\uDFFF])|(?<![\\uD800-\\uDBFF])[\\uDC00-\\uDFFF])"'))
 
         # List of where new lines occur
         self.newLines = [0]
@@ -265,11 +281,12 @@ class HTMLUnicodeInputStream(object):
                 self._bufferedCharacter = data[-1]
                 data = data[:-1]
 
-        self.reportCharacterErrors(data)
+        if self.reportCharacterErrors:
+            self.reportCharacterErrors(data)
 
-        # Replace invalid characters
-        # Note U+0000 is dealt with in the tokenizer
-        data = self.replaceCharactersRegexp.sub("\ufffd", data)
+            # Replace invalid characters
+            # Note U+0000 is dealt with in the tokenizer
+            data = self.replaceCharactersRegexp.sub("\ufffd", data)
 
         data = data.replace("\r\n", "\n")
         data = data.replace("\r", "\n")

included_dependencies/html5lib/sanitizer.py

@@ -2,11 +2,26 @@ from __future__ import absolute_import, division, unicode_literals
 
 import re
 from xml.sax.saxutils import escape, unescape
+from six.moves import urllib_parse as urlparse
 
 from .tokenizer import HTMLTokenizer
 from .constants import tokenTypes
 
 
+content_type_rgx = re.compile(r'''
+                               ^
+                               # Match a content type <application>/<type>
+                               (?P<content_type>[-a-zA-Z0-9.]+/[-a-zA-Z0-9.]+)
+                               # Match any character set and encoding
+                               (?:(?:;charset=(?:[-a-zA-Z0-9]+)(?:;(?:base64))?)
+                                 |(?:;(?:base64))?(?:;charset=(?:[-a-zA-Z0-9]+))?)
+                               # Assume the rest is data
+                               ,.*
+                               $
+                               ''',
+                              re.VERBOSE)
+
+
 class HTMLSanitizerMixin(object):
     """ sanitization of XHTML+MathML+SVG and of inline style attributes."""
 
@@ -100,8 +115,8 @@ class HTMLSanitizerMixin(object):
                       'xml:base', 'xml:lang', 'xml:space', 'xmlns', 'xmlns:xlink', 'y',
                       'y1', 'y2', 'zoomAndPan']
 
-    attr_val_is_uri = ['href', 'src', 'cite', 'action', 'longdesc', 'poster',
-                       'xlink:href', 'xml:base']
+    attr_val_is_uri = ['href', 'src', 'cite', 'action', 'longdesc', 'poster', 'background', 'datasrc',
+                       'dynsrc', 'lowsrc', 'ping', 'poster', 'xlink:href', 'xml:base']
 
     svg_attr_val_allows_ref = ['clip-path', 'color-profile', 'cursor', 'fill',
                                'filter', 'marker', 'marker-start', 'marker-mid', 'marker-end',
@@ -138,7 +153,9 @@ class HTMLSanitizerMixin(object):
     acceptable_protocols = ['ed2k', 'ftp', 'http', 'https', 'irc',
                             'mailto', 'news', 'gopher', 'nntp', 'telnet', 'webcal',
                             'xmpp', 'callto', 'feed', 'urn', 'aim', 'rsync', 'tag',
-                            'ssh', 'sftp', 'rtsp', 'afs']
+                            'ssh', 'sftp', 'rtsp', 'afs', 'data']
+
+    acceptable_content_types = ['image/png', 'image/jpeg', 'image/gif', 'image/webp', 'image/bmp', 'text/plain']
 
     # subclasses may define their own versions of these constants
     allowed_elements = acceptable_elements + mathml_elements + svg_elements
@@ -147,6 +164,7 @@ class HTMLSanitizerMixin(object):
     allowed_css_keywords = acceptable_css_keywords
     allowed_svg_properties = acceptable_svg_properties
     allowed_protocols = acceptable_protocols
+    allowed_content_types = acceptable_content_types
 
     # Sanitize the +html+, escaping all elements not in ALLOWED_ELEMENTS, and
     # stripping out all # attributes not in ALLOWED_ATTRIBUTES. Style
@@ -189,10 +207,17 @@ class HTMLSanitizerMixin(object):
                                        unescape(attrs[attr])).lower()
                 # remove replacement characters from unescaped characters
                 val_unescaped = val_unescaped.replace("\ufffd", "")
-                if (re.match("^[a-z0-9][-+.a-z0-9]*:", val_unescaped) and
-                    (val_unescaped.split(':')[0] not in
-                     self.allowed_protocols)):
-                    del attrs[attr]
+                uri = urlparse.urlparse(val_unescaped)
+                if uri:
+                    if uri.scheme not in self.allowed_protocols:
+                        del attrs[attr]
+                    if uri.scheme == 'data':
+                        m = content_type_rgx.match(uri.path)
+                        if not m:
+                            del attrs[attr]
+                        elif m.group('content_type') not in self.allowed_content_types:
+                            del attrs[attr]
+
             for attr in self.svg_attr_val_allows_ref:
                 if attr in attrs:
                     attrs[attr] = re.sub(r'url\s*\(\s*[^#\s][^)]+?\)',
@@ -245,7 +270,7 @@ class HTMLSanitizerMixin(object):
             elif prop.split('-')[0].lower() in ['background', 'border', 'margin',
                                                 'padding']:
                 for keyword in value.split():
-                    if not keyword in self.acceptable_css_keywords and \
+                    if keyword not in self.acceptable_css_keywords and \
                             not re.match("^(#[0-9a-f]+|rgb\(\d+%?,\d*%?,?\d*%?\)?|\d{0,2}\.?\d{0,2}(cm|em|ex|in|mm|pc|pt|px|%|,|\))?)$", keyword):
                         break
                 else:

included_dependencies/html5lib/serializer/htmlserializer.py

@@ -1,9 +1,6 @@
 from __future__ import absolute_import, division, unicode_literals
 from six import text_type
 
-import gettext
-_ = gettext.gettext
-
 try:
     from functools import reduce
 except ImportError:
@@ -35,7 +32,7 @@ else:
                 v = utils.surrogatePairToCodepoint(v)
             else:
                 v = ord(v)
-            if not v in encode_entity_map or k.islower():
+            if v not in encode_entity_map or k.islower():
                 # prefer < over < and similarly for &, >, etc.
                 encode_entity_map[v] = k
 
@@ -208,7 +205,7 @@ class HTMLSerializer(object):
                 if token["systemId"]:
                     if token["systemId"].find('"') >= 0:
                         if token["systemId"].find("'") >= 0:
-                            self.serializeError(_("System identifer contains both single and double quote characters"))
+                            self.serializeError("System identifer contains both single and double quote characters")
                         quote_char = "'"
                     else:
                         quote_char = '"'
@@ -220,7 +217,7 @@ class HTMLSerializer(object):
             elif type in ("Characters", "SpaceCharacters"):
                 if type == "SpaceCharacters" or in_cdata:
                     if in_cdata and token["data"].find("</") >= 0:
-                        self.serializeError(_("Unexpected </ in CDATA"))
+                        self.serializeError("Unexpected </ in CDATA")
                     yield self.encode(token["data"])
                 else:
                     yield self.encode(escape(token["data"]))
@@ -231,7 +228,7 @@ class HTMLSerializer(object):
                 if name in rcdataElements and not self.escape_rcdata:
                     in_cdata = True
                 elif in_cdata:
-                    self.serializeError(_("Unexpected child element of a CDATA element"))
+                    self.serializeError("Unexpected child element of a CDATA element")
                 for (attr_namespace, attr_name), attr_value in token["data"].items():
                     # TODO: Add namespace support here
                     k = attr_name
@@ -279,20 +276,20 @@ class HTMLSerializer(object):
                 if name in rcdataElements:
                     in_cdata = False
                 elif in_cdata:
-                    self.serializeError(_("Unexpected child element of a CDATA element"))
+                    self.serializeError("Unexpected child element of a CDATA element")
                 yield self.encodeStrict("</%s>" % name)
 
             elif type == "Comment":
                 data = token["data"]
                 if data.find("--") >= 0:
-                    self.serializeError(_("Comment contains --"))
+                    self.serializeError("Comment contains --")
                 yield self.encodeStrict("<!--%s-->" % token["data"])
 
             elif type == "Entity":
                 name = token["name"]
                 key = name + ";"
-                if not key in entities:
-                    self.serializeError(_("Entity %s not recognized" % name))
+                if key not in entities:
+                    self.serializeError("Entity %s not recognized" % name)
                 if self.resolve_entities and key not in xmlEntities:
                     data = entities[key]
                 else:

included_dependencies/html5lib/treebuilders/dom.py

@@ -158,7 +158,7 @@ def getDomBuilder(DomImplementation):
             else:
                 # HACK: allow text nodes as children of the document node
                 if hasattr(self.dom, '_child_node_types'):
-                    if not Node.TEXT_NODE in self.dom._child_node_types:
+                    if Node.TEXT_NODE not in self.dom._child_node_types:
                         self.dom._child_node_types = list(self.dom._child_node_types)
                         self.dom._child_node_types.append(Node.TEXT_NODE)
                 self.dom.appendChild(self.dom.createTextNode(data))

included_dependencies/html5lib/treewalkers/__init__.py

@@ -10,8 +10,12 @@ returning an iterator generating tokens.
 
 from __future__ import absolute_import, division, unicode_literals
 
+__all__ = ["getTreeWalker", "pprint", "dom", "etree", "genshistream", "lxmletree",
+           "pulldom"]
+
 import sys
 
+from .. import constants
 from ..utils import default_etree
 
 treeWalkerCache = {}
@@ -55,3 +59,89 @@ def getTreeWalker(treeType, implementation=None, **kwargs):
             # XXX: NEVER cache here, caching is done in the etree submodule
             return etree.getETreeModule(implementation, **kwargs).TreeWalker
     return treeWalkerCache.get(treeType)
+
+
+def concatenateCharacterTokens(tokens):
+    pendingCharacters = []
+    for token in tokens:
+        type = token["type"]
+        if type in ("Characters", "SpaceCharacters"):
+            pendingCharacters.append(token["data"])
+        else:
+            if pendingCharacters:
+                yield {"type": "Characters", "data": "".join(pendingCharacters)}
+                pendingCharacters = []
+            yield token
+    if pendingCharacters:
+        yield {"type": "Characters", "data": "".join(pendingCharacters)}
+
+
+def pprint(walker):
+    """Pretty printer for tree walkers"""
+    output = []
+    indent = 0
+    for token in concatenateCharacterTokens(walker):
+        type = token["type"]
+        if type in ("StartTag", "EmptyTag"):
+            # tag name
+            if token["namespace"] and token["namespace"] != constants.namespaces["html"]:
+                if token["namespace"] in constants.prefixes:
+                    ns = constants.prefixes[token["namespace"]]
+                else:
+                    ns = token["namespace"]
+                name = "%s %s" % (ns, token["name"])
+            else:
+                name = token["name"]
+            output.append("%s<%s>" % (" " * indent, name))
+            indent += 2
+            # attributes (sorted for consistent ordering)
+            attrs = token["data"]
+            for (namespace, localname), value in sorted(attrs.items()):
+                if namespace:
+                    if namespace in constants.prefixes:
+                        ns = constants.prefixes[namespace]
+                    else:
+                        ns = namespace
+                    name = "%s %s" % (ns, localname)
+                else:
+                    name = localname
+                output.append("%s%s=\"%s\"" % (" " * indent, name, value))
+            # self-closing
+            if type == "EmptyTag":
+                indent -= 2
+
+        elif type == "EndTag":
+            indent -= 2
+
+        elif type == "Comment":
+            output.append("%s<!-- %s -->" % (" " * indent, token["data"]))
+
+        elif type == "Doctype":
+            if token["name"]:
+                if token["publicId"]:
+                    output.append("""%s<!DOCTYPE %s "%s" "%s">""" %
+                                  (" " * indent,
+                                   token["name"],
+                                   token["publicId"],
+                                   token["systemId"] if token["systemId"] else ""))
+                elif token["systemId"]:
+                    output.append("""%s<!DOCTYPE %s "" "%s">""" %
+                                  (" " * indent,
+                                   token["name"],
+                                   token["systemId"]))
+                else:
+                    output.append("%s<!DOCTYPE %s>" % (" " * indent,
+                                                       token["name"]))
+            else:
+                output.append("%s<!DOCTYPE >" % (" " * indent,))
+
+        elif type == "Characters":
+            output.append("%s\"%s\"" % (" " * indent, token["data"]))
+
+        elif type == "SpaceCharacters":
+            assert False, "concatenateCharacterTokens should have got rid of all Space tokens"
+
+        else:
+            raise ValueError("Unknown token type, %s" % type)
+
+    return "\n".join(output)

included_dependencies/html5lib/treewalkers/_base.py

@@ -1,8 +1,8 @@
 from __future__ import absolute_import, division, unicode_literals
 from six import text_type, string_types
 
-import gettext
-_ = gettext.gettext
+__all__ = ["DOCUMENT", "DOCTYPE", "TEXT", "ELEMENT", "COMMENT", "ENTITY", "UNKNOWN",
+           "TreeWalker", "NonRecursiveTreeWalker"]
 
 from xml.dom import Node
 
@@ -58,7 +58,7 @@ class TreeWalker(object):
                "namespace": to_text(namespace),
                "data": attrs}
         if hasChildren:
-            yield self.error(_("Void element has children"))
+            yield self.error("Void element has children")
 
     def startTag(self, namespace, name, attrs):
         assert namespace is None or isinstance(namespace, string_types), type(namespace)
@@ -122,7 +122,7 @@ class TreeWalker(object):
         return {"type": "Entity", "name": text_type(name)}
 
     def unknown(self, nodeType):
-        return self.error(_("Unknown node type: ") + nodeType)
+        return self.error("Unknown node type: " + nodeType)
 
 
 class NonRecursiveTreeWalker(TreeWalker):

included_dependencies/html5lib/treewalkers/dom.py

@@ -2,9 +2,6 @@ from __future__ import absolute_import, division, unicode_literals
 
 from xml.dom import Node
 
-import gettext
-_ = gettext.gettext
-
 from . import _base
 
 

included_dependencies/html5lib/treewalkers/etree.py

@@ -7,8 +7,6 @@ except ImportError:
         from ordereddict import OrderedDict
     except ImportError:
         OrderedDict = dict
-import gettext
-_ = gettext.gettext
 
 import re
 

included_dependencies/html5lib/treewalkers/lxmletree.py

@@ -4,9 +4,6 @@ from six import text_type
 from lxml import etree
 from ..treebuilders.etree import tag_regexp
 
-from gettext import gettext
-_ = gettext
-
 from . import _base
 
 from .. import ihatexml
@@ -130,7 +127,7 @@ class TreeWalker(_base.NonRecursiveTreeWalker):
     def getNodeDetails(self, node):
         if isinstance(node, tuple):  # Text node
             node, key = node
-            assert key in ("text", "tail"), _("Text nodes are text or tail, found %s") % key
+            assert key in ("text", "tail"), "Text nodes are text or tail, found %s" % key
             return _base.TEXT, ensure_str(getattr(node, key))
 
         elif isinstance(node, Root):
@@ -169,7 +166,7 @@ class TreeWalker(_base.NonRecursiveTreeWalker):
                     attrs, len(node) > 0 or node.text)
 
     def getFirstChild(self, node):
-        assert not isinstance(node, tuple), _("Text nodes have no children")
+        assert not isinstance(node, tuple), "Text nodes have no children"
 
         assert len(node) or node.text, "Node has no children"
         if node.text:
@@ -180,7 +177,7 @@ class TreeWalker(_base.NonRecursiveTreeWalker):
     def getNextSibling(self, node):
         if isinstance(node, tuple):  # Text node
             node, key = node
-            assert key in ("text", "tail"), _("Text nodes are text or tail, found %s") % key
+            assert key in ("text", "tail"), "Text nodes are text or tail, found %s" % key
             if key == "text":
                 # XXX: we cannot use a "bool(node) and node[0] or None" construct here
                 # because node[0] might evaluate to False if it has no child element
@@ -196,7 +193,7 @@ class TreeWalker(_base.NonRecursiveTreeWalker):
     def getParentNode(self, node):
         if isinstance(node, tuple):  # Text node
             node, key = node
-            assert key in ("text", "tail"), _("Text nodes are text or tail, found %s") % key
+            assert key in ("text", "tail"), "Text nodes are text or tail, found %s" % key
             if key == "text":
                 return node
             # else: fallback to "normal" processing

included_dependencies/html5lib/utils.py

@@ -2,6 +2,8 @@ from __future__ import absolute_import, division, unicode_literals
 
 from types import ModuleType
 
+from six import text_type
+
 try:
     import xml.etree.cElementTree as default_etree
 except ImportError:
@@ -9,7 +11,26 @@ except ImportError:
 
 
 __all__ = ["default_etree", "MethodDispatcher", "isSurrogatePair",
-           "surrogatePairToCodepoint", "moduleFactoryFactory"]
+           "surrogatePairToCodepoint", "moduleFactoryFactory",
+           "supports_lone_surrogates"]
+
+
+# Platforms not supporting lone surrogates (\uD800-\uDFFF) should be
+# caught by the below test. In general this would be any platform
+# using UTF-16 as its encoding of unicode strings, such as
+# Jython. This is because UTF-16 itself is based on the use of such
+# surrogates, and there is no mechanism to further escape such
+# escapes.
+try:
+    _x = eval('"\\uD800"')
+    if not isinstance(_x, text_type):
+        # We need this with u"" because of http://bugs.jython.org/issue2039
+        _x = eval('u"\\uD800"')
+        assert isinstance(_x, text_type)
+except:
+    supports_lone_surrogates = False
+else:
+    supports_lone_surrogates = True
 
 
 class MethodDispatcher(dict):

makeplugin.py

@@ -25,7 +25,7 @@ if __name__=="__main__":
     exclude=['*.pyc','*~','*.xcf','*[0-9].png','*.po','*.pot','*default.mo','*Thumbs.db']
     
     os.chdir('calibre-plugin')
-    files=['plugin-defaults.ini','plugin-example.ini','about.txt',
+    files=['plugin-defaults.ini','plugin-example.ini','about.html',
            'images','translations']
     files.extend(glob('*.py'))
     files.extend(glob('plugin-import-name-*.txt'))
@@ -36,7 +36,7 @@ if __name__=="__main__":
 
     os.chdir('../included_dependencies')
     # 'a' for append
-    files=['six.py','bs4','html5lib','chardet']
+    files=['gif.py','six.py','bs4','html5lib','chardet']
     createZipFile("../"+filename,"a",
                   files,
                   exclude=exclude)

setup.py

@@ -10,13 +10,11 @@ https://github.com/pypa/sampleproject
 # Always prefer setuptools over distutils
 from setuptools import setup, find_packages
 # To use a consistent encoding
-from codecs import open
+import codecs
 from os import path
 
-here = path.abspath(path.dirname(__file__))
-
 # Get the long description from the relevant file
-with open(path.join(here, 'DESCRIPTION.rst'), encoding='utf-8') as f:
+with codecs.open('DESCRIPTION.rst', encoding='utf-8') as f:
     long_description = f.read()
 
 setup(
@@ -25,7 +23,7 @@ setup(
     # Versions should comply with PEP440.  For a discussion on single-sourcing
     # the version across setup.py and the project code, see
     # https://packaging.python.org/en/latest/single_source_version.html
-    version="2.2.0.b1",
+    version="2.2.12",
 
     description='A tool for downloading fanfiction to eBook formats',
     long_description=long_description,
@@ -46,7 +44,7 @@ setup(
         #   3 - Alpha
         #   4 - Beta
         #   5 - Production/Stable
-        'Development Status :: 4 - Beta',
+        'Development Status :: 5 - Production/Stable',
 
         'Environment :: Console',
 

webservice/app.yaml

@@ -1,6 +1,6 @@
-# ffd-retief-hrd fanfictiondownloader fanficfare
+# ffd-retief-hrd fanficfare
 application: fanficfare
-version: 2-2-0a1
+version: 2-2-12
 runtime: python27
 api_version: 1
 threadsafe: true

webservice/ffstorage.py

@@ -41,6 +41,8 @@ class DownloadMeta(db.Model):
 	completed =  db.BooleanProperty(default=False)
 	date = db.DateTimeProperty(auto_now_add=True)
 	version = db.StringProperty()
+	ch_begin = db.StringProperty()
+	ch_end = db.StringProperty()
 	# data_chunks is implicit from DownloadData def.
 
 class DownloadData(db.Model):

webservice/index.html

@@ -2,7 +2,7 @@
 <html>
   <head>
     <link href="/css/index.css" rel="stylesheet" type="text/css">
-    <title>FanFicFare - read Fanfiction from twilighted.net, fanfiction.net, fictionpress.com, fictionalley.org, ficwad.com, potionsandsnitches.net, harrypotterfanfiction.com, mediaminer.org on Kindle, Nook, Sony Reader, iPad, iPhone, Android, Aldiko, Stanza</title>
+    <title>FanFicFare - read Fanfiction from fanfiction.net, archiveofourown.org, fimfiction.net, fictionpress.com, fictionalley.org, ficwad.com, potionsandsnitches.net, harrypotterfanfiction.com, mediaminer.org and many others on Kindle, Nook, Sony Reader, iPad, iPhone, Android, Aldiko, Stanza, etc</title>
     <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
     
     <!-- Place this tag in your head or just before your close body tag -->
@@ -27,28 +27,28 @@
 	      much easier. </p>
 	  </div>
 
-	  <h3>Changes:</h3>
+	  <h3>New Name and URL</h3>
           <p>
             This version changes the name of the project from
-            FanFictionDownLoader(FFDL) to FanFicFare(FFF).  Along with
+            FanFictionDownLoader(FFDL) to FanFicFare.  Along with
             the name change, there are changes to the internal
             structuring of the project code, but not to the functionality.
           </p>
           <p>
             If you had saved configuration (including your
             username/password for fic sites) in FFDL, you will need to
-            copy it over to FFF.  Visit
+            copy it over to FanFicFare.  Visit
             the <a href="http://fanfictiondownloader.appspot.com/editconfig">FFDL
             User Config</a> page, copy your settings and then paste
             them on
-            the <a href="http://fanficfare.appspot.com/editconfig">FFF
+            the <a href="http://fanficfare.appspot.com/editconfig">FanFicFare
             User Config</a> page.
           </p>
-	    <!-- ul>
-              <li>Add devianthearts.com as a Base eFiction site.</li>
-              <li>Allow https URLs for fictionmania.tv.</li>
-              <li>Allow storiesofarda.com/chapterAllview.asp?SID=999 urls.</li>
-	    </ul -->
+	  <h3>Changes:</h3>
+	  <ul>
+            <li>New site: <a href="https://forum.questionablequesting.com/forums/creative-writing.19/">forum.questionablequesting.com</a> base_xenforoforum adapter</li>
+            <li>Add minor fix for portkey.org chapter parsing.</li>
+	  </ul>
 	  <p>
 	    Questions?  Check out our 
 	    <a href="https://github.com/JimmXinu/FanFicFare/wiki/FAQs">FAQs</a>.
@@ -57,8 +57,8 @@
 	    If you have any problems with this application, please
 	    report them in
 	    the <a href="http://groups.google.com/group/fanfic-downloader">FanFicFare Google Group</a>.  The
-	    <a href="http://fanfictiondownloader.appspot.com">FanFictionDownLoader(FFDL)
-	    Version</a> is also available for you to use if necessary.
+	    <a href="http://2-2-11.fanficfare.appspot.com">previous version
+	    </a> is also available for you to use if necessary.
 	  </p>
 	  <div id='error'>
 	    {{ error_message }}
@@ -119,7 +119,7 @@
 	    Once you have calibre installed and running, inside
 	    calibre, you can go to 'Get plugins to enhance calibre' or
 	    'Get new plugins' and
-	    install <a href="http://www.mobileread.com/forums/showthread.php?t=163261">FanFicFare</a>.
+	    install <a href="http://www.mobileread.com/forums/showthread.php?t=259221">FanFicFare</a>.
 	    
 	  </p>
       </div>
@@ -127,7 +127,7 @@
 	<h3>Supported sites:</h3>
 	<p>
 	  There's a 
- 	  <a href="https://github.com/JimmXinu/FanFicFare/Supportedsites">Supported
+ 	  <a href="https://github.com/JimmXinu/FanFicFare/wiki/Supportedsites">Supported
  	  Sites</a> page in our wiki.  If you have a site you'd like
  	  to see supported, please check there first.
 	</p>

webservice/main.py

@@ -62,7 +62,7 @@ class UserConfigServer(webapp2.RequestHandler):
     
     def getUserConfig(self,user,url,fileformat):
 
-        configuration = Configuration(adapters.getConfigSectionFor(url),fileformat)
+        configuration = Configuration(adapters.getConfigSectionsFor(url),fileformat)
         
         logging.debug('reading defaults.ini config file')
         configuration.read('fanficfare/defaults.ini')
@@ -366,6 +366,16 @@ class FanfictionDownloader(UserConfigServer):
             self.redirect('/')
             return
 
+        # Allow chapter range with URL.
+        # test1.com?sid=5[4-6]
+        mc = re.match(r"^(?P<url>.*?)(?:\[(?P<begin>\d+)?(?P<comma>[,-])?(?P<end>\d+)?\])?$",url)
+        #print("url:(%s) begin:(%s) end:(%s)"%(mc.group('url'),mc.group('begin'),mc.group('end')))
+        url = mc.group('url')
+        ch_begin = mc.group('begin')
+        ch_end = mc.group('end')
+        if ch_begin and not mc.group('comma'):
+            ch_end = ch_begin
+        
         logging.info("Queuing Download: %s" % url)
         login = self.request.get('login')
         password = self.request.get('password')
@@ -385,6 +395,7 @@ class FanfictionDownloader(UserConfigServer):
                 return
 
             adapter = adapters.getAdapter(configuration,url)
+            adapter.setChaptersRange(ch_begin,ch_end)
             logging.info('Created an adaper: %s' % adapter)
 
             if login or password:
@@ -408,6 +419,8 @@ class FanfictionDownloader(UserConfigServer):
             download.title = story.getMetadata('title')
             download.author = story.getMetadata('author')
             download.url = story.getMetadata('storyUrl')
+            download.ch_begin = ch_begin
+            download.ch_end = ch_end
             download.put()
 
             taskqueue.add(url='/fdowntask',
@@ -490,6 +503,7 @@ class FanfictionDownloaderTask(UserConfigServer):
         try:
             configuration = self.getUserConfig(user,url,format)
             adapter = adapters.getAdapter(configuration,url)
+            adapter.setChaptersRange(download.ch_begin,download.ch_end)
 
             logging.info('Created an adapter: %s' % adapter)