Bump version to v2.2.10

Prevent duplicated story URLs even without normalization

calibre-plugin/__init__.py

@@ -42,7 +42,7 @@ class FanFicFareBase(InterfaceActionBase):
     description         = _('UI plugin to download FanFiction stories from various sites.')
     supported_platforms = ['windows', 'osx', 'linux']
     author              = 'Jim Miller'
-    version             = (2, 2, 4)
+    version             = (2, 2, 10)
     minimum_calibre_version = (1, 48, 0)
 
     #: This field defines the GUI plugin class that contains all the code

calibre-plugin/config.py

@@ -55,6 +55,9 @@ try:
 except NameError:
     pass # load_translations() added in calibre 1.9
 
+from calibre.library.field_metadata import FieldMetadata
+field_metadata = FieldMetadata()
+
 # There are a number of things used several times that shouldn't be
 # translated.  This is just a way to make that easier by keeping them
 # out of the _() strings.
@@ -75,6 +78,8 @@ no_trans = { 'pini':'personal.ini',
              'p':'password',
              }
 
+STD_COLS_SKIP = ['size','cover','news','ondevice','path','series_sort','sort']
+
 from calibre_plugins.fanficfare_plugin.prefs \
     import (prefs, PREFS_NAMESPACE, updatecalcover_order, calcover_save_options,
             gencalcover_order, SAVE_YES, SAVE_NO)
@@ -254,6 +259,7 @@ class ConfigWidget(QWidget):
         prefs['checkforseriesurlid'] = self.basic_tab.checkforseriesurlid.isChecked()
         prefs['checkforurlchange'] = self.basic_tab.checkforurlchange.isChecked()
         prefs['injectseries'] = self.basic_tab.injectseries.isChecked()
+        prefs['matchtitleauth'] = self.basic_tab.matchtitleauth.isChecked()
         prefs['smarten_punctuation'] = self.basic_tab.smarten_punctuation.isChecked()
         prefs['reject_always'] = self.basic_tab.reject_always.isChecked()
 
@@ -275,6 +281,8 @@ class ConfigWidget(QWidget):
             # if they've removed everything, reset to default.
             prefs['personal.ini'] = get_resources('plugin-example.ini')
 
+        prefs['cal_cols_pass_in'] = self.personalini_tab.cal_cols_pass_in.isChecked()
+            
         # Covers tab
         prefs['updatecalcover'] = calcover_save_options[unicode(self.calibrecover_tab.updatecalcover.currentText())]
         # for backward compatibility:
@@ -316,10 +324,15 @@ class ConfigWidget(QWidget):
             colsnewonly[col] = checkbox.isChecked()
         prefs['std_cols_newonly'] = colsnewonly
 
+        prefs['set_author_url'] =self.std_columns_tab.set_author_url.isChecked()
+
         # Custom Columns tab
         # error column
         prefs['errorcol'] = unicode(convert_qvariant(self.cust_columns_tab.errorcol.itemData(self.cust_columns_tab.errorcol.currentIndex())))
 
+        # metadata column
+        prefs['savemetacol'] = unicode(convert_qvariant(self.cust_columns_tab.savemetacol.itemData(self.cust_columns_tab.savemetacol.currentIndex())))
+
         # cust cols tab
         colsmap = {}
         for (col,combo) in self.cust_columns_tab.custcol_dropdowns.iteritems():
@@ -336,12 +349,13 @@ class ConfigWidget(QWidget):
         
         prefs['allow_custcol_from_ini'] = self.cust_columns_tab.allow_custcol_from_ini.isChecked()
 
-        prefs['imapserver'] = unicode(self.imap_tab.imapserver.text())
-        prefs['imapuser'] = unicode(self.imap_tab.imapuser.text())
-        prefs['imappass'] = unicode(self.imap_tab.imappass.text())
-        prefs['imapfolder'] = unicode(self.imap_tab.imapfolder.text())
+        prefs['imapserver'] = unicode(self.imap_tab.imapserver.text()).strip()
+        prefs['imapuser'] = unicode(self.imap_tab.imapuser.text()).strip()
+        prefs['imappass'] = unicode(self.imap_tab.imappass.text()).strip()
+        prefs['imapfolder'] = unicode(self.imap_tab.imapfolder.text()).strip()
         prefs['imapmarkread'] = self.imap_tab.imapmarkread.isChecked()
         prefs['imapsessionpass'] = self.imap_tab.imapsessionpass.isChecked()
+        prefs['auto_reject_from_email'] = self.imap_tab.auto_reject_from_email.isChecked()
         
         prefs.save_to_db()
 
@@ -508,6 +522,11 @@ class BasicTab(QWidget):
         self.injectseries.setChecked(prefs['injectseries'])
         self.l.addWidget(self.injectseries)
 
+        self.matchtitleauth = QCheckBox(_("Search by Title/Author(s) for If Story Already Exists?"),self)
+        self.matchtitleauth.setToolTip(_("When checking <i>If Story Already Exists</i> FanFicFare will first match by URL Identifier.  But if not found, it can also search existing books by Title and Author(s)."))
+        self.matchtitleauth.setChecked(prefs['matchtitleauth'])
+        self.l.addWidget(self.matchtitleauth)
+
         rej_gb = groupbox = QGroupBox(_("Reject List"))
         self.l = QVBoxLayout()
         groupbox.setLayout(self.l)
@@ -637,15 +656,25 @@ class PersonalIniTab(QWidget):
         self.ini_button.clicked.connect(self.add_ini_button)
         self.l.addWidget(self.ini_button)
         
+        label = QLabel(_("Changes will only be saved if you click 'OK' to leave Customize FanFicFare."))
+        label.setWordWrap(True)
+        self.l.addWidget(label)
+        
         self.defaults = QPushButton(_('View Defaults')+' (plugin-defaults.ini)', self)
         self.defaults.setToolTip(_("View all of the plugin's configurable settings\nand their default settings."))
         self.defaults.clicked.connect(self.show_defaults)
         self.l.addWidget(self.defaults)
 
-        label = QLabel(_("Changes will only be saved if you click 'OK' to leave Customize FanFicFare."))
-        label.setWordWrap(True)
-        self.l.addWidget(label)
+        self.cal_cols_pass_in = QCheckBox(_('Pass Calibre Columns into FanFicFare on Update/Overwrite')%no_trans,self)
+        self.cal_cols_pass_in.setToolTip(_("If checked, when updating/overwriting an existing book, FanFicFare will have the Calibre Columns available to use in replace_metadata, title_page, etc.<br>Click the button below to see the Calibre Column namess.")%no_trans)
+        self.cal_cols_pass_in.setChecked(prefs['cal_cols_pass_in'])
+        self.l.addWidget(self.cal_cols_pass_in)
         
+        self.showcalcols = QPushButton(_('Show Calibre Column Names'), self)
+        self.showcalcols.setToolTip(_("FanFicFare can pass the Calibre Columns into the download/update process.<br>This will show you the columns available by name."))
+        self.showcalcols.clicked.connect(self.show_showcalcols)
+        self.l.addWidget(self.showcalcols)
+
         self.l.insertStretch(-1)
         # let edit box fill the space.
         
@@ -671,6 +700,27 @@ class PersonalIniTab(QWidget):
         if d.result() == d.Accepted:
             self.personalini = d.get_plain_text()
                 
+    def show_showcalcols(self):
+        lines=[]#[('calibre_std_user_categories',_('User Categories'))]
+        for k,f in field_metadata.iteritems():
+            if f['name'] and k not in STD_COLS_SKIP: # only if it has a human readable name.
+                lines.append(('calibre_std_'+k,f['name']))
+            
+        for k, column in self.plugin_action.gui.library_view.model().custom_columns.iteritems():
+            if k != prefs['savemetacol']:
+                # custom always have name.
+                lines.append(('calibre_cust_'+k[1:],column['name']))
+
+        lines.sort() # sort by key.
+
+        EditTextDialog(self,
+                       '\n'.join(['%s (%s)'%(l,k) for (k,l) in lines]),
+                       icon=self.windowIcon(),
+                       title=_('Calibre Column Entry Names'),
+                       label=_('Label (entry_name)'),
+                       read_only=True,
+                       save_size_name='fff:showcalcols').exec_()
+        
 class ReadingListTab(QWidget):
 
     def __init__(self, parent_dialog, plugin_action):
@@ -1188,7 +1238,6 @@ class CustomColumnsTab(QWidget):
         self.allow_custcol_from_ini.setChecked(prefs['allow_custcol_from_ini'])
         self.l.addWidget(self.allow_custcol_from_ini)
         
-        self.l.addSpacing(5)
         label = QLabel(_("Special column:"))
         label.setWordWrap(True)
         self.l.addWidget(label)
@@ -1207,6 +1256,21 @@ class CustomColumnsTab(QWidget):
         self.errorcol.setCurrentIndex(self.errorcol.findData(prefs['errorcol']))
         horz.addWidget(self.errorcol)
         self.l.addLayout(horz)
+        
+        horz = QHBoxLayout()
+        label = QLabel(_("Saved Metadata Column:"))
+        tooltip=_("If set, FanFicFare will save a copy of all its metadata in this column when the book is downloaded or updated.<br/>The metadata from this column can later be used to update custom columns without having to request the metadata from the server again.<br/>(Long Text columns only.)")
+        label.setToolTip(tooltip)
+        horz.addWidget(label)
+        self.savemetacol = QComboBox(self)
+        self.savemetacol.setToolTip(tooltip)
+        self.savemetacol.addItem('','')
+        for key, column in custom_columns.iteritems():
+            if column['datatype'] in ('comments'):
+                self.savemetacol.addItem(column['name'],key)
+        self.savemetacol.setCurrentIndex(self.savemetacol.findData(prefs['savemetacol']))
+        horz.addWidget(self.savemetacol)
+        self.l.addLayout(horz)
 
         #print("prefs['custom_cols'] %s"%prefs['custom_cols'])
 
@@ -1255,6 +1319,17 @@ class StandardColumnsTab(QWidget):
             horz.addWidget(newonlycheck)
             
             self.l.addLayout(horz)
+            
+        self.l.addSpacing(5)
+        label = QLabel(_("Other Standard Column Options"))
+        label.setWordWrap(True)
+        self.l.addWidget(label)
+        self.l.addSpacing(5)
+
+        self.set_author_url = QCheckBox(_('Set Calibre Author URL'),self)
+        self.set_author_url.setToolTip(_("Set Calibre Author URL to Author's URL on story site."))
+        self.set_author_url.setChecked(prefs['set_author_url'])
+        self.l.addWidget(self.set_author_url)
         
         self.l.insertStretch(-1)
 
@@ -1327,6 +1402,12 @@ class ImapTab(QWidget):
         self.l.addWidget(self.imapmarkread,row,0,1,-1)
         row+=1
             
+        self.auto_reject_from_email = QCheckBox(_('Discard URLs on Reject List'),self)
+        self.auto_reject_from_email.setToolTip(_('If checked, FanFicFare will silently discard story URLs from emails that are on your Reject URL List.<br>Otherwise they will appear and you will see the normal Reject URL dialog.<br>The Emails will still be marked Read if configured to.'))
+        self.auto_reject_from_email.setChecked(prefs['auto_reject_from_email'])
+        self.l.addWidget(self.auto_reject_from_email,row,0,1,-1)
+        row+=1
+        
         label = QLabel(_("<b>It's safest if you create a separate email account that you use only "
                          "for your story update notices.  FanFicFare and calibre cannot guarantee that "
                          "malicious code cannot get your email password once you've entered it. "

calibre-plugin/dialogs.py

@@ -79,14 +79,16 @@ UPDATE=_('Update EPUB if New Chapters')
 UPDATEALWAYS=_('Update EPUB Always')
 OVERWRITE=_('Overwrite if Newer')
 OVERWRITEALWAYS=_('Overwrite Always')
-CALIBREONLY=_('Update Calibre Metadata Only')
+CALIBREONLY=_('Update Calibre Metadata from Web Site')
+CALIBREONLYSAVECOL=_('Update Calibre Metadata from Saved Metadata Column')
 collision_order=[SKIP,
                  ADDNEW,
                  UPDATE,
                  UPDATEALWAYS,
                  OVERWRITE,
                  OVERWRITEALWAYS,
-                 CALIBREONLY,]
+                 CALIBREONLY,
+                 CALIBREONLYSAVECOL,]
 
 # best idea I've had for how to deal with config/pref saving the
 # collision name in english.
@@ -97,6 +99,7 @@ SAVE_UPDATEALWAYS='Update EPUB Always'
 SAVE_OVERWRITE='Overwrite if Newer'
 SAVE_OVERWRITEALWAYS='Overwrite Always'
 SAVE_CALIBREONLY='Update Calibre Metadata Only'
+SAVE_CALIBREONLYSAVECOL='Update Calibre Metadata Only(Saved Column)'
 save_collisions={
     SKIP:SAVE_SKIP,
     ADDNEW:SAVE_ADDNEW,
@@ -112,6 +115,7 @@ save_collisions={
     SAVE_OVERWRITE:OVERWRITE,
     SAVE_OVERWRITEALWAYS:OVERWRITEALWAYS,
     SAVE_CALIBREONLY:CALIBREONLY,
+    SAVE_CALIBREONLYSAVECOL:CALIBREONLYSAVECOL,
     }
     
 anthology_collision_order=[UPDATE,
@@ -303,7 +307,7 @@ class AddNewDialog(SizePersistedDialog):
         self.collision.setToolTip("CollisionToolTip")
         # add collision options
         self.set_collisions()
-        i = self.collision.findText(save_collisions[prefs['collision']])
+        i = self.collision.findText(save_collisions[self.prefs['collision']])
         if i > -1:
             self.collision.setCurrentIndex(i)
         self.collisionlabel.setBuddy(self.collision)
@@ -317,14 +321,14 @@ class AddNewDialog(SizePersistedDialog):
         horz = QHBoxLayout()
         self.updatemeta = QCheckBox(_('Update Calibre &Metadata?'),self)
         self.updatemeta.setToolTip(_("Update metadata for existing stories in Calibre from web site?\n(Columns set to 'New Only' in the column tabs will only be set for new books.)"))
-        self.updatemeta.setChecked(prefs['updatemeta'])
+        self.updatemeta.setChecked(self.prefs['updatemeta'])
         horz.addWidget(self.updatemeta)
         self.mergehide.append(self.updatemeta)
         self.mergeupdateshow.append(self.updatemeta)
 
         self.updateepubcover = QCheckBox(_('Update EPUB Cover?'),self)
         self.updateepubcover.setToolTip(_('Update book cover image from site or defaults (if found) <i>inside</i> the EPUB when EPUB is updated.'))
-        self.updateepubcover.setChecked(prefs['updateepubcover'])
+        self.updateepubcover.setChecked(self.prefs['updateepubcover'])
         horz.addWidget(self.updateepubcover)
         self.mergehide.append(self.updateepubcover)
         
@@ -354,7 +358,10 @@ class AddNewDialog(SizePersistedDialog):
                     extrapayload=None):
         # rather than mutex in fff_plugin, just bail here if it's
         # already in use.
-        if self.isVisible(): return
+        if self.isVisible():
+            if url_list_text: # add to open box.
+               self.url.setText( '\n'.join([self.get_urlstext(), url_list_text]) )
+            return
 
         try:
             self.go_signal.disconnect()
@@ -427,12 +434,19 @@ class AddNewDialog(SizePersistedDialog):
         prev=self.collision.currentText()
         self.collision.clear()
         if self.merge:
-            order = anthology_collision_order
+            order = list(anthology_collision_order)
         else:
-            order = collision_order
+            order = list(collision_order)
+            ## Remove options that aren't valid.
+            if self.fileform.currentText() != 'epub':
+                order.remove(UPDATE)
+                order.remove(UPDATEALWAYS)
+            if self.prefs['savemetacol'] == '':
+                order.remove(CALIBREONLYSAVECOL)
+
         for o in order:
-            if self.merge or self.fileform.currentText() == 'epub' or o not in [UPDATE,UPDATEALWAYS]:
-                self.collision.addItem(o)
+            self.collision.addItem(o)
+            
         i = self.collision.findText(prev)
         if i > -1:
             self.collision.setCurrentIndex(i)
@@ -783,7 +797,7 @@ class UpdateExistingDialog(SizePersistedDialog):
         self.fileform.addItem('mobi')
         self.fileform.addItem('html')
         self.fileform.addItem('txt')
-        self.fileform.setCurrentIndex(self.fileform.findText(prefs['fileform']))
+        self.fileform.setCurrentIndex(self.fileform.findText(self.prefs['fileform']))
         self.fileform.setToolTip(_('Choose output format to create.  May set default from plugin configuration.'))
         self.fileform.activated.connect(self.set_collisions)
         label.setBuddy(self.fileform)
@@ -795,7 +809,7 @@ class UpdateExistingDialog(SizePersistedDialog):
         self.collision.setToolTip(_("What sort of update to perform.  May set default from plugin configuration."))
         # add collision options
         self.set_collisions()
-        i = self.collision.findText(save_collisions[prefs['collision']])
+        i = self.collision.findText(save_collisions[self.prefs['collision']])
         if i > -1:
             self.collision.setCurrentIndex(i)
         label.setBuddy(self.collision)
@@ -803,12 +817,12 @@ class UpdateExistingDialog(SizePersistedDialog):
 
         self.updatemeta = QCheckBox(_('Update Calibre &Metadata?'),self)
         self.updatemeta.setToolTip(_("Update metadata for existing stories in Calibre from web site?\n(Columns set to 'New Only' in the column tabs will only be set for new books.)"))
-        self.updatemeta.setChecked(prefs['updatemeta'])
+        self.updatemeta.setChecked(self.prefs['updatemeta'])
         gbl.addWidget(self.updatemeta)
                 
         self.updateepubcover = QCheckBox(_('Update EPUB Cover?'),self)
         self.updateepubcover.setToolTip(_('Update book cover image from site or defaults (if found) <i>inside</i> the EPUB when EPUB is updated.'))
-        self.updateepubcover.setChecked(prefs['updateepubcover'])
+        self.updateepubcover.setChecked(self.prefs['updateepubcover'])
         gbl.addWidget(self.updateepubcover)
 
 
@@ -827,10 +841,18 @@ class UpdateExistingDialog(SizePersistedDialog):
     def set_collisions(self):
         prev=self.collision.currentText()
         self.collision.clear()
-        for o in collision_order:
-            if o not in [ADDNEW,SKIP] and \
-                    (self.fileform.currentText() == 'epub' or o not in [UPDATE,UPDATEALWAYS]):
-                self.collision.addItem(o)        
+        order = list(collision_order)
+        order.remove(ADDNEW)
+        order.remove(SKIP)
+        if self.fileform.currentText() != 'epub':
+            order.remove(UPDATE)
+            order.remove(UPDATEALWAYS)
+        if self.prefs['savemetacol'] == '':
+            order.remove(CALIBREONLYSAVECOL)
+            
+        for o in order:
+            self.collision.addItem(o)
+            
         i = self.collision.findText(prev)
         if i > -1:
             self.collision.setCurrentIndex(i)
@@ -1132,6 +1154,7 @@ class EditTextDialog(SizePersistedDialog):
 
     def __init__(self, parent, text,
                  icon=None, title=None, label=None, tooltip=None,
+                 read_only=False,
                  rejectreasons=[],reasonslabel=None,
                  save_size_name='fff:edit text dialog',
                  ):
@@ -1148,6 +1171,7 @@ class EditTextDialog(SizePersistedDialog):
         
         self.textedit = QTextEdit(self)
         self.textedit.setLineWrapMode(QTextEdit.NoWrap)
+        self.textedit.setReadOnly(read_only)
         self.textedit.setText(text)
         self.l.addWidget(self.textedit)
 

calibre-plugin/fff_plugin.py

@@ -13,24 +13,24 @@ logger = logging.getLogger(__name__)
 import time, os, copy, threading, re, platform, sys
 from StringIO import StringIO
 from functools import partial
-from datetime import datetime, time
+from datetime import datetime, time, date
 from string import Template
 import urllib
 import email
 import traceback
 
 try:
-    from PyQt5.Qt import (QApplication, QMenu, QTimer)
+    from PyQt5.Qt import (QApplication, QMenu, QTimer, QCursor, Qt)
     from PyQt5.QtCore import QBuffer
 except ImportError as e:
-    from PyQt4.Qt import (QApplication, QMenu, QTimer)
+    from PyQt4.Qt import (QApplication, QMenu, QTimer, QCursor, Qt)
     from PyQt4.QtCore import QBuffer
 
 from calibre.constants import numeric_version as calibre_version
 
 from calibre.ptempfile import PersistentTemporaryFile, PersistentTemporaryDirectory, remove_dir
 from calibre.ebooks.metadata import MetaInformation
-from calibre.ebooks.metadata.meta import get_metadata
+from calibre.ebooks.metadata.meta import get_metadata as calibre_get_metadata
 from calibre.gui2 import error_dialog, warning_dialog, question_dialog, info_dialog
 from calibre.gui2.dialogs.message_box import ViewLog
 from calibre.gui2.dialogs.confirm_delete import confirm
@@ -55,21 +55,38 @@ try:
 except:
     HAS_CALGC=False
 
-from calibre_plugins.fanficfare_plugin.common_utils import (set_plugin_icon_resources, get_icon,
-                                                            create_menu_action_unique, get_library_uuid)
+from calibre.library.field_metadata import FieldMetadata
+field_metadata = FieldMetadata()
+    
+from calibre_plugins.fanficfare_plugin.common_utils import (
+    set_plugin_icon_resources, get_icon, create_menu_action_unique,
+    get_library_uuid)
 
-from calibre_plugins.fanficfare_plugin.fanficfare import adapters, exceptions
-from calibre_plugins.fanficfare_plugin.fanficfare.epubutils import get_dcsource, get_dcsource_chaptercount, get_story_url_from_html
-from calibre_plugins.fanficfare_plugin.fanficfare.geturls import get_urls_from_page, get_urls_from_html, get_urls_from_text, get_urls_from_imap
+from calibre_plugins.fanficfare_plugin.fanficfare import (
+    adapters, exceptions)
+
+from calibre_plugins.fanficfare_plugin.fanficfare.epubutils import (
+    get_dcsource, get_dcsource_chaptercount, get_story_url_from_html)
+
+from calibre_plugins.fanficfare_plugin.fanficfare.geturls import (
+    get_urls_from_page, get_urls_from_html,get_urls_from_text,
+    get_urls_from_imap)
+
+from calibre_plugins.fanficfare_plugin.fff_util import (
+    get_fff_adapter, get_fff_config, get_fff_personalini)
+
+from calibre_plugins.fanficfare_plugin.config import (
+    permitted_values, rejecturllist, STD_COLS_SKIP)
+
+from calibre_plugins.fanficfare_plugin.prefs import (
+    prefs, SAVE_YES, SAVE_NO, SAVE_YES_IF_IMG, SAVE_YES_UNLESS_IMG)
 
-from calibre_plugins.fanficfare_plugin.fff_util import (get_fff_adapter, get_fff_config, get_fff_personalini)
-from calibre_plugins.fanficfare_plugin.config import (permitted_values, rejecturllist)
-from calibre_plugins.fanficfare_plugin.prefs import (prefs, SAVE_YES, SAVE_NO,
-                                                     SAVE_YES_IF_IMG, SAVE_YES_UNLESS_IMG)
 from calibre_plugins.fanficfare_plugin.dialogs import (
     AddNewDialog, UpdateExistingDialog,
-    LoopProgressDialog, UserPassDialog, AboutDialog, CollectURLDialog, RejectListDialog, EmailPassDialog,
+    LoopProgressDialog, UserPassDialog, AboutDialog, CollectURLDialog,
+    RejectListDialog, EmailPassDialog,
     OVERWRITE, OVERWRITEALWAYS, UPDATE, UPDATEALWAYS, ADDNEW, SKIP, CALIBREONLY,
+    CALIBREONLYSAVECOL,
     NotGoingToDownload, RejectUrlEntry )
 
 # because calibre immediately transforms html into zip and don't want
@@ -421,17 +438,31 @@ class FanFicFarePlugin(InterfaceAction):
             if prefs['imapsessionpass']:
                 self.imap_pass = imap_pass
 
+        self.busy_cursor()
+        self.gui.status_bar.show_message(_('Fetching Story URLs from Email...'))
         url_list = get_urls_from_imap(prefs['imapserver'],
                                       prefs['imapuser'],
                                       imap_pass,
                                       prefs['imapfolder'],
                                       prefs['imapmarkread'],)
+        reject_list=set()
+        if prefs['auto_reject_from_email']:
+            # need to normalize for reject list.
+            reject_list = set([x for x in url_list if rejecturllist.check(adapters.getNormalStoryURLSite(x)[0])])
+        url_list = url_list - reject_list
 
+        self.gui.status_bar.show_message(_('Finished Fetching Story URLs from Email.'),3000)
+        self.restore_cursor()
+        
         if url_list:
             self.add_dialog("\n".join(url_list),merge=False)
         else:
+            
+            msg = _('No Valid Story URLs Found in Unread Emails.')
+            if reject_list:
+                msg = msg + '<p>'+(_('(%d Story URLs Skipped, on Rejected URL List)')%len(reject_list))+'</p>'
             info_dialog(self.gui, _('Get Story URLs from Email'),
-                        _('No Valid Story URLs Found in Unread Emails.'),
+                        msg,
                         show=True,
                         show_copy_button=False)
 
@@ -452,8 +483,14 @@ class FanFicFarePlugin(InterfaceAction):
             return
         url = u"%s"%d.url.text()
 
+        self.busy_cursor()
+        self.gui.status_bar.show_message(_('Fetching Story URLs from Page...'))
+
         url_list = self.get_urls_from_page(url)
 
+        self.gui.status_bar.show_message(_('Finished Fetching Story URLs from Page.'),3000)
+        self.restore_cursor()
+        
         if url_list:
             self.add_dialog("\n".join(url_list),merge=d.anthology,anthology_url=url)
         else:
@@ -638,6 +675,9 @@ class FanFicFarePlugin(InterfaceAction):
             remove_dir(tdir)
             return
 
+        self.busy_cursor()
+        self.gui.status_bar.show_message(_('Fetching Story URLs for Series...'))
+        
         # get list from identifiers:url/uri if present, but only if
         # it's *not* a valid story URL.
         mergeurl = self.get_story_url(db,book_id)
@@ -646,6 +686,9 @@ class FanFicFarePlugin(InterfaceAction):
 
         url_list_text = "\n".join(url_list)
 
+        self.gui.status_bar.show_message(_('Finished Fetching Story URLs for Series.'),3000)
+        self.restore_cursor()
+        
         #print("urlmapfile:%s"%urlmapfile)
 
         # AddNewDialog collects URLs, format and presents buttons.
@@ -767,10 +810,10 @@ class FanFicFarePlugin(InterfaceAction):
         # No need to do anything with perfs here, but we could.
         prefs
 
-    def make_id_searchstr(self,url):
+    def do_id_search(self,url):
         # older idents can be uri vs url and have | instead of : after
         # http, plus many sites are now switching to https.
-        return 'identifiers:"~ur(i|l):~^%s$"'%re.sub(r'https?\\\:','https?(\:|\|)',re.escape(url))
+        return self.gui.current_db.search_getting_ids('identifiers:"~ur(i|l):~^%s$"'%re.sub(r'https?\\\:','https?(\:|\|)',re.escape(url)),None)
 
     def prep_downloads(self, options, books, merge=False, extrapayload=None):
         '''Fetch metadata for stories from servers, launch BG job when done.'''
@@ -785,6 +828,7 @@ class FanFicFarePlugin(InterfaceAction):
         options['version'] = self.version
         logger.debug(self.version)
         options['personal.ini'] = get_fff_personalini()
+        options['savemetacol'] = prefs['savemetacol']
 
         #print("prep_downloads:%s"%books)
 
@@ -876,8 +920,6 @@ class FanFicFarePlugin(InterfaceAction):
             # book has already been flagged bad for whatever reason.
             return
 
-        skip_date_update = False
-
         adapter = get_fff_adapter(url,fileform)
         ## save and share cookiejar and pagecache between all
         ## downloads.
@@ -888,67 +930,91 @@ class FanFicFarePlugin(InterfaceAction):
             options['cookiejar'] = adapter.get_empty_cookiejar()
         adapter.set_cookiejar(options['cookiejar'])
 
-        # reduce foreground sleep time for ffnet when few books.
-        if 'ffnetcount' in options and \
-                adapter.getConfig('tweak_fg_sleep') and \
-                adapter.getSiteDomain() == 'www.fanfiction.net':
-            minslp = float(adapter.getConfig('min_fg_sleep'))
-            maxslp = float(adapter.getConfig('max_fg_sleep'))
-            dwnlds = float(adapter.getConfig('max_fg_sleep_at_downloads'))
-            m = (maxslp-minslp) / (dwnlds-1)
-            b = minslp - m
-            slp = min(maxslp,m*float(options['ffnetcount'])+b)
-            #print("m:%s b:%s = %s"%(m,b,slp))
-            adapter.set_sleep(slp)
-
-        ## three tries, that's enough if both user/pass & is_adult needed,
-        ## or a couple tries of one or the other
-        for x in range(0,2):
-            try:
-                adapter.getStoryMetadataOnly(get_cover=False)
-            except exceptions.FailedToLogin, f:
-                logger.warn("Login Failed, Need Username/Password.")
-                userpass = UserPassDialog(self.gui,url,f)
-                userpass.exec_() # exec_ will make it act modal
-                if userpass.status:
-                    adapter.username = userpass.user.text()
-                    adapter.password = userpass.passwd.text()
-
-            except exceptions.AdultCheckRequired:
-                if question_dialog(self.gui, _('Are You an Adult?'), '<p>'+
-                                   _("%s requires that you be an adult.  Please confirm you are an adult in your locale:")%url,
-                                   show_copy_button=False):
-                    adapter.is_adult=True
-
-        # let other exceptions percolate up.
-        story = adapter.getStoryMetadataOnly(get_cover=False)
-
-        series = story.getMetadata('series')
-        if not merge and series and prefs['checkforseriesurlid']:
-            # try to find *series anthology* by *seriesUrl* identifier url or uri first.
-            searchstr = self.make_id_searchstr(story.getMetadata('seriesUrl'))
-            identicalbooks = db.search_getting_ids(searchstr, None)
-            # print("searchstr:%s"%searchstr)
-            # print("identicalbooks:%s"%identicalbooks)
-            if len(identicalbooks) > 0 and question_dialog(self.gui, _('Skip Story?'),'''
-                                                              <h3>%s</h3>
-                                                              <p>%s</p>
-                                                              <p>%s</p>
-                                                              <p>%s</p>
-                                                           '''%(
-                                                           _('Skip Anthology Story?'),
-                                                           _('"<b>%s</b>" is in series "<b><a href="%s">%s</a></b>" that you have an anthology book for.')%(story.getMetadata('title'),story.getMetadata('seriesUrl'),series[:series.index(' [')]),
-                                                           _("Click '<b>Yes</b>' to Skip."),
-                                                           _("Click '<b>No</b>' to download anyway.")),
-                                                           show_copy_button=False):
-                book['comment'] = _("Story in Series Anthology(%s).")%series
-                book['title'] = story.getMetadata('title')
-                book['author'] = [story.getMetadata('author')]
-                book['good']=False
-                book['icon']='rotate-right.png'
-                book['status'] = _('Skipped')
-                return
+        if collision in (CALIBREONLY, CALIBREONLYSAVECOL):
+            ## Getting metadata from configured column.
+            custom_columns = self.gui.library_view.model().custom_columns
+            if ( collision in (CALIBREONLYSAVECOL) and
+                 prefs['savemetacol'] != '' and 
+                 prefs['savemetacol'] in custom_columns ):
 
+                savedmeta_book_id = book['calibre_id']
+                # won't have calibre_id if update by URL vs book.
+                if not savedmeta_book_id:
+                    identicalbooks = self.do_id_search(url)
+                    if len(identicalbooks) == 1:
+                        savedmeta_book_id = identicalbooks.pop()
+                        
+                if savedmeta_book_id:
+                    label = custom_columns[prefs['savemetacol']]['label']
+                    savedmetadata = db.get_custom(savedmeta_book_id, label=label, index_is_id=True)
+                else:
+                    savedmetadata = None
+                    
+                if savedmetadata:
+                    # sets flag inside story so getStoryMetadataOnly won't hit server.
+                    adapter.setStoryMetadata(savedmetadata)
+                
+            # let other exceptions percolate up.
+            story = adapter.getStoryMetadataOnly(get_cover=False)
+        else:
+            # reduce foreground sleep time for ffnet when few books.
+            if 'ffnetcount' in options and \
+                    adapter.getConfig('tweak_fg_sleep') and \
+                    adapter.getSiteDomain() == 'www.fanfiction.net':
+                minslp = float(adapter.getConfig('min_fg_sleep'))
+                maxslp = float(adapter.getConfig('max_fg_sleep'))
+                dwnlds = float(adapter.getConfig('max_fg_sleep_at_downloads'))
+                m = (maxslp-minslp) / (dwnlds-1)
+                b = minslp - m
+                slp = min(maxslp,m*float(options['ffnetcount'])+b)
+                #print("m:%s b:%s = %s"%(m,b,slp))
+                adapter.set_sleep(slp)
+    
+            ## three tries, that's enough if both user/pass & is_adult needed,
+            ## or a couple tries of one or the other
+            for x in range(0,2):
+                try:
+                    adapter.getStoryMetadataOnly(get_cover=False)
+                except exceptions.FailedToLogin, f:
+                    logger.warn("Login Failed, Need Username/Password.")
+                    userpass = UserPassDialog(self.gui,url,f)
+                    userpass.exec_() # exec_ will make it act modal
+                    if userpass.status:
+                        adapter.username = userpass.user.text()
+                        adapter.password = userpass.passwd.text()
+    
+                except exceptions.AdultCheckRequired:
+                    if question_dialog(self.gui, _('Are You an Adult?'), '<p>'+
+                                       _("%s requires that you be an adult.  Please confirm you are an adult in your locale:")%url,
+                                       show_copy_button=False):
+                        adapter.is_adult=True
+    
+            # let other exceptions percolate up.
+            story = adapter.getStoryMetadataOnly(get_cover=False)
+    
+            series = story.getMetadata('series')
+            if not merge and series and prefs['checkforseriesurlid']:
+                # try to find *series anthology* by *seriesUrl* identifier url or uri first.
+                identicalbooks = self.do_id_search(story.getMetadata('seriesUrl'))
+                # print("identicalbooks:%s"%identicalbooks)
+                if len(identicalbooks) > 0 and question_dialog(self.gui, _('Skip Story?'),'''
+                                                                  <h3>%s</h3>
+                                                                  <p>%s</p>
+                                                                  <p>%s</p>
+                                                                  <p>%s</p>
+                                                               '''%(
+                                                               _('Skip Anthology Story?'),
+                                                               _('"<b>%s</b>" is in series "<b><a href="%s">%s</a></b>" that you have an anthology book for.')%(story.getMetadata('title'),story.getMetadata('seriesUrl'),series[:series.index(' [')]),
+                                                               _("Click '<b>Yes</b>' to Skip."),
+                                                               _("Click '<b>No</b>' to download anyway.")),
+                                                               show_copy_button=False):
+                    book['comment'] = _("Story in Series Anthology(%s).")%series
+                    book['title'] = story.getMetadata('title')
+                    book['author'] = [story.getMetadata('author')]
+                    book['good']=False
+                    book['icon']='rotate-right.png'
+                    book['status'] = _('Skipped')
+                    return
 
         ################################################################################################################################################33
 
@@ -958,7 +1024,10 @@ class FanFicFarePlugin(InterfaceAction):
 
         # all_metadata duplicates some data, but also includes extra_entries, etc.
         book['all_metadata'] = story.getAllMetadata(removeallentities=True)
-
+        if prefs['savemetacol'] != '':
+            # get metadata to save in configured column.
+            book['savemetacol'] = story.dump_html_metadata()
+        
         book['title'] = story.getMetadata("title", removeallentities=True)
         book['author_sort'] = book['author'] = story.getList("author", removeallentities=True)
         book['publisher'] = story.getMetadata("site")
@@ -985,7 +1054,7 @@ class FanFicFarePlugin(InterfaceAction):
             book['timestamp'] = None # need *something* there for calibre.
 
         if not merge:# skip all the collision code when d/ling for merging.
-            if collision in (CALIBREONLY):
+            if collision in (CALIBREONLY, CALIBREONLYSAVECOL):
                 book['icon'] = 'metadata.png'
                 book['status'] = _('Meta')
 
@@ -1003,11 +1072,9 @@ class FanFicFarePlugin(InterfaceAction):
                 logger.debug("from URL(%s)"%url)
 
                 # try to find by identifier url or uri first.
-                searchstr = self.make_id_searchstr(url)
-                identicalbooks = db.search_getting_ids(searchstr, None)
-                # print("searchstr:%s"%searchstr)
+                identicalbooks = self.do_id_search(url)
                 # print("identicalbooks:%s"%identicalbooks)
-                if len(identicalbooks) < 1:
+                if len(identicalbooks) < 1 and prefs['matchtitleauth']:
                     # find dups
                     authlist = story.getList("author", removeallentities=True)
                     mi = MetaInformation(story.getMetadata("title", removeallentities=True),
@@ -1026,7 +1093,7 @@ class FanFicFarePlugin(InterfaceAction):
                     raise NotGoingToDownload(_("More than one identical book by Identifer URL or title/author(s)--can't tell which book to update/overwrite."),"minusminus.png")
 
                 ## changed: add new book when CALIBREONLY if none found.
-                if collision == CALIBREONLY and not identicalbooks:
+                if collision in (CALIBREONLY, CALIBREONLYSAVECOL) and not identicalbooks:
                     collision = ADDNEW
                     options['collision'] = ADDNEW
 
@@ -1081,7 +1148,7 @@ class FanFicFarePlugin(InterfaceAction):
                                 return
 
             if book_id != None and collision != ADDNEW:
-                if collision in (CALIBREONLY):
+                if collision in (CALIBREONLY, CALIBREONLYSAVECOL):
                     book['comment'] = _('Metadata collected.')
                     # don't need temp file created below.
                     return
@@ -1099,9 +1166,6 @@ class FanFicFarePlugin(InterfaceAction):
                         if chaptercount == urlchaptercount:
                             if collision == UPDATE:
                                 raise NotGoingToDownload(_("Already contains %d chapters.")%chaptercount,'edit-undo.png')
-                            else:
-                                # UPDATEALWAYS
-                                skip_date_update = True
                         elif chaptercount > urlchaptercount:
                             raise NotGoingToDownload(_("Existing epub contains %d chapters, web site only has %d. Use Overwrite to force update.") % (chaptercount,urlchaptercount),'dialog_error.png')
                         elif chaptercount == 0:
@@ -1140,6 +1204,52 @@ class FanFicFarePlugin(InterfaceAction):
                     #print("calibre_series:%s [%s]"%book['calibre_series'])
 
         if book['good']: # there shouldn't be any !'good' books at this point.
+
+            ## Filling calibre_std_* and calibre_cust_* metadata
+            book['calibre_columns']={}
+            if prefs['cal_cols_pass_in']:
+                # std columns
+                mi = db.get_metadata(book['calibre_id'],index_is_id=True)
+                # book['calibre_columns']['calibre_std_identifiers']=\
+                #     {'val':', '.join(["%s:%s"%(k,v) for (k,v) in mi.get_identifiers().iteritems()]),
+                #                      'label':_('Ids')}
+                for k in mi.standard_field_keys():
+                # for k in mi:
+                    if k in STD_COLS_SKIP:
+                        continue
+                    (label,value,v,fmd) = mi.format_field_extended(k)
+                    if not label and k in field_metadata:
+                        label=field_metadata[k]['name']
+                    key='calibre_std_'+k
+                    
+                    # if k == 'user_categories':
+                    #     value=u', '.join(mi.get(k))
+                    #     label=_('User Categories')
+        
+                    if label: # only if it has a human readable name.
+                        if value is None or not book['calibre_id']:
+                            ## if existing book, populate existing calibre column
+                            ## values in metadata, else '' to hide.
+                            value=''                    
+                        book['calibre_columns'][key]={'val':value,'label':label}
+                        #logger.debug("%s(%s): %s"%(label,key,value))
+        
+                # custom columns
+                for k, column in self.gui.library_view.model().custom_columns.iteritems():
+                    if k != prefs['savemetacol']:
+                        key='calibre_cust_'+k[1:]
+                        label=column['name']
+                        value=db.get_custom(book['calibre_id'],
+                                            label=column['label'],
+                                            index_is_id=True)
+                        # custom always have name.
+                        if value is None or not book['calibre_id']:
+                            ## if existing book, populate existing calibre column
+                            ## values in metadata, else '' to hide.
+                            value=''
+                        book['calibre_columns'][key]={'val':value,'label':label}
+                        # logger.debug("%s(%s): %s"%(label,key,value))
+            
             # if still 'good', make a temp file to write the output to.
             # For HTML format users, make the filename inside the zip something reasonable.
             # For crazy long titles/authors, limit it to 200chars.
@@ -1151,17 +1261,6 @@ class FanFicFarePlugin(InterfaceAction):
             logger.debug("outfile:"+tmp.name)
             book['outfile'] = tmp.name
 
-            # cookiejar = PersistentTemporaryFile(prefix=story.formatFileName("${title}-${author}-",allowunsafefilename=False)[:100],
-            #                                     suffix='.cookiejar',
-            #                                     dir=options['tdir'])
-            # adapter.save_cookiejar(cookiejar.name)
-            # book['cookiejar'] = cookiejar.name
-            # pagecache = PersistentTemporaryFile(prefix=story.formatFileName("${title}-${author}-",allowunsafefilename=False)[:100],
-            #                                     suffix='.pagecache',
-            #                                     dir=options['tdir'])
-            # adapter.save_pagecache(pagecache.name)
-            # book['pagecache'] = pagecache.name
-
         return
 
     def start_download_job(self,book_list,
@@ -1178,7 +1277,7 @@ class FanFicFarePlugin(InterfaceAction):
 
         ## No need to BG process when CALIBREONLY!  Fake it.
         #print("options:%s"%options)
-        if options['collision'] == CALIBREONLY:
+        if options['collision'] in (CALIBREONLY, CALIBREONLYSAVECOL):
             class NotJob(object):
                 def __init__(self,result):
                     self.failed=False
@@ -1263,10 +1362,10 @@ class FanFicFarePlugin(InterfaceAction):
         logger.debug("add/update %s %s"%(book['title'],book['url']))
         mi = self.make_mi_from_book(book)
 
-        if options['collision'] != CALIBREONLY:
+        if options['collision'] not in (CALIBREONLY, CALIBREONLYSAVECOL):
             self.add_book_or_update_format(book,options,prefs,mi)
 
-        if options['collision'] == CALIBREONLY or \
+        if options['collision'] in (CALIBREONLY, CALIBREONLYSAVECOL) or \
                 ( (options['updatemeta'] or book['added']) and book['good'] ):
             try:
                 self.update_metadata(db, book['calibre_id'], book, mi, options)
@@ -1293,7 +1392,7 @@ class FanFicFarePlugin(InterfaceAction):
         failed_list = filter(lambda x : not x['good'] , book_list)
         failed_ids = [ x['calibre_id'] for x in failed_list ]
 
-        if options['collision'] != CALIBREONLY and \
+        if options['collision'] not in (CALIBREONLY, CALIBREONLYSAVECOL) and \
                 (prefs['addtolists'] or prefs['addtoreadlists']):
             self.update_reading_lists(all_ids,add=True)
 
@@ -1357,7 +1456,7 @@ class FanFicFarePlugin(InterfaceAction):
             if countpagesstats:
                 cp_plugin.count_statistics(all_ids,countpagesstats)
 
-        if prefs['autoconvert'] and options['collision'] != CALIBREONLY:
+        if prefs['autoconvert'] and options['collision'] not in (CALIBREONLY, CALIBREONLYSAVECOL):
             self.gui.status_bar.show_message(_('Starting auto conversion of %d books.')%(len(all_ids)), 3000)
             self.gui.iactions['Convert Books'].auto_convert_auto_add(all_ids)
 
@@ -1639,6 +1738,11 @@ class FanFicFarePlugin(InterfaceAction):
         #print("all_metadata: %s"%book['all_metadata'])
         custom_columns = self.gui.library_view.model().custom_columns
 
+        # save metadata to configured column
+        if 'savemetacol' in book and prefs['savemetacol'] != '' and prefs['savemetacol'] in custom_columns:
+            label = custom_columns[prefs['savemetacol']]['label']
+            self.set_custom(db, book_id, 'comment', book['savemetacol'], label=label, commit=True)
+
         #print("prefs['custom_cols'] %s"%prefs['custom_cols'])
         for col, meta in prefs['custom_cols'].iteritems():
             #print("setting %s to %s"%(col,meta))
@@ -1739,7 +1843,7 @@ class FanFicFarePlugin(InterfaceAction):
 
         # set author link if found.  All current adapters have authorUrl, except anonymous on AO3.
         # Moved down so author's already in the DB.
-        if 'authorUrl' in book['all_metadata']:
+        if 'authorUrl' in book['all_metadata'] and prefs['set_author_url']:
             authurls = book['all_metadata']['authorUrl'].split(", ")
             authorlist = [ a.replace('&',';') for a in book['author'] ]
             authorids = db.new_api.get_item_ids('authors',authorlist)
@@ -1763,7 +1867,7 @@ class FanFicFarePlugin(InterfaceAction):
             or (prefs['updatecalcover'] == SAVE_YES_IF_IMG ## yes, if image.
                 and book['all_metadata']['cover_image'] )): # in ('specific','first','default','old')
             existingepub = db.format(book_id,'EPUB',index_is_id=True, as_file=True)
-            epubmi = get_metadata(existingepub,'EPUB')
+            epubmi = calibre_get_metadata(existingepub,'EPUB')
             if epubmi.cover_data[1] is not None:
                 try:
                     db.set_cover(book_id, epubmi.cover_data[1])
@@ -2049,7 +2153,7 @@ class FanFicFarePlugin(InterfaceAction):
             existingepub = None
             if path == None and db.has_format(book_id,'EPUB',index_is_id=True):
                 existingepub = db.format(book_id,'EPUB',index_is_id=True, as_file=True)
-                mi = get_metadata(existingepub,'EPUB')
+                mi = calibre_get_metadata(existingepub,'EPUB')
                 identifiers = mi.get_identifiers()
                 if 'url' in identifiers:
                     # print("url from get_metadata:%s"%identifiers['url'].replace('|',':'))
@@ -2203,6 +2307,12 @@ class FanFicFarePlugin(InterfaceAction):
 
         return book
 
+    def busy_cursor(self):
+        QApplication.setOverrideCursor(QCursor(Qt.WaitCursor))
+
+    def restore_cursor(self):
+        QApplication.restoreOverrideCursor()
+    
 def split_text_to_urls(urls):
     # remove dups while preserving order.
     dups=set()

calibre-plugin/jobs.py

@@ -11,6 +11,7 @@ import logging
 logger = logging.getLogger(__name__)
 
 import time, traceback
+from StringIO import StringIO
     
 from calibre.utils.ipc.server import Server
 from calibre.utils.ipc.job import ParallelJob
@@ -99,7 +100,7 @@ def do_download_for_worker(book,options,notification=lambda x,y:x):
     with fffbase:
         
         from calibre_plugins.fanficfare_plugin.dialogs import (NotGoingToDownload,
-                OVERWRITE, OVERWRITEALWAYS, UPDATE, UPDATEALWAYS, ADDNEW, SKIP, CALIBREONLY)
+                OVERWRITE, OVERWRITEALWAYS, UPDATE, UPDATEALWAYS, ADDNEW, SKIP, CALIBREONLY, CALIBREONLYSAVECOL)
         from calibre_plugins.fanficfare_plugin.fanficfare import adapters, writers, exceptions
         from calibre_plugins.fanficfare_plugin.fanficfare.epubutils import get_update_data
         
@@ -151,10 +152,12 @@ def do_download_for_worker(book,options,notification=lambda x,y:x):
             outfile = book['outfile']
     
             ## No need to download at all.  Shouldn't ever get down here.
-            if options['collision'] in (CALIBREONLY):
+            if options['collision'] in (CALIBREONLY, CALIBREONLYSAVECOL):
                 logger.info("Skipping CALIBREONLY 'update' down inside worker--this shouldn't be happening...")
                 book['comment'] = 'Metadata collected.'
                 book['all_metadata'] = story.getAllMetadata(removeallentities=True)
+                if options['savemetacol'] != '':
+                    book['savemetacol'] = story.dump_html_metadata()
                 
             ## checks were done earlier, it's new or not dup or newer--just write it.
             elif options['collision'] in (ADDNEW, SKIP, OVERWRITE, OVERWRITEALWAYS) or \
@@ -162,24 +165,20 @@ def do_download_for_worker(book,options,notification=lambda x,y:x):
     
                 # preserve logfile even on overwrite.
                 if 'epub_for_update' in book:
-                    (urlignore,
-                     chaptercountignore,
-                     oldchaptersignore,
-                     oldimgsignore,
-                     oldcoverignore,
-                     calibrebookmarkignore,
-                     # only logfile set in adapter, so others aren't used.
-                     adapter.logfile) = get_update_data(book['epub_for_update'])
-    
+                    
+                    adapter.logfile = get_update_data(book['epub_for_update'])[6]
                     # change the existing entries id to notid so
                     # write_epub writes a whole new set to indicate overwrite.
                     if adapter.logfile:
                         adapter.logfile = adapter.logfile.replace("span id","span notid")
                 
                 logger.info("write to %s"%outfile)
+                inject_cal_cols(book,story,configuration)
                 writer.writeStory(outfilename=outfile, forceOverwrite=True)
                 book['comment'] = 'Download %s completed, %s chapters.'%(options['fileform'],story.getMetadata("numChapters"))
                 book['all_metadata'] = story.getAllMetadata(removeallentities=True)
+                if options['savemetacol'] != '':
+                    book['savemetacol'] = story.dump_html_metadata()
                 
             ## checks were done earlier, just update it.
             elif 'epub_for_update' in book and options['collision'] in (UPDATE, UPDATEALWAYS):
@@ -193,13 +192,15 @@ def do_download_for_worker(book,options,notification=lambda x,y:x):
                  adapter.oldimgs,
                  adapter.oldcover,
                  adapter.calibrebookmark,
-                 adapter.logfile) = get_update_data(book['epub_for_update'])
+                 adapter.logfile) = get_update_data(book['epub_for_update'])[0:7]
     
                 # dup handling from fff_plugin needed for anthology updates.
                 if options['collision'] == UPDATE:
                     if chaptercount == urlchaptercount:
                         book['comment']=_("Already contains %d chapters.  Reuse as is.")%chaptercount
                         book['all_metadata'] = story.getAllMetadata(removeallentities=True)
+                        if options['savemetacol'] != '':
+                            book['savemetacol'] = story.dump_html_metadata()
                         book['outfile'] = book['epub_for_update'] # for anthology merge ops.
                         return book
     
@@ -214,12 +215,15 @@ def do_download_for_worker(book,options,notification=lambda x,y:x):
                 logger.info("Do update - epub(%d) vs url(%d)" % (chaptercount, urlchaptercount))
                 logger.info("write to %s"%outfile)
     
+                inject_cal_cols(book,story,configuration)
                 writer.writeStory(outfilename=outfile, forceOverwrite=True)
                 
                 book['comment'] = _('Update %s completed, added %s chapters for %s total.')%\
                     (options['fileform'],(urlchaptercount-chaptercount),urlchaptercount)
                 book['all_metadata'] = story.getAllMetadata(removeallentities=True)
-            
+                if options['savemetacol'] != '':
+                    book['savemetacol'] = story.dump_html_metadata()
+
             if options['smarten_punctuation'] and options['fileform'] == "epub" \
                     and calibre_version >= (0, 9, 39):
                 # for smarten punc
@@ -253,3 +257,22 @@ def do_download_for_worker(book,options,notification=lambda x,y:x):
             
         #time.sleep(10)
     return book
+
+## calibre's columns for an existing book are pased in and injected
+## into the story's metadata.  For convenience, we also add labels and
+## valid_entries for them in a special [injected] section that has
+## even less precedence than [defaults]
+def inject_cal_cols(book,story,configuration):
+    configuration.remove_section('injected')
+    if 'calibre_columns' in book:
+        injectini = ['[injected]']
+        extra_valid = []
+        for k, v in book['calibre_columns'].iteritems():
+            story.setMetadata(k,v['val'])
+            injectini.append('%s_label:%s'%(k,v['label']))
+            extra_valid.append(k)
+        if extra_valid: # if empty, there's nothing to add.
+            injectini.append("add_to_extra_valid_entries:,"+','.join(extra_valid))
+            configuration.readfp(StringIO('\n'.join(injectini)))
+            #print("added:\n%s\n"%('\n'.join(injectini)))
+    

calibre-plugin/plugin-defaults.ini

@@ -330,6 +330,11 @@ sort_ships:false
 ## User-agent
 user_agent:FFF/2.X
 
+## Virtually all eFiction Base adapters allow downloading the whole story in
+## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
+## metadata and chapters can be loaded in one step
+bulk_load:true
+
 ## Each output format has a section that overrides [defaults]
 [html]
 
@@ -593,6 +598,18 @@ hits_label:Hits
 collections_label:Collections
 bookmarks_label:Bookmarks
 
+## AO3 doesn't have anything it calls 'genre'.  The adapter used to be
+## hardcoded to include the site specific metadata freeformtags &
+## ao3categories in the standard metadata field genre.  By making it
+## configurable, users can change it.
+include_in_genre: freeformtags, ao3categories
+
+## AO3 uses the word 'category' differently than most sites.  The
+## adapter used to be hardcoded to include the site specific metadata
+## fandom in the standard metadata field category.  By making it
+## configurable, users can change it.
+include_in_category:fandoms
+
 ## freeformtags was previously typo'ed as freefromtags.  This way,
 ## freefromtags will still work for people who've used it.
 include_in_freefromtags:freeformtags
@@ -604,7 +621,7 @@ include_in_freefromtags:freeformtags
 #extra_subject_tags:fandoms,freeformtags,ao3categories
 
 ## AO3 chapters can include several different types of notes.  We've
-## traditional included them all in the chapter text, but this allows
+## traditionally included them all in the chapter text, but this allows
 ## you to customize which you include.  Copy this parameter to your
 ## personal.ini and list the ones you don't want.
 #exclude_notes:authorheadnotes,chaptersummary,chapterheadnotes,chapterfootnotes,authorfootnotes
@@ -754,11 +771,6 @@ extraships:Spike/Buffy
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
-## Virtually all eFiction Base adapters allow downloading the whole story in
-## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
-## metadata and chapters can be loaded in one step
-bulk_load:true
-
 [dramione.org]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -939,11 +951,6 @@ extraships:Harry Potter/Hermione Granger
 #username:YourName
 #password:yourpassword
 
-## Virtually all eFiction Base adapters allow downloading the whole story in
-## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
-## metadata and chapters can be loaded in one step
-bulk_load:true
-
 extra_valid_entries: readings,romance
 extra_titlepage_entries: readings,romance
 readings_label: Readings
@@ -1091,11 +1098,6 @@ extracategories:Glee RPF
 extracharacters:Darren Criss, Chris Colfer
 extraships:Darren Criss/Chris Colfer
 
-## Virtually all eFiction Base adapters allow downloading the whole story in
-## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
-## metadata and chapters can be loaded in one step
-bulk_load:true
-
 [ksarchive.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:Star Trek
@@ -1115,11 +1117,6 @@ eroticatags_label:Erotica Tags
 extra_titlepage_entries: eroticatags
 
 [lotrfanfiction.com]
-## Virtually all eFiction Base adapters allow downloading the whole story in
-## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
-## metadata and chapters can be loaded in one step
-bulk_load:true
-
 extra_valid_entries: readings
 readings_label: Readings
 
@@ -1369,11 +1366,6 @@ extracategories:Transgender
 ## confirm they are adult for adult content.
 #is_adult:true
 
-## Virtually all eFiction Base adapters allow downloading the whole story in
-## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
-## metadata and chapters can be loaded in one step
-bulk_load:true
-
 [thehexfiles.net]
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter
@@ -1393,11 +1385,6 @@ extraships:Harry Potter/Draco Malfoy
 ## personal.ini, not defaults.ini.
 #is_adult:true
 
-## Virtually all eFiction Base adapters allow downloading the whole story in
-## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
-## metadata and chapters can be loaded in one step
-bulk_load:true
-
 ## Site dedicated to these categories/characters/ships
 extracategories:Criminal Minds
 
@@ -1407,11 +1394,6 @@ extracategories:Criminal Minds
 ## personal.ini, not defaults.ini.
 #is_adult:true
 
-## Virtually all eFiction Base adapters allow downloading the whole story in
-## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
-## metadata and chapters can be loaded in one step
-bulk_load:true
-
 extra_valid_entries: readings,challenge
 extra_titlepage_entries: readings,challenge
 challenge_label: Challenge
@@ -1765,11 +1747,6 @@ extraships:InuYasha/Kagome
 ## Site dedicated to these categories/characters/ships
 extracategories:Lord of the Rings
 
-## Virtually all eFiction Base adapters allow downloading the whole story in
-## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
-## metadata and chapters can be loaded in one step
-bulk_load:true
-
 [www.mediaminer.org]
 
 [www.midnightwhispers.ca]

calibre-plugin/prefs.py

@@ -51,6 +51,7 @@ PREFS_KEY_SETTINGS = 'settings'
 # take from here.
 default_prefs = {}
 default_prefs['personal.ini'] = get_resources('plugin-example.ini')
+default_prefs['cal_cols_pass_in'] = False
 default_prefs['rejecturls'] = ''
 default_prefs['rejectreasons'] = '''Sucked
 Boring
@@ -75,6 +76,7 @@ default_prefs['lookforurlinhtml'] = False
 default_prefs['checkforseriesurlid'] = True
 default_prefs['checkforurlchange'] = True
 default_prefs['injectseries'] = False
+default_prefs['matchtitleauth'] = True
 default_prefs['smarten_punctuation'] = False
 default_prefs['show_est_time'] = False
 
@@ -98,11 +100,13 @@ default_prefs['countpagesstats'] = []
 default_prefs['wordcountmissing'] = False
 
 default_prefs['errorcol'] = ''
+default_prefs['savemetacol'] = ''
 default_prefs['custom_cols'] = {}
 default_prefs['custom_cols_newonly'] = {}
 default_prefs['allow_custcol_from_ini'] = True
 
 default_prefs['std_cols_newonly'] = {}
+default_prefs['set_author_url'] = True
 
 default_prefs['imapserver'] = ''
 default_prefs['imapuser'] = ''
@@ -110,6 +114,7 @@ default_prefs['imappass'] = ''
 default_prefs['imapsessionpass'] = False
 default_prefs['imapfolder'] = 'INBOX'
 default_prefs['imapmarkread'] = True
+default_prefs['auto_reject_from_email'] = False
 
 def set_library_config(library_config,db):
     db.prefs.set_namespaced(PREFS_NAMESPACE,

fanficfare/adapters/adapter_archiveofourownorg.py

@@ -230,7 +230,6 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
         fandoms = a.findAll('a',{'class':"tag"})
         for fandom in fandoms:
             self.story.addToList('fandoms',fandom.string)
-            self.story.addToList('category',fandom.string)
 		
         a = metasoup.find('dd',{'class':"warning tags"})
         if a != None:
@@ -243,7 +242,6 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
             genres = a.findAll('a',{'class':"tag"})
             for genre in genres:
                 self.story.addToList('freeformtags',genre.string)
-                self.story.addToList('genre',genre.string)
                 
         a = metasoup.find('dd',{'class':"category tags"})
         if a != None:
@@ -251,7 +249,6 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
             for genre in genres:
                 if genre != "Gen":
                     self.story.addToList('ao3categories',genre.string)
-                    self.story.addToList('genre',genre.string)
 		
         a = metasoup.find('dd',{'class':"character tags"})
         if a != None:

fanficfare/adapters/adapter_ficwadcom.py

@@ -135,7 +135,7 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
         self.story.setMetadata('title', stripHTML(titleh4.a))
 
         # Find authorid and URL from... author url.
-        a = soup.find('span',{'class':'author'}).find('a', href=re.compile(r"^/author/\d+"))
+        a = soup.find('span',{'class':'author'}).find('a', href=re.compile(r"^/a/"))
         self.story.setMetadata('authorId',a['href'].split('/')[2])
         self.story.setMetadata('authorUrl','http://'+self.host+a['href'])
         self.story.setMetadata('author',a.string)

fanficfare/adapters/adapter_grangerenchantedcom.py

@@ -176,7 +176,7 @@ class GrangerEnchantedCom(BaseSiteAdapter):
             raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
             
         # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
+        soup = self.make_soup(data)
         # print data
 
         # Now go hunting for all the meta data and the chapter list.
@@ -217,11 +217,10 @@ class GrangerEnchantedCom(BaseSiteAdapter):
             if 'Summary' in label:
                 ## Everything until the next span class='label'
                 svalue = ""
-                while not defaultGetattr(value,'class') == 'label':
-                    svalue += str(value)
+                while value and not defaultGetattr(value,'class') == 'label' and '<span class="label">' not in unicode(value):
+                    svalue += unicode(value)
                     value = value.nextSibling
                 self.setDescription(url,svalue)
-                #self.story.setMetadata('description',stripHTML(svalue))
 
             if 'Rated' in label:
                 self.story.setMetadata('rating', value)
@@ -271,7 +270,7 @@ class GrangerEnchantedCom(BaseSiteAdapter):
             series_url = 'http://'+self.host+'/'+self.section+'/'+a['href']
 
             # use BeautifulSoup HTML parser to make everything easier to find.
-            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
+            seriessoup = self.make_soup(self._fetchUrl(series_url))
             # can't use ^viewstory...$ in case of higher rated stories with javascript href.
             storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
             i=1
@@ -300,8 +299,7 @@ class GrangerEnchantedCom(BaseSiteAdapter):
 
         logger.debug('Getting chapter text from: %s' % url)
 
-        soup = bs.BeautifulSoup(self._fetchUrl(url),
-                                selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        soup = self.make_soup(self._fetchUrl(url))
         
         div = soup.find('div', {'id' : 'story1'})
 

fanficfare/adapters/adapter_mediaminerorg.py

@@ -22,7 +22,6 @@ import re
 import urllib
 import urllib2
 
-from .. import BeautifulSoup as bs
 from ..htmlcleanup import stripHTML
 from .. import exceptions as exceptions
 
@@ -42,7 +41,12 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
         # get storyId from url--url validation guarantees query correct
         m = re.match(self.getSiteURLPattern(),url)
         if m:
-            self.story.setMetadata('storyId',m.group('id'))
+            if m.group('id'):
+                self.story.setMetadata('storyId',m.group('id'))
+            elif m.group('id2'):
+                self.story.setMetadata('storyId',m.group('id2'))
+            elif m.group('id3'):
+                self.story.setMetadata('storyId',m.group('id2'))
             
             # normalized story URL.
             self._setURL('http://' + self.getSiteDomain() + '/fanfic/view_st.php/'+self.story.getMetadata('storyId'))
@@ -62,8 +66,17 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
     def getSiteURLPattern(self):
         ##  http://www.mediaminer.org/fanfic/view_st.php/76882
         ##  http://www.mediaminer.org/fanfic/view_ch.php/167618/594087#fic_c
+        ##  http://www.mediaminer.org/fanfic/view_ch.php?submit=View+Chapter&id=105816&cid=357151
+        ##  http://www.mediaminer.org/fanfic/view_ch.php?cid=612153&submit=View+Chapter&id=171668
         return re.escape("http://"+self.getSiteDomain())+\
-            "/fanfic/view_(st|ch)\.php/"+r"(?P<id>\d+)(/\d+(#fic_c)?)?$"
+            r"/fanfic/view_(st|ch)\.php"+\
+            r"(/(?P<id>\d+)(/\d+(#fic_c)?)?/?|"+\
+            r"\?((submit=View(\+| )Chapter|id=(?P<id2>\d+)|cid=\d+)&?)+)"
+
+    # Override stripURLParameters so the id parameter won't get stripped
+    @classmethod
+    def stripURLParameters(cls, url):
+        return url
 
     def extractChapterUrlsAndMetadata(self):
 
@@ -71,7 +84,7 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
         logger.debug("URL: "+url)
 
         try:
-            data = self._fetchUrl(url)
+            data = self._fetchUrl(url+'/') # trailing / gets 'chapter list' page even for one-shots.
         except urllib2.HTTPError, e:
             if e.code == 404:
                 raise exceptions.StoryDoesNotExist(self.url)
@@ -79,7 +92,7 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
                 raise e
 
         # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
+        soup = self.make_soup(data)
 
         # [ A - All Readers ], strip '[' ']'
         ## Above title because we remove the smtxt font to get title.
@@ -106,18 +119,12 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
         title = soup.find('td',{'class':'ffh'})
         for font in title.findAll('font'):
             font.extract() # removes 'font' tags from inside the td.        
-        if title.has_key('colspan'):
+        if title.has_attr('colspan'):
             titlet = stripHTML(title)
         else:
             ## No colspan, it's part chapter title--even if it's a one-shot.
             titlet = ':'.join(stripHTML(title).split(':')[:-1]) # strip trailing 'Chapter X' or chapter title
         self.story.setMetadata('title',titlet)
-        ## The story title is difficult to reliably parse from the
-        ## story pages.  Getting it from the author page is, but costs
-        ## another fetch.
-        # authsoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
-        # titlea = authsoup.find('a',{'href':'/fanfic/view_st.php/'+self.story.getMetadata('storyId')})
-        # self.story.setMetadata('title',titlea.text)
 
         # save date from first for later.
         firstdate=None
@@ -137,7 +144,9 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
                 # save date from first for later.
                 if not firstdate:
                     firstdate = m.group(3)
-                self.chapterUrls.append((chapter,'http://'+self.host+'/fanfic/view_ch.php/'+self.story.getMetadata('storyId')+'/'+option['value']))
+                # http://www.mediaminer.org/fanfic/view_ch.php?cid=376587&submit=View+Chapter&id=105816
+                # self.chapterUrls.append((chapter,'http://'+self.host+'/fanfic/view_ch.php/'+self.story.getMetadata('storyId')+'/'+option['value']))
+                self.chapterUrls.append((chapter,'http://'+self.host+'/fanfic/view_ch.php?submit=View Chapter&id='+self.story.getMetadata('storyId')+'&cid='+option['value']))
         self.story.setMetadata('numChapters',len(self.chapterUrls))
 
         # category
@@ -193,38 +202,37 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
         logger.debug('Getting chapter text from: %s' % url)
 
         data=self._fetchUrl(url)
-        soup = bs.BeautifulStoneSoup(data,
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        soup = self.make_soup(data)
 
-        anchor = soup.find('a',{'name':'fic_c'})
+        header = soup.find('div',{'class':'post-meta clearfix '})
+        # print("data:%s"%data)
 
-        if None == anchor:
+        chapter=self.make_soup('<div class="story"></div>').find('div')
+
+        if None == header:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
 
         ## find divs with align=left, those are paragraphs in newer stories.
-        divlist = anchor.findAllNext('div',{'align':'left'})
+        divlist = header.findAllNext('div',{'align':'left'})
         if divlist:
             for div in divlist:
                 div.name='p' # convert to <p> mediaminer uses div with
                              # a margin for paragraphs.
-                anchor.append(div) # cheat!  stuff all the content
-                                   # divs into anchor just as a
-                                   # holder.
+                chapter.append(div)
                 del div['style']
                 del div['align']
-            anchor.name='div'
-            return self.utf8FromSoup(url,anchor)
+            return self.utf8FromSoup(url,chapter)
         
         else:
             logger.debug('Using kludgey text find for older mediaminer story.')
             ## Some older mediaminer stories are unparsable with BeautifulSoup.
             ## Really nasty formatting.  Sooo... Cheat!  Parse it ourselves a bit first.
             ## Story stuff falls between:
-            data = "<div id='HERE'>" + data[data.find('<a name="fic_c">'):] +"</div>"
-            soup = bs.BeautifulStoneSoup(data,
-                                         selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+            data = "<div id='HERE'>" + data[data.find('<div class="adWrap">'):data.find('<div class="addthis_sharing_toolbox">')] +"</div>"
+            soup = self.make_soup(data)
             for tag in soup.findAll('td',{'class':'ffh'}) + \
                     soup.findAll('div',{'class':'acl'}) + \
+                    soup.findAll('div',{'class':'adWrap'}) + \
                     soup.findAll('div',{'class':'footer smtxt'}) + \
                     soup.findAll('table',{'class':'tbbrdr'}):
                 tag.extract() # remove tag from soup.

fanficfare/adapters/adapter_nhamagicalworldsus.py

@@ -16,222 +16,24 @@
 #
 
 # Software: eFiction
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return NHAMagicalWorldsUsAdapter
+from base_efiction_adapter import BaseEfictionAdapter
 
 # Class name has to be unique.  Our convention is camel case the
 # sitename with Adapter at the end.  www is skipped.
-class NHAMagicalWorldsUsAdapter(BaseSiteAdapter):
+class NHAMagicalWorldsUsAdapter(BaseEfictionAdapter):
 
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        
-		
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','nha')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = " %d/%m/%y"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
+    @staticmethod
     def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
         return 'nha.magical-worlds.us'
 
     @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-        url = self.url
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-            
-        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
-        if m != None:
-            if self.is_adult or self.getConfig("is_adult"):
-                # We tried the default and still got a warning, so
-                # let's pull the warning number from the 'continue'
-                # link and reload data.
-                addurl = m.group(1)
-                # correct stupid & error in url.
-                addurl = addurl.replace("&","&")
-                url = self.url+'&index=1'+addurl
-                logger.debug("URL 2nd try: "+url)
-
-                try:
-                    data = self._fetchUrl(url)
-                except urllib2.HTTPError, e:
-                    if e.code == 404:
-                        raise exceptions.StoryDoesNotExist(self.url)
-                    else:
-                        raise e    
-            else:
-                raise exceptions.AdultCheckRequired(self.url)
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-            
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
-		
-        try:
-            # in case link points somewhere other than the first chapter
-            a = soup.findAll('option')[1]['value']
-            self.story.setMetadata('storyId',a.split('=',)[1])
-            url = 'http://'+self.host+'/'+a
-            soup = bs.BeautifulSoup(self._fetchUrl(url))
-        except:
-            pass
-		
-        for info in asoup.findAll('table', {'width' : '100%', 'bordercolor' : re.compile(r'#')}):
-            a = info.find('a')
-            if 'viewstory.php?sid='+self.story.getMetadata('storyId') == a['href'] or \
-                    ('viewstory.php?sid='+self.story.getMetadata('storyId')+'&') in a['href']:
-                self.story.setMetadata('title',stripHTML(a))
-                break
-		
-
-        # Find the chapters:
-        chapters=soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+'&chapter=\d+$'))
-        if len(chapters) == 0:
-            self.chapterUrls.append((self.story.getMetadata('title'),url))
-        else:
-            for chapter in chapters:
-                # just in case there's tags, like <i> in chapter titles.
-                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d):
-            try:
-                return d.name
-            except:
-                return ""
-				
-        cats = info.findAll('a',href=re.compile('categories.php'))
-        for cat in cats:
-            self.story.addToList('category',cat.string)
-					
-        a = info.find('a', href=re.compile(r'viewuser.php'))
-        val = a.nextSibling
-        svalue = ""
-        while not defaultGetattr(val) == 'br':
-            val = val.nextSibling
-        val = val.nextSibling
-        while not defaultGetattr(val) == 'br':
-            svalue += unicode(val)
-            val = val.nextSibling
-        self.setDescription(url,svalue)
+    def getSiteAbbrev(self):
+        return 'nha'
         
-        #does not provide convenient way to get word count
-        labels = info.findAll('i')
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = stripHTML(labelspan)
-
-            if 'Rating' in label:
-                self.story.setMetadata('rating', value.split(' -')[0])
-
-            if 'Genres' in label:
-                genres = value.string.split(', ')
-                for genre in genres:
-                    if 'None' not in genre:
-                        self.story.addToList('genre',genre.split(' -')[0])
-            
-            if 'Characters' in label:
-                chars = value.string.split(', ')
-                for char in chars:
-                    if 'None' not in char:
-                        self.story.addToList('characters',char.split(' -')[0])
-
-            if 'Warnings' in label:
-                warnings = value.string.split(', ')
-                for warning in warnings:
-                    if 'None' not in warning:
-                        self.story.addToList('warnings',warning.split(' -')[0])
-
-            if 'Completed' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(value.split(' -')[0], self.dateformat))
-            
-            if 'Updated' in label:
-                # there's a stray [ at the end.
-                #value = value[0:-1]
-                self.story.setMetadata('dateUpdated', makeDate(value.split(' -')[0], self.dateformat))
-
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-		
-        data = self._fetchUrl(url)
-
-        soup = bs.BeautifulSoup(data, selfClosingTags=('br','hr','span','center')) # some chapters seem to be hanging up on those tags, so it is safer to close them
-        
-        story = soup.find('div', {"id" : "story"})
-
-        if None == story:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    @classmethod
+    def getDateFormat(self):
+        return "%d/%m/%y" 
+                           
+def getClass():
+    return NHAMagicalWorldsUsAdapter
     
-        return self.utf8FromSoup(url,story)

fanficfare/adapters/adapter_ponyfictionarchivenet.py

@@ -171,7 +171,8 @@ class PonyFictionArchiveNetAdapter(BaseSiteAdapter):
             self.story.addToList('warnings',warning.string)
         
         status = soup.find('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
-        self.story.setMetadata('status',status.string)
+        if status: # apparently this site can have stories with neither In-Progress or Complete.
+            self.story.setMetadata('status',status.string)
         
         section = soup.findAll('span', {'class' : 'General'})[1]            
         

fanficfare/adapters/adapter_storiesonlinenet.py

@@ -114,7 +114,7 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
         return True
 
     ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
+    def doExtractChapterUrlsAndMetadata(self, get_cover=True):
 
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
@@ -259,7 +259,7 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
                 logger.debug("Retrieving Universe - have page")
                 if universe_soup:
                     logger.debug("Retrieving Universe - looking for name")
-                    universe_name = universe_soup.find('h1', {'id' : 'ptitle'}).text.partition(' —')[0]
+                    universe_name = universe_soup.find('h1', {'id' : 'ptitle'}).text.partition('—')[0]
                     logger.debug("Universes name: '{0}'".format(universe_name))
 
                 self.story.setMetadata('universeUrl',universeUrl)
@@ -301,7 +301,21 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
 
             if 'Updated' in label:
                 self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-#
+
+        # Some books have a cover in the index page.
+        # Samples are:
+        #     http://storiesonline.net/s/11999
+        #     http://storiesonline.net/s/10823
+        if get_cover:
+            logger.debug("Looking for the cover image...")
+            cover_url = ""
+            img = soup.find('img')
+            if img:
+                cover_url=img['src']
+            logger.debug("cover_url: %s"%cover_url)
+            if cover_url:
+                self.setCoverImage(url,cover_url)
+
         status = lc4.find('span', {'class' : 'ab'})
         if  status != None:
             self.story.setMetadata('status', 'In-Progress')

fanficfare/adapters/base_adapter.py

@@ -404,6 +404,13 @@ class BaseSiteAdapter(Configurable):
             self.metadataDone = True
         return self.story
 
+    def setStoryMetadata(self,metahtml):
+        if metahtml:
+            self.story.load_html_metadata(metahtml)
+            self.metadataDone = True
+            if not self.story.getMetadataRaw('dateUpdated'):
+                self.story.setMetadata('dateUpdated',self.story.getMetadataRaw('datePublished'))
+    
     def hookForUpdates(self,chaptercount):
         "Usually not needed."
         return chaptercount

fanficfare/adapters/base_efiction_adapter.py

@@ -264,7 +264,6 @@ class BaseEfictionAdapter(BaseSiteAdapter):
                 else:
                     super(NameOfMyAdapter, self).handleMetadata(key, value)
         """
-        # logger.debug("metadata: '%s' == '%s'" % (key, value))
         if value == 'None':
             return
         elif key == 'Summary':
@@ -287,7 +286,7 @@ class BaseEfictionAdapter(BaseSiteAdapter):
                 self.story.addToList('challenge', val)
         elif key == 'Chapters':
             self.story.setMetadata('numChapters', int(value))
-        elif key == 'Rating':
+        elif key == 'Rating' or key == 'Rated':
             self.story.setMetadata('rating', value)
         elif key == 'Word count':
             self.story.setMetadata('numWords', value)

fanficfare/cli.py

@@ -318,7 +318,13 @@ def do_download(arg,
                 # update now handled by pre-populating the old
                 # images and chapters in the adapter rather than
                 # merging epubs.
-                url, chaptercount, adapter.oldchapters, adapter.oldimgs, adapter.oldcover, adapter.calibrebookmark, adapter.logfile = get_update_data(output_filename)
+                (url,
+                 chaptercount,
+                 adapter.oldchapters,
+                 adapter.oldimgs,
+                 adapter.oldcover,
+                 adapter.calibrebookmark,
+                 adapter.logfile) = (get_update_data(output_filename))[0:7]
 
                 print 'Do update - epub(%d) vs url(%d)' % (chaptercount, urlchaptercount)
 

fanficfare/configurable.py

@@ -41,7 +41,38 @@ def re_compile(regex,line):
         return re.compile(regex)
     except Exception, e: 
         raise exceptions.RegularExpresssionFailed(e,regex,line)
-    
+
+# fall back labels.
+titleLabels = {
+    'category':'Category',
+    'genre':'Genre',
+    'language':'Language',
+    'status':'Status',
+    'series':'Series',
+    'characters':'Characters',
+    'ships':'Relationships',
+    'datePublished':'Published',
+    'dateUpdated':'Updated',
+    'dateCreated':'Packaged',
+    'rating':'Rating',
+    'warnings':'Warnings',
+    'numChapters':'Chapters',
+    'numWords':'Words',
+    'site':'Site',
+    'storyId':'Story ID',
+    'authorId':'Author ID',
+    'extratags':'Extra Tags',
+    'title':'Title',
+    'storyUrl':'Story URL',
+    'description':'Summary',
+    'author':'Author',
+    'authorUrl':'Author URL',
+    'formatname':'File Format',
+    'formatext':'File Extension',
+    'siteabbrev':'Site Abbrev',
+    'version':'Downloader Version'
+    }
+
 formatsections = ['html','txt','epub','mobi']
 othersections = ['defaults','overrides']
 
@@ -111,7 +142,8 @@ def get_valid_set_options():
                
                'fix_fimf_blockquotes':(['fimfiction.net'],None,boollist),
                'fail_on_password':(['fimfiction.net'],None,boollist),
-               'do_update_hook':(['fimfiction.net'],None,boollist),
+               'do_update_hook':(['fimfiction.net',
+                                  'archiveofourown.org'],None,boollist),
 
                'force_login':(['phoenixsong.net'],None,boollist),
                'non_breaking_spaces':(['fictionmania.tv'],None,boollist),
@@ -304,7 +336,8 @@ class Configuration(ConfigParser.SafeConfigParser):
 
         self.linenos=dict() # key by section or section,key -> lineno
         
-        self.sectionslist = ['defaults']
+        ## [injected] section has even less priority than [defaults]
+        self.sectionslist = ['defaults','injected']
 
         if site.startswith("www."):
             sitewith = site
@@ -606,3 +639,13 @@ class Configurable(object):
 
     def get_config_list(self, sections, key):
         return self.configuration.get_config_list(sections,key)
+
+    def get_label(self, entry):
+        if self.hasConfig(entry+"_label"):
+            label=self.getConfig(entry+"_label")
+        elif entry in titleLabels:
+            label=titleLabels[entry]
+        else:
+            label=entry.title()
+        return label
+

fanficfare/defaults.ini

@@ -327,6 +327,11 @@ sort_ships:false
 ## User-agent
 user_agent:FFF/2.X
 
+## Virtually all eFiction Base adapters allow downloading the whole story in
+## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
+## metadata and chapters can be loaded in one step
+bulk_load:true
+
 ## Each output format has a section that overrides [defaults]
 [html]
 
@@ -597,6 +602,18 @@ hits_label:Hits
 collections_label:Collections
 bookmarks_label:Bookmarks
 
+## AO3 doesn't have anything it calls 'genre'.  The adapter used to be
+## hardcoded to include the site specific metadata freeformtags &
+## ao3categories in the standard metadata field genre.  By making it
+## configurable, users can change it.
+include_in_genre: freeformtags, ao3categories
+
+## AO3 uses the word 'category' differently than most sites.  The
+## adapter used to be hardcoded to include the site specific metadata
+## fandom in the standard metadata field category.  By making it
+## configurable, users can change it.
+include_in_category:fandoms
+
 ## freeformtags was previously typo'ed as freefromtags.  This way,
 ## freefromtags will still work for people who've used it.
 include_in_freefromtags:freeformtags
@@ -608,7 +625,7 @@ include_in_freefromtags:freeformtags
 #extra_subject_tags:fandoms,freeformtags,ao3categories
 
 ## AO3 chapters can include several different types of notes.  We've
-## traditional included them all in the chapter text, but this allows
+## traditionally included them all in the chapter text, but this allows
 ## you to customize which you include.  Copy this parameter to your
 ## personal.ini and list the ones you don't want.
 #exclude_notes:authorheadnotes,chaptersummary,chapterheadnotes,chapterfootnotes,authorfootnotes
@@ -758,11 +775,6 @@ extraships:Spike/Buffy
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
-## Virtually all eFiction Base adapters allow downloading the whole story in
-## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
-## metadata and chapters can be loaded in one step
-bulk_load:true
-
 [dramione.org]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -925,11 +937,6 @@ extraships:Harry Potter/Hermione Granger
 #username:YourName
 #password:yourpassword
 
-## Virtually all eFiction Base adapters allow downloading the whole story in
-## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
-## metadata and chapters can be loaded in one step
-bulk_load:true
-
 extra_valid_entries: readings,romance
 extra_titlepage_entries: readings,romance
 readings_label: Readings
@@ -1077,11 +1084,6 @@ extracategories:Glee RPF
 extracharacters:Darren Criss, Chris Colfer
 extraships:Darren Criss/Chris Colfer
 
-## Virtually all eFiction Base adapters allow downloading the whole story in
-## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
-## metadata and chapters can be loaded in one step
-bulk_load:true
-
 [ksarchive.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:Star Trek
@@ -1101,11 +1103,6 @@ eroticatags_label:Erotica Tags
 extra_titlepage_entries: eroticatags
 
 [lotrfanfiction.com]
-## Virtually all eFiction Base adapters allow downloading the whole story in
-## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
-## metadata and chapters can be loaded in one step
-bulk_load:true
-
 extra_valid_entries: readings
 readings_label: Readings
 
@@ -1355,11 +1352,6 @@ extracategories:Transgender
 ## confirm they are adult for adult content.
 #is_adult:true
 
-## Virtually all eFiction Base adapters allow downloading the whole story in
-## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
-## metadata and chapters can be loaded in one step
-bulk_load:true
-
 [thehexfiles.net]
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter
@@ -1379,11 +1371,6 @@ extraships:Harry Potter/Draco Malfoy
 ## personal.ini, not defaults.ini.
 #is_adult:true
 
-## Virtually all eFiction Base adapters allow downloading the whole story in
-## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
-## metadata and chapters can be loaded in one step
-bulk_load:true
-
 ## Site dedicated to these categories/characters/ships
 extracategories:Criminal Minds
 
@@ -1393,11 +1380,6 @@ extracategories:Criminal Minds
 ## personal.ini, not defaults.ini.
 #is_adult:true
 
-## Virtually all eFiction Base adapters allow downloading the whole story in
-## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
-## metadata and chapters can be loaded in one step
-bulk_load:true
-
 extra_valid_entries: readings,challenge
 extra_titlepage_entries: readings,challenge
 challenge_label: Challenge
@@ -1466,6 +1448,8 @@ extra_titlepage_entries:readings,awards
 awards_label:Awards
 readings_label:Readings
 
+## some sites include images that we don't ever want becoming the
+## cover image.  This lets you exclude them.
 cover_exclusion_regexp:art/.*Awards.jpg
 
 [voracity2.e-fic.com]
@@ -1743,11 +1727,6 @@ extraships:InuYasha/Kagome
 ## Site dedicated to these categories/characters/ships
 extracategories:Lord of the Rings
 
-## Virtually all eFiction Base adapters allow downloading the whole story in
-## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
-## metadata and chapters can be loaded in one step
-bulk_load:true
-
 [www.mediaminer.org]
 
 [www.midnightwhispers.ca]

fanficfare/geturls.py

@@ -15,12 +15,15 @@
 # limitations under the License.
 #
 
-import re
-import urlparse
-import urllib2 as u2
-
-import imaplib
+import collections
 import email
+import imaplib
+import re
+import urllib2 as u2
+import urlparse
+
+import logging
+logger = logging.getLogger(__name__)
 
 from BeautifulSoup import BeautifulSoup 
 from gziphttp import GZipProcessor
@@ -74,28 +77,25 @@ def get_urls_from_page(url,configuration=None,normalize=False):
     return get_urls_from_html(data,url,configuration,normalize,restrictsearch)
 
 def get_urls_from_html(data,url=None,configuration=None,normalize=False,restrictsearch=None):
+    urls = collections.OrderedDict()
 
-    normalized = [] # normalized url
-    retlist = [] # orig urls.
-    
     if not configuration:
         configuration = Configuration("test1.com","EPUB")
 
     soup = BeautifulSoup(data)
     if restrictsearch:
         soup = soup.find(*restrictsearch)
-        #print("restrict search:%s"%soup)
+        #logger.debug("restrict search:%s"%soup)
     
     for a in soup.findAll('a'):
         if a.has_key('href'):
-            #print("a['href']:%s"%a['href'])
+            #logger.debug("a['href']:%s"%a['href'])
             href = form_url(url,a['href'])
-            #print("1 urlhref:%s"%href)
+            #logger.debug("1 urlhref:%s"%href)
             # this (should) catch normal story links, some javascript
             # 'are you old enough' links, and 'Report This' links.
-            # The 'normalized' set prevents duplicates.
             if 'story.php' in a['href']:
-                #print("trying:%s"%a['href'])
+                #logger.debug("trying:%s"%a['href'])
                 m = re.search(r"(?P<sid>(view)?story\.php\?(sid|psid|no|story|stid)=\d+)",a['href'])
                 if m != None:
                     href = form_url(a['href'] if '//' in a['href'] else url,
@@ -103,34 +103,32 @@ def get_urls_from_html(data,url=None,configuration=None,normalize=False,restrict
                     
             try:
                 href = href.replace('&index=1','')
-                #print("2 urlhref:%s"%href)
+                #logger.debug("2 urlhref:%s"%href)
                 adapter = adapters.getAdapter(configuration,href)
-                #print("found adapter")
-                if adapter.story.getMetadata('storyUrl') not in normalized:
-                    normalized.append(adapter.story.getMetadata('storyUrl'))
-                    retlist.append(href)
+                #logger.debug("found adapter")
+                if adapter.story.getMetadata('storyUrl') not in urls:
+                    urls[adapter.story.getMetadata('storyUrl')] = [href]
+                else:
+                    urls[adapter.story.getMetadata('storyUrl')].append(href)
             except Exception, e:
-                #print e
+                #logger.debug e
                 pass
 
-    if normalize:
-        return normalized
-    else:
-        return retlist
+    # Simply return the longest URL with the assumption that it contains the
+    # most user readable metadata, if not normalized
+    return urls.keys() if normalize else [max(value, key=len) for key, value in urls.items()]
+
 
 def get_urls_from_text(data,configuration=None,normalize=False):
-
-    normalized = [] # normalized url
-    retlist = [] # orig urls.
+    urls = collections.OrderedDict()
     data=unicode(data)
-    
+
     if not configuration:
         configuration = Configuration("test1.com","EPUB")
     
     for href in re.findall('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+', data):
         # this (should) catch normal story links, some javascript
         # 'are you old enough' links, and 'Report This' links.
-        # The 'normalized' set prevents duplicates.
         if 'story.php' in href:
             m = re.search(r"(?P<sid>(view)?story\.php\?(sid|psid|no|story|stid)=\d+)",href)
             if m != None:
@@ -138,16 +136,17 @@ def get_urls_from_text(data,configuration=None,normalize=False):
         try:
             href = href.replace('&index=1','')
             adapter = adapters.getAdapter(configuration,href)
-            if adapter.story.getMetadata('storyUrl') not in normalized:
-                normalized.append(adapter.story.getMetadata('storyUrl'))
-                retlist.append(href)
+            if adapter.story.getMetadata('storyUrl') not in urls:
+                urls[adapter.story.getMetadata('storyUrl')] = [href]
+            else:
+                urls[adapter.story.getMetadata('storyUrl')].append(href)
         except:
             pass
 
-    if normalize:
-        return normalized
-    else:
-        return retlist
+    # Simply return the longest URL with the assumption that it contains the
+    # most user readable metadata, if not normalized
+    return urls.keys() if normalize else [max(value, key=len) for key, value in urls.items()]
+
 
 def form_url(parenturl,url):
      url = url.strip() # ran across an image with a space in the
@@ -177,7 +176,8 @@ def form_url(parenturl,url):
      return returl
        
 def get_urls_from_imap(srv,user,passwd,folder,markread=True):
-    
+
+    logger.debug("get_urls_from_imap srv:(%s)"%srv)
     mail = imaplib.IMAP4_SSL(srv)
     mail.login(user, passwd)
     mail.list()
@@ -186,8 +186,8 @@ def get_urls_from_imap(srv,user,passwd,folder,markread=True):
     
     result, data = mail.uid('search', None, "UNSEEN")
     
-    #print("result:%s"%result)
-    #print("data:%s"%data)
+    #logger.debug("result:%s"%result)
+    #logger.debug("data:%s"%data)
     urls=set()
     
     #latest_email_uid = data[0].split()[-1]
@@ -195,8 +195,8 @@ def get_urls_from_imap(srv,user,passwd,folder,markread=True):
 
         result, data = mail.uid('fetch', email_uid, '(BODY.PEEK[])') #RFC822
     
-        #print("result:%s"%result)
-        #print("data:%s"%data)
+        #logger.debug("result:%s"%result)
+        #logger.debug("data:%s"%data)
     
         raw_email = data[0][1]
         
@@ -205,28 +205,28 @@ def get_urls_from_imap(srv,user,passwd,folder,markread=True):
     
         email_message = email.message_from_string(raw_email)
      
-        #print "To:%s"%email_message['To']
-        #print "From:%s"%email_message['From']
-        #print "Subject:%s"%email_message['Subject']
+        #logger.debug "To:%s"%email_message['To']
+        #logger.debug "From:%s"%email_message['From']
+        #logger.debug "Subject:%s"%email_message['Subject']
     
-    #    print("payload:%s"%email_message.get_payload())
+    #    logger.debug("payload:%s"%email_message.get_payload())
 
         urllist=[]
         for part in email_message.walk():
             try:
-            #print("part mime:%s"%part.get_content_type())
+            #logger.debug("part mime:%s"%part.get_content_type())
                 if part.get_content_type() == 'text/plain':
                     urllist.extend(get_urls_from_text(part.get_payload(decode=True)))
                 if part.get_content_type() == 'text/html':
                     urllist.extend(get_urls_from_html(part.get_payload(decode=True)))
             except Exception as e:
-                print("Failed to read email content: %s"%e)
-        #print "urls:%s"%get_urls_from_text(get_first_text_block(email_message))
+                logger.error("Failed to read email content: %s"%e)
+        #logger.debug "urls:%s"%get_urls_from_text(get_first_text_block(email_message))
 
         if urllist and markread:
             #obj.store(data[0].replace(' ',','),'+FLAGS','\Seen')
             r,d = mail.uid('store',email_uid,'+FLAGS','(\\SEEN)')
-            #print("seen result:%s->%s"%(email_uid,r))
+            #logger.debug("seen result:%s->%s"%(email_uid,r))
                 
         [ urls.add(x) for x in urllist ]
     

fanficfare/htmlcleanup.py

@@ -81,10 +81,10 @@ def removeEntities(text):
     
     try:
         t = text.decode('utf-8')
-    except UnicodeEncodeError, e:
+    except (UnicodeEncodeError,UnicodeDecodeError), e:
         try:
             t = text.encode ('ascii', 'xmlcharrefreplace') 
-        except UnicodeEncodeError, e:
+        except (UnicodeEncodeError,UnicodeDecodeError), e:
             t = text
     text = t 
     # replace numeric versions of [&<>] with named versions,

fanficfare/story.py

@@ -18,12 +18,16 @@
 import os, re
 import urlparse
 import string
+import json
+import datetime
 from math import floor
 from functools import partial
 import logging
 logger = logging.getLogger(__name__)
 import urlparse as up
 
+import bs4
+
 import exceptions
 from htmlcleanup import conditionalRemoveEntities, removeAllEntities
 from configurable import Configurable, re_compile
@@ -544,6 +548,69 @@ class Story(Configurable):
         else:
             return self.join_list(key,retlist)
 
+    # for saving an html-ified copy of metadata.
+    def dump_html_metadata(self):
+        lines=[]
+        for k,v in sorted(self.metadata.iteritems()):
+            classes=['metadata']
+            if isinstance(v, (datetime.date, datetime.datetime, datetime.time)):
+                classes.append("datetime")
+                val = v.isoformat()
+            elif isinstance(v,list):
+                classes.append("list")
+                if '' in v:
+                    v.remove('')
+                if None in v:
+                    v.remove(None)
+                #logger.debug("k:%s v:%s"%(k,v))
+                # force ints/floats to strings.
+                val = "<ul>\n<li>%s</li>\n</ul>" % "</li>\n<li>".join([ "%s"%x for x in v ])
+            elif isinstance(v, (int)):
+                classes.append("int")
+                val = v
+            else:
+                val = v
+
+            # don't include items passed in for calibre cols, etc.
+            if not k.startswith('calibre_') and k not in ['output_css']:
+                lines.append("<p><span class='label'>%s</span>: <div class='%s' id='%s'>%s</div><p>\n"%(
+                        self.get_label(k),
+                        " ".join(classes),
+                        k,val))
+        return "\n".join(lines)
+
+    # for loading an html-ified copy of metadata.
+    def load_html_metadata(self,data):
+        soup = bs4.BeautifulSoup(data,'html5lib')
+        for tag in soup.find_all('div','metadata'):
+            val = None
+            if 'datetime' in tag['class']:
+                v = tag.string
+                try:
+                    val = datetime.datetime.strptime(v, '%Y-%m-%dT%H:%M:%S.%f')
+                except ValueError:
+                    try:
+                        val = datetime.datetime.strptime(v, '%Y-%m-%dT%H:%M:%S')
+                    except ValueError:
+                        try:
+                            val = datetime.datetime.strptime(v, '%Y-%m-%d')
+                        except ValueError:
+                            pass
+            elif 'list' in tag['class']:
+                val = []
+                for i in tag.find_all('li'):
+                    val.append(i.string)
+            elif 'int' in tag['class']:
+                val = int(tag.string)
+            else:
+                val = unicode("\n".join([ unicode(c) for c in tag.contents ]))
+                
+            #logger.debug("key(%s)=val(%s)"%(tag['id'],val))
+            if val:
+                self.metadata[tag['id']]=val
+                
+        # self.metadata = json.loads(s, object_hook=datetime_decoder)
+        
     def getMetadataRaw(self,key):
         if self.isValidMetaEntry(key) and self.metadata.has_key(key):
             return self.metadata[key]
@@ -671,9 +738,6 @@ class Story(Configurable):
         if not value in self.metadata[listname]:
             self.metadata[listname].append(value)
 
-        if listname == 'category' and self.getConfig('add_genre_when_multi_category') and len(self.metadata[listname]) > 1:
-            self.addToList('genre',self.getConfig('add_genre_when_multi_category'))
-
     def isList(self,listname):
         'Everything set with an include_in_* is considered a list.'
         return self.isListType(listname) or \
@@ -702,6 +766,8 @@ class Story(Configurable):
                                             doreplacements=doreplacements)]
             else:
                 retlist = self.getMetadataRaw(listname)
+                if retlist is None:
+                    retlist = []
 
         if retlist:
             if doreplacements:
@@ -715,6 +781,13 @@ class Story(Configurable):
 
             retlist = filter( lambda x : x!=None and x!='' ,retlist)
 
+        if listname == 'genre' and self.getConfig('add_genre_when_multi_category') and len(self.getList('category',
+                                                                                                        removeallentities=False,
+                                                                                                        # to avoid inf loops if genre/cat substs
+                                                                                                        doreplacements=False
+                                                                                                        )) > 1:
+            retlist.append(self.getConfig('add_genre_when_multi_category'))
+
         # reorder ships so b/a and c/b/a become a/b and a/b/c.  Only on '/',
         # use replace_metadata to change separator first if needed.
         # ships=>[ ]*(/|&amp;|&)[ ]*=>/
@@ -929,4 +1002,3 @@ def commaGroups(s):
         groups.append(s[-3:])
         s = s[:-3]
     return s + ','.join(reversed(groups))
-

fanficfare/writers/base_writer.py

@@ -45,36 +45,6 @@ class BaseStoryWriter(Configurable):
         self.adapter = adapter
         self.story = adapter.getStoryMetadataOnly() # only cache the metadata initially.
         
-        # fall back labels.
-        self.titleLabels = {
-            'category':'Category',
-            'genre':'Genre',
-            'language':'Language',
-            'status':'Status',
-            'series':'Series',
-            'characters':'Characters',
-            'ships':'Relationships',
-            'datePublished':'Published',
-            'dateUpdated':'Updated',
-            'dateCreated':'Packaged',
-            'rating':'Rating',
-            'warnings':'Warnings',
-            'numChapters':'Chapters',
-            'numWords':'Words',
-            'site':'Site',
-            'storyId':'Story ID',
-            'authorId':'Author ID',
-            'extratags':'Extra Tags',
-            'title':'Title',
-            'storyUrl':'Story URL',
-            'description':'Summary',
-            'author':'Author',
-            'authorUrl':'Author URL',
-            'formatname':'File Format',
-            'formatext':'File Extension',
-            'siteabbrev':'Site Abbrev',
-            'version':'Downloader Version'
-            }
         self.story.setMetadata('formatname',self.getFormatName())
         self.story.setMetadata('formatext',self.getFormatExt())
 
@@ -135,15 +105,16 @@ class BaseStoryWriter(Configurable):
                             TEMPLATE=WIDE_ENTRY
                         else:
                             TEMPLATE=ENTRY
-                            
-                        if self.hasConfig(entry+"_label"):
-                            label=self.getConfig(entry+"_label")
-                        elif entry in self.titleLabels:
-                            logger.debug("Using fallback label for %s_label"%entry)
-                            label=self.titleLabels[entry]
-                        else:
-                            label="%s"%entry.title()
-                            logger.debug("No known label for %s, fallback to '%s'"%(entry,label))
+
+                        label=self.get_label(entry)
+                        # if self.hasConfig(entry+"_label"):
+                        #     label=self.getConfig(entry+"_label")
+                        # elif entry in self.titleLabels:
+                        #     logger.debug("Using fallback label for %s_label"%entry)
+                        #     label=self.titleLabels[entry]
+                        # else:
+                        #     label="%s"%entry.title()
+                        #     logger.debug("No known label for %s, fallback to '%s'"%(entry,label))
 
                         # If the label for the title entry is empty, use the
                         # 'no title' option if there is one.

fanficfare/writers/writer_epub.py

@@ -258,14 +258,15 @@ div { margin: 0pt; padding: 0pt; }
             if self.isValidMetaEntry(entry):
                 val = self.story.getMetadata(entry)
                 if val and ( entry not in oldvalues or val != oldvalues[entry] ):
-                    if self.hasConfig(entry+"_label"):
-                        label=self.getConfig(entry+"_label")
-                    elif entry in self.titleLabels:
-                        logger.debug("Using fallback label for %s_label"%entry)
-                        label=self.titleLabels[entry]
-                    else:
-                        label="%s"%entry.title()
-                        logger.debug("No known label for %s, fallback to '%s'"%(entry,label))
+                    label=self.get_label(entry)
+                    # if self.hasConfig(entry+"_label"):
+                    #     label=self.getConfig(entry+"_label")
+                    # elif entry in self.titleLabels:
+                    #     logger.debug("Using fallback label for %s_label"%entry)
+                    #     label=self.titleLabels[entry]
+                    # else:
+                    #     label="%s"%entry.title()
+                    #     logger.debug("No known label for %s, fallback to '%s'"%(entry,label))
 
                     retval = retval + ENTRY.substitute({'id':entry,
                                                         'label':label,

included_dependencies/html5lib/__init__.py

@@ -20,4 +20,4 @@ from .serializer import serialize
 
 __all__ = ["HTMLParser", "parse", "parseFragment", "getTreeBuilder",
            "getTreeWalker", "serialize"]
-__version__ = "0.999"
+__version__ = "0.99999"

included_dependencies/html5lib/constants.py

@@ -1,292 +1,290 @@
 from __future__ import absolute_import, division, unicode_literals
 
 import string
-import gettext
-_ = gettext.gettext
 
 EOF = None
 
 E = {
     "null-character":
-        _("Null character in input stream, replaced with U+FFFD."),
+        "Null character in input stream, replaced with U+FFFD.",
     "invalid-codepoint":
-        _("Invalid codepoint in stream."),
+        "Invalid codepoint in stream.",
     "incorrectly-placed-solidus":
-        _("Solidus (/) incorrectly placed in tag."),
+        "Solidus (/) incorrectly placed in tag.",
     "incorrect-cr-newline-entity":
-        _("Incorrect CR newline entity, replaced with LF."),
+        "Incorrect CR newline entity, replaced with LF.",
     "illegal-windows-1252-entity":
-        _("Entity used with illegal number (windows-1252 reference)."),
+        "Entity used with illegal number (windows-1252 reference).",
     "cant-convert-numeric-entity":
-        _("Numeric entity couldn't be converted to character "
-          "(codepoint U+%(charAsInt)08x)."),
+        "Numeric entity couldn't be converted to character "
+        "(codepoint U+%(charAsInt)08x).",
     "illegal-codepoint-for-numeric-entity":
-        _("Numeric entity represents an illegal codepoint: "
-          "U+%(charAsInt)08x."),
+        "Numeric entity represents an illegal codepoint: "
+        "U+%(charAsInt)08x.",
     "numeric-entity-without-semicolon":
-        _("Numeric entity didn't end with ';'."),
+        "Numeric entity didn't end with ';'.",
     "expected-numeric-entity-but-got-eof":
-        _("Numeric entity expected. Got end of file instead."),
+        "Numeric entity expected. Got end of file instead.",
     "expected-numeric-entity":
-        _("Numeric entity expected but none found."),
+        "Numeric entity expected but none found.",
     "named-entity-without-semicolon":
-        _("Named entity didn't end with ';'."),
+        "Named entity didn't end with ';'.",
     "expected-named-entity":
-        _("Named entity expected. Got none."),
+        "Named entity expected. Got none.",
     "attributes-in-end-tag":
-        _("End tag contains unexpected attributes."),
+        "End tag contains unexpected attributes.",
     'self-closing-flag-on-end-tag':
-        _("End tag contains unexpected self-closing flag."),
+        "End tag contains unexpected self-closing flag.",
     "expected-tag-name-but-got-right-bracket":
-        _("Expected tag name. Got '>' instead."),
+        "Expected tag name. Got '>' instead.",
     "expected-tag-name-but-got-question-mark":
-        _("Expected tag name. Got '?' instead. (HTML doesn't "
-          "support processing instructions.)"),
+        "Expected tag name. Got '?' instead. (HTML doesn't "
+        "support processing instructions.)",
     "expected-tag-name":
-        _("Expected tag name. Got something else instead"),
+        "Expected tag name. Got something else instead",
     "expected-closing-tag-but-got-right-bracket":
-        _("Expected closing tag. Got '>' instead. Ignoring '</>'."),
+        "Expected closing tag. Got '>' instead. Ignoring '</>'.",
     "expected-closing-tag-but-got-eof":
-        _("Expected closing tag. Unexpected end of file."),
+        "Expected closing tag. Unexpected end of file.",
     "expected-closing-tag-but-got-char":
-        _("Expected closing tag. Unexpected character '%(data)s' found."),
+        "Expected closing tag. Unexpected character '%(data)s' found.",
     "eof-in-tag-name":
-        _("Unexpected end of file in the tag name."),
+        "Unexpected end of file in the tag name.",
     "expected-attribute-name-but-got-eof":
-        _("Unexpected end of file. Expected attribute name instead."),
+        "Unexpected end of file. Expected attribute name instead.",
     "eof-in-attribute-name":
-        _("Unexpected end of file in attribute name."),
+        "Unexpected end of file in attribute name.",
     "invalid-character-in-attribute-name":
-        _("Invalid character in attribute name"),
+        "Invalid character in attribute name",
     "duplicate-attribute":
-        _("Dropped duplicate attribute on tag."),
+        "Dropped duplicate attribute on tag.",
     "expected-end-of-tag-name-but-got-eof":
-        _("Unexpected end of file. Expected = or end of tag."),
+        "Unexpected end of file. Expected = or end of tag.",
     "expected-attribute-value-but-got-eof":
-        _("Unexpected end of file. Expected attribute value."),
+        "Unexpected end of file. Expected attribute value.",
     "expected-attribute-value-but-got-right-bracket":
-        _("Expected attribute value. Got '>' instead."),
+        "Expected attribute value. Got '>' instead.",
     'equals-in-unquoted-attribute-value':
-        _("Unexpected = in unquoted attribute"),
+        "Unexpected = in unquoted attribute",
     'unexpected-character-in-unquoted-attribute-value':
-        _("Unexpected character in unquoted attribute"),
+        "Unexpected character in unquoted attribute",
     "invalid-character-after-attribute-name":
-        _("Unexpected character after attribute name."),
+        "Unexpected character after attribute name.",
     "unexpected-character-after-attribute-value":
-        _("Unexpected character after attribute value."),
+        "Unexpected character after attribute value.",
     "eof-in-attribute-value-double-quote":
-        _("Unexpected end of file in attribute value (\")."),
+        "Unexpected end of file in attribute value (\").",
     "eof-in-attribute-value-single-quote":
-        _("Unexpected end of file in attribute value (')."),
+        "Unexpected end of file in attribute value (').",
     "eof-in-attribute-value-no-quotes":
-        _("Unexpected end of file in attribute value."),
+        "Unexpected end of file in attribute value.",
     "unexpected-EOF-after-solidus-in-tag":
-        _("Unexpected end of file in tag. Expected >"),
+        "Unexpected end of file in tag. Expected >",
     "unexpected-character-after-solidus-in-tag":
-        _("Unexpected character after / in tag. Expected >"),
+        "Unexpected character after / in tag. Expected >",
     "expected-dashes-or-doctype":
-        _("Expected '--' or 'DOCTYPE'. Not found."),
+        "Expected '--' or 'DOCTYPE'. Not found.",
     "unexpected-bang-after-double-dash-in-comment":
-        _("Unexpected ! after -- in comment"),
+        "Unexpected ! after -- in comment",
     "unexpected-space-after-double-dash-in-comment":
-        _("Unexpected space after -- in comment"),
+        "Unexpected space after -- in comment",
     "incorrect-comment":
-        _("Incorrect comment."),
+        "Incorrect comment.",
     "eof-in-comment":
-        _("Unexpected end of file in comment."),
+        "Unexpected end of file in comment.",
     "eof-in-comment-end-dash":
-        _("Unexpected end of file in comment (-)"),
+        "Unexpected end of file in comment (-)",
     "unexpected-dash-after-double-dash-in-comment":
-        _("Unexpected '-' after '--' found in comment."),
+        "Unexpected '-' after '--' found in comment.",
     "eof-in-comment-double-dash":
-        _("Unexpected end of file in comment (--)."),
+        "Unexpected end of file in comment (--).",
     "eof-in-comment-end-space-state":
-        _("Unexpected end of file in comment."),
+        "Unexpected end of file in comment.",
     "eof-in-comment-end-bang-state":
-        _("Unexpected end of file in comment."),
+        "Unexpected end of file in comment.",
     "unexpected-char-in-comment":
-        _("Unexpected character in comment found."),
+        "Unexpected character in comment found.",
     "need-space-after-doctype":
-        _("No space after literal string 'DOCTYPE'."),
+        "No space after literal string 'DOCTYPE'.",
     "expected-doctype-name-but-got-right-bracket":
-        _("Unexpected > character. Expected DOCTYPE name."),
+        "Unexpected > character. Expected DOCTYPE name.",
     "expected-doctype-name-but-got-eof":
-        _("Unexpected end of file. Expected DOCTYPE name."),
+        "Unexpected end of file. Expected DOCTYPE name.",
     "eof-in-doctype-name":
-        _("Unexpected end of file in DOCTYPE name."),
+        "Unexpected end of file in DOCTYPE name.",
     "eof-in-doctype":
-        _("Unexpected end of file in DOCTYPE."),
+        "Unexpected end of file in DOCTYPE.",
     "expected-space-or-right-bracket-in-doctype":
-        _("Expected space or '>'. Got '%(data)s'"),
+        "Expected space or '>'. Got '%(data)s'",
     "unexpected-end-of-doctype":
-        _("Unexpected end of DOCTYPE."),
+        "Unexpected end of DOCTYPE.",
     "unexpected-char-in-doctype":
-        _("Unexpected character in DOCTYPE."),
+        "Unexpected character in DOCTYPE.",
     "eof-in-innerhtml":
-        _("XXX innerHTML EOF"),
+        "XXX innerHTML EOF",
     "unexpected-doctype":
-        _("Unexpected DOCTYPE. Ignored."),
+        "Unexpected DOCTYPE. Ignored.",
     "non-html-root":
-        _("html needs to be the first start tag."),
+        "html needs to be the first start tag.",
     "expected-doctype-but-got-eof":
-        _("Unexpected End of file. Expected DOCTYPE."),
+        "Unexpected End of file. Expected DOCTYPE.",
     "unknown-doctype":
-        _("Erroneous DOCTYPE."),
+        "Erroneous DOCTYPE.",
     "expected-doctype-but-got-chars":
-        _("Unexpected non-space characters. Expected DOCTYPE."),
+        "Unexpected non-space characters. Expected DOCTYPE.",
     "expected-doctype-but-got-start-tag":
-        _("Unexpected start tag (%(name)s). Expected DOCTYPE."),
+        "Unexpected start tag (%(name)s). Expected DOCTYPE.",
     "expected-doctype-but-got-end-tag":
-        _("Unexpected end tag (%(name)s). Expected DOCTYPE."),
+        "Unexpected end tag (%(name)s). Expected DOCTYPE.",
     "end-tag-after-implied-root":
-        _("Unexpected end tag (%(name)s) after the (implied) root element."),
+        "Unexpected end tag (%(name)s) after the (implied) root element.",
     "expected-named-closing-tag-but-got-eof":
-        _("Unexpected end of file. Expected end tag (%(name)s)."),
+        "Unexpected end of file. Expected end tag (%(name)s).",
     "two-heads-are-not-better-than-one":
-        _("Unexpected start tag head in existing head. Ignored."),
+        "Unexpected start tag head in existing head. Ignored.",
     "unexpected-end-tag":
-        _("Unexpected end tag (%(name)s). Ignored."),
+        "Unexpected end tag (%(name)s). Ignored.",
     "unexpected-start-tag-out-of-my-head":
-        _("Unexpected start tag (%(name)s) that can be in head. Moved."),
+        "Unexpected start tag (%(name)s) that can be in head. Moved.",
     "unexpected-start-tag":
-        _("Unexpected start tag (%(name)s)."),
+        "Unexpected start tag (%(name)s).",
     "missing-end-tag":
-        _("Missing end tag (%(name)s)."),
+        "Missing end tag (%(name)s).",
     "missing-end-tags":
-        _("Missing end tags (%(name)s)."),
+        "Missing end tags (%(name)s).",
     "unexpected-start-tag-implies-end-tag":
-        _("Unexpected start tag (%(startName)s) "
-          "implies end tag (%(endName)s)."),
+        "Unexpected start tag (%(startName)s) "
+        "implies end tag (%(endName)s).",
     "unexpected-start-tag-treated-as":
-        _("Unexpected start tag (%(originalName)s). Treated as %(newName)s."),
+        "Unexpected start tag (%(originalName)s). Treated as %(newName)s.",
     "deprecated-tag":
-        _("Unexpected start tag %(name)s. Don't use it!"),
+        "Unexpected start tag %(name)s. Don't use it!",
     "unexpected-start-tag-ignored":
-        _("Unexpected start tag %(name)s. Ignored."),
+        "Unexpected start tag %(name)s. Ignored.",
     "expected-one-end-tag-but-got-another":
-        _("Unexpected end tag (%(gotName)s). "
-          "Missing end tag (%(expectedName)s)."),
+        "Unexpected end tag (%(gotName)s). "
+        "Missing end tag (%(expectedName)s).",
     "end-tag-too-early":
-        _("End tag (%(name)s) seen too early. Expected other end tag."),
+        "End tag (%(name)s) seen too early. Expected other end tag.",
     "end-tag-too-early-named":
-        _("Unexpected end tag (%(gotName)s). Expected end tag (%(expectedName)s)."),
+        "Unexpected end tag (%(gotName)s). Expected end tag (%(expectedName)s).",
     "end-tag-too-early-ignored":
-        _("End tag (%(name)s) seen too early. Ignored."),
+        "End tag (%(name)s) seen too early. Ignored.",
     "adoption-agency-1.1":
-        _("End tag (%(name)s) violates step 1, "
-          "paragraph 1 of the adoption agency algorithm."),
+        "End tag (%(name)s) violates step 1, "
+        "paragraph 1 of the adoption agency algorithm.",
     "adoption-agency-1.2":
-        _("End tag (%(name)s) violates step 1, "
-          "paragraph 2 of the adoption agency algorithm."),
+        "End tag (%(name)s) violates step 1, "
+        "paragraph 2 of the adoption agency algorithm.",
     "adoption-agency-1.3":
-        _("End tag (%(name)s) violates step 1, "
-          "paragraph 3 of the adoption agency algorithm."),
+        "End tag (%(name)s) violates step 1, "
+        "paragraph 3 of the adoption agency algorithm.",
     "adoption-agency-4.4":
-        _("End tag (%(name)s) violates step 4, "
-          "paragraph 4 of the adoption agency algorithm."),
+        "End tag (%(name)s) violates step 4, "
+        "paragraph 4 of the adoption agency algorithm.",
     "unexpected-end-tag-treated-as":
-        _("Unexpected end tag (%(originalName)s). Treated as %(newName)s."),
+        "Unexpected end tag (%(originalName)s). Treated as %(newName)s.",
     "no-end-tag":
-        _("This element (%(name)s) has no end tag."),
+        "This element (%(name)s) has no end tag.",
     "unexpected-implied-end-tag-in-table":
-        _("Unexpected implied end tag (%(name)s) in the table phase."),
+        "Unexpected implied end tag (%(name)s) in the table phase.",
     "unexpected-implied-end-tag-in-table-body":
-        _("Unexpected implied end tag (%(name)s) in the table body phase."),
+        "Unexpected implied end tag (%(name)s) in the table body phase.",
     "unexpected-char-implies-table-voodoo":
-        _("Unexpected non-space characters in "
-          "table context caused voodoo mode."),
+        "Unexpected non-space characters in "
+        "table context caused voodoo mode.",
     "unexpected-hidden-input-in-table":
-        _("Unexpected input with type hidden in table context."),
+        "Unexpected input with type hidden in table context.",
     "unexpected-form-in-table":
-        _("Unexpected form in table context."),
+        "Unexpected form in table context.",
     "unexpected-start-tag-implies-table-voodoo":
-        _("Unexpected start tag (%(name)s) in "
-          "table context caused voodoo mode."),
+        "Unexpected start tag (%(name)s) in "
+        "table context caused voodoo mode.",
     "unexpected-end-tag-implies-table-voodoo":
-        _("Unexpected end tag (%(name)s) in "
-          "table context caused voodoo mode."),
+        "Unexpected end tag (%(name)s) in "
+        "table context caused voodoo mode.",
     "unexpected-cell-in-table-body":
-        _("Unexpected table cell start tag (%(name)s) "
-          "in the table body phase."),
+        "Unexpected table cell start tag (%(name)s) "
+        "in the table body phase.",
     "unexpected-cell-end-tag":
-        _("Got table cell end tag (%(name)s) "
-          "while required end tags are missing."),
+        "Got table cell end tag (%(name)s) "
+        "while required end tags are missing.",
     "unexpected-end-tag-in-table-body":
-        _("Unexpected end tag (%(name)s) in the table body phase. Ignored."),
+        "Unexpected end tag (%(name)s) in the table body phase. Ignored.",
     "unexpected-implied-end-tag-in-table-row":
-        _("Unexpected implied end tag (%(name)s) in the table row phase."),
+        "Unexpected implied end tag (%(name)s) in the table row phase.",
     "unexpected-end-tag-in-table-row":
-        _("Unexpected end tag (%(name)s) in the table row phase. Ignored."),
+        "Unexpected end tag (%(name)s) in the table row phase. Ignored.",
     "unexpected-select-in-select":
-        _("Unexpected select start tag in the select phase "
-          "treated as select end tag."),
+        "Unexpected select start tag in the select phase "
+        "treated as select end tag.",
     "unexpected-input-in-select":
-        _("Unexpected input start tag in the select phase."),
+        "Unexpected input start tag in the select phase.",
     "unexpected-start-tag-in-select":
-        _("Unexpected start tag token (%(name)s in the select phase. "
-          "Ignored."),
+        "Unexpected start tag token (%(name)s in the select phase. "
+        "Ignored.",
     "unexpected-end-tag-in-select":
-        _("Unexpected end tag (%(name)s) in the select phase. Ignored."),
+        "Unexpected end tag (%(name)s) in the select phase. Ignored.",
     "unexpected-table-element-start-tag-in-select-in-table":
-        _("Unexpected table element start tag (%(name)s) in the select in table phase."),
+        "Unexpected table element start tag (%(name)s) in the select in table phase.",
     "unexpected-table-element-end-tag-in-select-in-table":
-        _("Unexpected table element end tag (%(name)s) in the select in table phase."),
+        "Unexpected table element end tag (%(name)s) in the select in table phase.",
     "unexpected-char-after-body":
-        _("Unexpected non-space characters in the after body phase."),
+        "Unexpected non-space characters in the after body phase.",
     "unexpected-start-tag-after-body":
-        _("Unexpected start tag token (%(name)s)"
-          " in the after body phase."),
+        "Unexpected start tag token (%(name)s)"
+        " in the after body phase.",
     "unexpected-end-tag-after-body":
-        _("Unexpected end tag token (%(name)s)"
-          " in the after body phase."),
+        "Unexpected end tag token (%(name)s)"
+        " in the after body phase.",
     "unexpected-char-in-frameset":
-        _("Unexpected characters in the frameset phase. Characters ignored."),
+        "Unexpected characters in the frameset phase. Characters ignored.",
     "unexpected-start-tag-in-frameset":
-        _("Unexpected start tag token (%(name)s)"
-          " in the frameset phase. Ignored."),
+        "Unexpected start tag token (%(name)s)"
+        " in the frameset phase. Ignored.",
     "unexpected-frameset-in-frameset-innerhtml":
-        _("Unexpected end tag token (frameset) "
-          "in the frameset phase (innerHTML)."),
+        "Unexpected end tag token (frameset) "
+        "in the frameset phase (innerHTML).",
     "unexpected-end-tag-in-frameset":
-        _("Unexpected end tag token (%(name)s)"
-          " in the frameset phase. Ignored."),
+        "Unexpected end tag token (%(name)s)"
+        " in the frameset phase. Ignored.",
     "unexpected-char-after-frameset":
-        _("Unexpected non-space characters in the "
-          "after frameset phase. Ignored."),
+        "Unexpected non-space characters in the "
+        "after frameset phase. Ignored.",
     "unexpected-start-tag-after-frameset":
-        _("Unexpected start tag (%(name)s)"
-          " in the after frameset phase. Ignored."),
+        "Unexpected start tag (%(name)s)"
+        " in the after frameset phase. Ignored.",
     "unexpected-end-tag-after-frameset":
-        _("Unexpected end tag (%(name)s)"
-          " in the after frameset phase. Ignored."),
+        "Unexpected end tag (%(name)s)"
+        " in the after frameset phase. Ignored.",
     "unexpected-end-tag-after-body-innerhtml":
-        _("Unexpected end tag after body(innerHtml)"),
+        "Unexpected end tag after body(innerHtml)",
     "expected-eof-but-got-char":
-        _("Unexpected non-space characters. Expected end of file."),
+        "Unexpected non-space characters. Expected end of file.",
     "expected-eof-but-got-start-tag":
-        _("Unexpected start tag (%(name)s)"
-          ". Expected end of file."),
+        "Unexpected start tag (%(name)s)"
+        ". Expected end of file.",
     "expected-eof-but-got-end-tag":
-        _("Unexpected end tag (%(name)s)"
-          ". Expected end of file."),
+        "Unexpected end tag (%(name)s)"
+        ". Expected end of file.",
     "eof-in-table":
-        _("Unexpected end of file. Expected table content."),
+        "Unexpected end of file. Expected table content.",
     "eof-in-select":
-        _("Unexpected end of file. Expected select content."),
+        "Unexpected end of file. Expected select content.",
     "eof-in-frameset":
-        _("Unexpected end of file. Expected frameset content."),
+        "Unexpected end of file. Expected frameset content.",
     "eof-in-script-in-script":
-        _("Unexpected end of file. Expected script content."),
+        "Unexpected end of file. Expected script content.",
     "eof-in-foreign-lands":
-        _("Unexpected end of file. Expected foreign content"),
+        "Unexpected end of file. Expected foreign content",
     "non-void-element-with-trailing-solidus":
-        _("Trailing solidus not allowed on element %(name)s"),
+        "Trailing solidus not allowed on element %(name)s",
     "unexpected-html-element-in-foreign-content":
-        _("Element %(name)s not allowed in a non-html context"),
+        "Element %(name)s not allowed in a non-html context",
     "unexpected-end-tag-before-html":
-        _("Unexpected end tag (%(name)s) before html."),
+        "Unexpected end tag (%(name)s) before html.",
     "XXX-undefined-error":
-        _("Undefined error (this sucks and should be fixed)"),
+        "Undefined error (this sucks and should be fixed)",
 }
 
 namespaces = {
@@ -298,7 +296,7 @@ namespaces = {
     "xmlns": "http://www.w3.org/2000/xmlns/"
 }
 
-scopingElements = frozenset((
+scopingElements = frozenset([
     (namespaces["html"], "applet"),
     (namespaces["html"], "caption"),
     (namespaces["html"], "html"),
@@ -316,9 +314,9 @@ scopingElements = frozenset((
     (namespaces["svg"], "foreignObject"),
     (namespaces["svg"], "desc"),
     (namespaces["svg"], "title"),
-))
+])
 
-formattingElements = frozenset((
+formattingElements = frozenset([
     (namespaces["html"], "a"),
     (namespaces["html"], "b"),
     (namespaces["html"], "big"),
@@ -333,9 +331,9 @@ formattingElements = frozenset((
     (namespaces["html"], "strong"),
     (namespaces["html"], "tt"),
     (namespaces["html"], "u")
-))
+])
 
-specialElements = frozenset((
+specialElements = frozenset([
     (namespaces["html"], "address"),
     (namespaces["html"], "applet"),
     (namespaces["html"], "area"),
@@ -416,22 +414,22 @@ specialElements = frozenset((
     (namespaces["html"], "wbr"),
     (namespaces["html"], "xmp"),
     (namespaces["svg"], "foreignObject")
-))
+])
 
-htmlIntegrationPointElements = frozenset((
+htmlIntegrationPointElements = frozenset([
     (namespaces["mathml"], "annotaion-xml"),
     (namespaces["svg"], "foreignObject"),
     (namespaces["svg"], "desc"),
     (namespaces["svg"], "title")
-))
+])
 
-mathmlTextIntegrationPointElements = frozenset((
+mathmlTextIntegrationPointElements = frozenset([
     (namespaces["mathml"], "mi"),
     (namespaces["mathml"], "mo"),
     (namespaces["mathml"], "mn"),
     (namespaces["mathml"], "ms"),
     (namespaces["mathml"], "mtext")
-))
+])
 
 adjustForeignAttributes = {
     "xlink:actuate": ("xlink", "actuate", namespaces["xlink"]),
@@ -451,21 +449,21 @@ adjustForeignAttributes = {
 unadjustForeignAttributes = dict([((ns, local), qname) for qname, (prefix, local, ns) in
                                   adjustForeignAttributes.items()])
 
-spaceCharacters = frozenset((
+spaceCharacters = frozenset([
     "\t",
     "\n",
     "\u000C",
     " ",
     "\r"
-))
+])
 
-tableInsertModeElements = frozenset((
+tableInsertModeElements = frozenset([
     "table",
     "tbody",
     "tfoot",
     "thead",
     "tr"
-))
+])
 
 asciiLowercase = frozenset(string.ascii_lowercase)
 asciiUppercase = frozenset(string.ascii_uppercase)
@@ -486,7 +484,7 @@ headingElements = (
     "h6"
 )
 
-voidElements = frozenset((
+voidElements = frozenset([
     "base",
     "command",
     "event-source",
@@ -502,11 +500,11 @@ voidElements = frozenset((
     "input",
     "source",
     "track"
-))
+])
 
-cdataElements = frozenset(('title', 'textarea'))
+cdataElements = frozenset(['title', 'textarea'])
 
-rcdataElements = frozenset((
+rcdataElements = frozenset([
     'style',
     'script',
     'xmp',
@@ -514,27 +512,27 @@ rcdataElements = frozenset((
     'noembed',
     'noframes',
     'noscript'
-))
+])
 
 booleanAttributes = {
-    "": frozenset(("irrelevant",)),
-    "style": frozenset(("scoped",)),
-    "img": frozenset(("ismap",)),
-    "audio": frozenset(("autoplay", "controls")),
-    "video": frozenset(("autoplay", "controls")),
-    "script": frozenset(("defer", "async")),
-    "details": frozenset(("open",)),
-    "datagrid": frozenset(("multiple", "disabled")),
-    "command": frozenset(("hidden", "disabled", "checked", "default")),
-    "hr": frozenset(("noshade")),
-    "menu": frozenset(("autosubmit",)),
-    "fieldset": frozenset(("disabled", "readonly")),
-    "option": frozenset(("disabled", "readonly", "selected")),
-    "optgroup": frozenset(("disabled", "readonly")),
-    "button": frozenset(("disabled", "autofocus")),
-    "input": frozenset(("disabled", "readonly", "required", "autofocus", "checked", "ismap")),
-    "select": frozenset(("disabled", "readonly", "autofocus", "multiple")),
-    "output": frozenset(("disabled", "readonly")),
+    "": frozenset(["irrelevant"]),
+    "style": frozenset(["scoped"]),
+    "img": frozenset(["ismap"]),
+    "audio": frozenset(["autoplay", "controls"]),
+    "video": frozenset(["autoplay", "controls"]),
+    "script": frozenset(["defer", "async"]),
+    "details": frozenset(["open"]),
+    "datagrid": frozenset(["multiple", "disabled"]),
+    "command": frozenset(["hidden", "disabled", "checked", "default"]),
+    "hr": frozenset(["noshade"]),
+    "menu": frozenset(["autosubmit"]),
+    "fieldset": frozenset(["disabled", "readonly"]),
+    "option": frozenset(["disabled", "readonly", "selected"]),
+    "optgroup": frozenset(["disabled", "readonly"]),
+    "button": frozenset(["disabled", "autofocus"]),
+    "input": frozenset(["disabled", "readonly", "required", "autofocus", "checked", "ismap"]),
+    "select": frozenset(["disabled", "readonly", "autofocus", "multiple"]),
+    "output": frozenset(["disabled", "readonly"]),
 }
 
 # entitiesWindows1252 has to be _ordered_ and needs to have an index. It
@@ -574,7 +572,7 @@ entitiesWindows1252 = (
     376     # 0x9F  0x0178  LATIN CAPITAL LETTER Y WITH DIAERESIS
 )
 
-xmlEntities = frozenset(('lt;', 'gt;', 'amp;', 'apos;', 'quot;'))
+xmlEntities = frozenset(['lt;', 'gt;', 'amp;', 'apos;', 'quot;'])
 
 entities = {
     "AElig": "\xc6",
@@ -3088,8 +3086,8 @@ tokenTypes = {
     "ParseError": 7
 }
 
-tagTokenTypes = frozenset((tokenTypes["StartTag"], tokenTypes["EndTag"],
-                           tokenTypes["EmptyTag"]))
+tagTokenTypes = frozenset([tokenTypes["StartTag"], tokenTypes["EndTag"],
+                           tokenTypes["EmptyTag"]])
 
 
 prefixes = dict([(v, k) for k, v in namespaces.items()])

included_dependencies/html5lib/filters/lint.py

@@ -1,8 +1,5 @@
 from __future__ import absolute_import, division, unicode_literals
 
-from gettext import gettext
-_ = gettext
-
 from . import _base
 from ..constants import cdataElements, rcdataElements, voidElements
 
@@ -23,24 +20,24 @@ class Filter(_base.Filter):
             if type in ("StartTag", "EmptyTag"):
                 name = token["name"]
                 if contentModelFlag != "PCDATA":
-                    raise LintError(_("StartTag not in PCDATA content model flag: %(tag)s") % {"tag": name})
+                    raise LintError("StartTag not in PCDATA content model flag: %(tag)s" % {"tag": name})
                 if not isinstance(name, str):
-                    raise LintError(_("Tag name is not a string: %(tag)r") % {"tag": name})
+                    raise LintError("Tag name is not a string: %(tag)r" % {"tag": name})
                 if not name:
-                    raise LintError(_("Empty tag name"))
+                    raise LintError("Empty tag name")
                 if type == "StartTag" and name in voidElements:
-                    raise LintError(_("Void element reported as StartTag token: %(tag)s") % {"tag": name})
+                    raise LintError("Void element reported as StartTag token: %(tag)s" % {"tag": name})
                 elif type == "EmptyTag" and name not in voidElements:
-                    raise LintError(_("Non-void element reported as EmptyTag token: %(tag)s") % {"tag": token["name"]})
+                    raise LintError("Non-void element reported as EmptyTag token: %(tag)s" % {"tag": token["name"]})
                 if type == "StartTag":
                     open_elements.append(name)
                 for name, value in token["data"]:
                     if not isinstance(name, str):
-                        raise LintError(_("Attribute name is not a string: %(name)r") % {"name": name})
+                        raise LintError("Attribute name is not a string: %(name)r" % {"name": name})
                     if not name:
-                        raise LintError(_("Empty attribute name"))
+                        raise LintError("Empty attribute name")
                     if not isinstance(value, str):
-                        raise LintError(_("Attribute value is not a string: %(value)r") % {"value": value})
+                        raise LintError("Attribute value is not a string: %(value)r" % {"value": value})
                 if name in cdataElements:
                     contentModelFlag = "CDATA"
                 elif name in rcdataElements:
@@ -51,43 +48,43 @@ class Filter(_base.Filter):
             elif type == "EndTag":
                 name = token["name"]
                 if not isinstance(name, str):
-                    raise LintError(_("Tag name is not a string: %(tag)r") % {"tag": name})
+                    raise LintError("Tag name is not a string: %(tag)r" % {"tag": name})
                 if not name:
-                    raise LintError(_("Empty tag name"))
+                    raise LintError("Empty tag name")
                 if name in voidElements:
-                    raise LintError(_("Void element reported as EndTag token: %(tag)s") % {"tag": name})
+                    raise LintError("Void element reported as EndTag token: %(tag)s" % {"tag": name})
                 start_name = open_elements.pop()
                 if start_name != name:
-                    raise LintError(_("EndTag (%(end)s) does not match StartTag (%(start)s)") % {"end": name, "start": start_name})
+                    raise LintError("EndTag (%(end)s) does not match StartTag (%(start)s)" % {"end": name, "start": start_name})
                 contentModelFlag = "PCDATA"
 
             elif type == "Comment":
                 if contentModelFlag != "PCDATA":
-                    raise LintError(_("Comment not in PCDATA content model flag"))
+                    raise LintError("Comment not in PCDATA content model flag")
 
             elif type in ("Characters", "SpaceCharacters"):
                 data = token["data"]
                 if not isinstance(data, str):
-                    raise LintError(_("Attribute name is not a string: %(name)r") % {"name": data})
+                    raise LintError("Attribute name is not a string: %(name)r" % {"name": data})
                 if not data:
-                    raise LintError(_("%(type)s token with empty data") % {"type": type})
+                    raise LintError("%(type)s token with empty data" % {"type": type})
                 if type == "SpaceCharacters":
                     data = data.strip(spaceCharacters)
                     if data:
-                        raise LintError(_("Non-space character(s) found in SpaceCharacters token: %(token)r") % {"token": data})
+                        raise LintError("Non-space character(s) found in SpaceCharacters token: %(token)r" % {"token": data})
 
             elif type == "Doctype":
                 name = token["name"]
                 if contentModelFlag != "PCDATA":
-                    raise LintError(_("Doctype not in PCDATA content model flag: %(name)s") % {"name": name})
+                    raise LintError("Doctype not in PCDATA content model flag: %(name)s" % {"name": name})
                 if not isinstance(name, str):
-                    raise LintError(_("Tag name is not a string: %(tag)r") % {"tag": name})
+                    raise LintError("Tag name is not a string: %(tag)r" % {"tag": name})
                 # XXX: what to do with token["data"] ?
 
             elif type in ("ParseError", "SerializeError"):
                 pass
 
             else:
-                raise LintError(_("Unknown token type: %(type)s") % {"type": type})
+                raise LintError("Unknown token type: %(type)s" % {"type": type})
 
             yield token

included_dependencies/html5lib/html5parser.py

@@ -18,6 +18,7 @@ from .constants import cdataElements, rcdataElements
 from .constants import tokenTypes, ReparseException, namespaces
 from .constants import htmlIntegrationPointElements, mathmlTextIntegrationPointElements
 from .constants import adjustForeignAttributes as adjustForeignAttributesMap
+from .constants import E
 
 
 def parse(doc, treebuilder="etree", encoding=None,
@@ -129,6 +130,17 @@ class HTMLParser(object):
 
         self.framesetOK = True
 
+    @property
+    def documentEncoding(self):
+        """The name of the character encoding
+        that was used to decode the input stream,
+        or :obj:`None` if that is not determined yet.
+
+        """
+        if not hasattr(self, 'tokenizer'):
+            return None
+        return self.tokenizer.stream.charEncoding[0]
+
     def isHTMLIntegrationPoint(self, element):
         if (element.name == "annotation-xml" and
                 element.namespace == namespaces["mathml"]):
@@ -245,7 +257,7 @@ class HTMLParser(object):
         # XXX The idea is to make errorcode mandatory.
         self.errors.append((self.tokenizer.stream.position(), errorcode, datavars))
         if self.strict:
-            raise ParseError
+            raise ParseError(E[errorcode] % datavars)
 
     def normalizeToken(self, token):
         """ HTML5 specific normalizations to the token stream """
@@ -868,7 +880,7 @@ def getPhases(debug):
             self.startTagHandler = utils.MethodDispatcher([
                 ("html", self.startTagHtml),
                 (("base", "basefont", "bgsound", "command", "link", "meta",
-                  "noframes", "script", "style", "title"),
+                  "script", "style", "title"),
                  self.startTagProcessInHead),
                 ("body", self.startTagBody),
                 ("frameset", self.startTagFrameset),
@@ -1205,8 +1217,7 @@ def getPhases(debug):
             attributes["name"] = "isindex"
             self.processStartTag(impliedTagToken("input", "StartTag",
                                                  attributes=attributes,
-                                                 selfClosing=
-                                                 token["selfClosing"]))
+                                                 selfClosing=token["selfClosing"]))
             self.processEndTag(impliedTagToken("label"))
             self.processStartTag(impliedTagToken("hr", "StartTag"))
             self.processEndTag(impliedTagToken("form"))

included_dependencies/html5lib/inputstream.py

@@ -28,7 +28,18 @@ asciiLettersBytes = frozenset([item.encode("ascii") for item in asciiLetters])
 asciiUppercaseBytes = frozenset([item.encode("ascii") for item in asciiUppercase])
 spacesAngleBrackets = spaceCharactersBytes | frozenset([b">", b"<"])
 
-invalid_unicode_re = re.compile("[\u0001-\u0008\u000B\u000E-\u001F\u007F-\u009F\uD800-\uDFFF\uFDD0-\uFDEF\uFFFE\uFFFF\U0001FFFE\U0001FFFF\U0002FFFE\U0002FFFF\U0003FFFE\U0003FFFF\U0004FFFE\U0004FFFF\U0005FFFE\U0005FFFF\U0006FFFE\U0006FFFF\U0007FFFE\U0007FFFF\U0008FFFE\U0008FFFF\U0009FFFE\U0009FFFF\U000AFFFE\U000AFFFF\U000BFFFE\U000BFFFF\U000CFFFE\U000CFFFF\U000DFFFE\U000DFFFF\U000EFFFE\U000EFFFF\U000FFFFE\U000FFFFF\U0010FFFE\U0010FFFF]")
+
+invalid_unicode_no_surrogate = "[\u0001-\u0008\u000B\u000E-\u001F\u007F-\u009F\uFDD0-\uFDEF\uFFFE\uFFFF\U0001FFFE\U0001FFFF\U0002FFFE\U0002FFFF\U0003FFFE\U0003FFFF\U0004FFFE\U0004FFFF\U0005FFFE\U0005FFFF\U0006FFFE\U0006FFFF\U0007FFFE\U0007FFFF\U0008FFFE\U0008FFFF\U0009FFFE\U0009FFFF\U000AFFFE\U000AFFFF\U000BFFFE\U000BFFFF\U000CFFFE\U000CFFFF\U000DFFFE\U000DFFFF\U000EFFFE\U000EFFFF\U000FFFFE\U000FFFFF\U0010FFFE\U0010FFFF]"
+
+if utils.supports_lone_surrogates:
+    # Use one extra step of indirection and create surrogates with
+    # unichr. Not using this indirection would introduce an illegal
+    # unicode literal on platforms not supporting such lone
+    # surrogates.
+    invalid_unicode_re = re.compile(invalid_unicode_no_surrogate +
+                                    eval('"\\uD800-\\uDFFF"'))
+else:
+    invalid_unicode_re = re.compile(invalid_unicode_no_surrogate)
 
 non_bmp_invalid_codepoints = set([0x1FFFE, 0x1FFFF, 0x2FFFE, 0x2FFFF, 0x3FFFE,
                                   0x3FFFF, 0x4FFFE, 0x4FFFF, 0x5FFFE, 0x5FFFF,
@@ -164,13 +175,18 @@ class HTMLUnicodeInputStream(object):
 
         """
 
-        # Craziness
-        if len("\U0010FFFF") == 1:
+        if not utils.supports_lone_surrogates:
+            # Such platforms will have already checked for such
+            # surrogate errors, so no need to do this checking.
+            self.reportCharacterErrors = None
+            self.replaceCharactersRegexp = None
+        elif len("\U0010FFFF") == 1:
             self.reportCharacterErrors = self.characterErrorsUCS4
-            self.replaceCharactersRegexp = re.compile("[\uD800-\uDFFF]")
+            self.replaceCharactersRegexp = re.compile(eval('"[\\uD800-\\uDFFF]"'))
         else:
             self.reportCharacterErrors = self.characterErrorsUCS2
-            self.replaceCharactersRegexp = re.compile("([\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?<![\uD800-\uDBFF])[\uDC00-\uDFFF])")
+            self.replaceCharactersRegexp = re.compile(
+                eval('"([\\uD800-\\uDBFF](?![\\uDC00-\\uDFFF])|(?<![\\uD800-\\uDBFF])[\\uDC00-\\uDFFF])"'))
 
         # List of where new lines occur
         self.newLines = [0]
@@ -265,11 +281,12 @@ class HTMLUnicodeInputStream(object):
                 self._bufferedCharacter = data[-1]
                 data = data[:-1]
 
-        self.reportCharacterErrors(data)
+        if self.reportCharacterErrors:
+            self.reportCharacterErrors(data)
 
-        # Replace invalid characters
-        # Note U+0000 is dealt with in the tokenizer
-        data = self.replaceCharactersRegexp.sub("\ufffd", data)
+            # Replace invalid characters
+            # Note U+0000 is dealt with in the tokenizer
+            data = self.replaceCharactersRegexp.sub("\ufffd", data)
 
         data = data.replace("\r\n", "\n")
         data = data.replace("\r", "\n")

included_dependencies/html5lib/sanitizer.py

@@ -2,11 +2,26 @@ from __future__ import absolute_import, division, unicode_literals
 
 import re
 from xml.sax.saxutils import escape, unescape
+from six.moves import urllib_parse as urlparse
 
 from .tokenizer import HTMLTokenizer
 from .constants import tokenTypes
 
 
+content_type_rgx = re.compile(r'''
+                               ^
+                               # Match a content type <application>/<type>
+                               (?P<content_type>[-a-zA-Z0-9.]+/[-a-zA-Z0-9.]+)
+                               # Match any character set and encoding
+                               (?:(?:;charset=(?:[-a-zA-Z0-9]+)(?:;(?:base64))?)
+                                 |(?:;(?:base64))?(?:;charset=(?:[-a-zA-Z0-9]+))?)
+                               # Assume the rest is data
+                               ,.*
+                               $
+                               ''',
+                              re.VERBOSE)
+
+
 class HTMLSanitizerMixin(object):
     """ sanitization of XHTML+MathML+SVG and of inline style attributes."""
 
@@ -100,8 +115,8 @@ class HTMLSanitizerMixin(object):
                       'xml:base', 'xml:lang', 'xml:space', 'xmlns', 'xmlns:xlink', 'y',
                       'y1', 'y2', 'zoomAndPan']
 
-    attr_val_is_uri = ['href', 'src', 'cite', 'action', 'longdesc', 'poster',
-                       'xlink:href', 'xml:base']
+    attr_val_is_uri = ['href', 'src', 'cite', 'action', 'longdesc', 'poster', 'background', 'datasrc',
+                       'dynsrc', 'lowsrc', 'ping', 'poster', 'xlink:href', 'xml:base']
 
     svg_attr_val_allows_ref = ['clip-path', 'color-profile', 'cursor', 'fill',
                                'filter', 'marker', 'marker-start', 'marker-mid', 'marker-end',
@@ -138,7 +153,9 @@ class HTMLSanitizerMixin(object):
     acceptable_protocols = ['ed2k', 'ftp', 'http', 'https', 'irc',
                             'mailto', 'news', 'gopher', 'nntp', 'telnet', 'webcal',
                             'xmpp', 'callto', 'feed', 'urn', 'aim', 'rsync', 'tag',
-                            'ssh', 'sftp', 'rtsp', 'afs']
+                            'ssh', 'sftp', 'rtsp', 'afs', 'data']
+
+    acceptable_content_types = ['image/png', 'image/jpeg', 'image/gif', 'image/webp', 'image/bmp', 'text/plain']
 
     # subclasses may define their own versions of these constants
     allowed_elements = acceptable_elements + mathml_elements + svg_elements
@@ -147,6 +164,7 @@ class HTMLSanitizerMixin(object):
     allowed_css_keywords = acceptable_css_keywords
     allowed_svg_properties = acceptable_svg_properties
     allowed_protocols = acceptable_protocols
+    allowed_content_types = acceptable_content_types
 
     # Sanitize the +html+, escaping all elements not in ALLOWED_ELEMENTS, and
     # stripping out all # attributes not in ALLOWED_ATTRIBUTES. Style
@@ -189,10 +207,17 @@ class HTMLSanitizerMixin(object):
                                        unescape(attrs[attr])).lower()
                 # remove replacement characters from unescaped characters
                 val_unescaped = val_unescaped.replace("\ufffd", "")
-                if (re.match("^[a-z0-9][-+.a-z0-9]*:", val_unescaped) and
-                    (val_unescaped.split(':')[0] not in
-                     self.allowed_protocols)):
-                    del attrs[attr]
+                uri = urlparse.urlparse(val_unescaped)
+                if uri:
+                    if uri.scheme not in self.allowed_protocols:
+                        del attrs[attr]
+                    if uri.scheme == 'data':
+                        m = content_type_rgx.match(uri.path)
+                        if not m:
+                            del attrs[attr]
+                        elif m.group('content_type') not in self.allowed_content_types:
+                            del attrs[attr]
+
             for attr in self.svg_attr_val_allows_ref:
                 if attr in attrs:
                     attrs[attr] = re.sub(r'url\s*\(\s*[^#\s][^)]+?\)',
@@ -245,7 +270,7 @@ class HTMLSanitizerMixin(object):
             elif prop.split('-')[0].lower() in ['background', 'border', 'margin',
                                                 'padding']:
                 for keyword in value.split():
-                    if not keyword in self.acceptable_css_keywords and \
+                    if keyword not in self.acceptable_css_keywords and \
                             not re.match("^(#[0-9a-f]+|rgb\(\d+%?,\d*%?,?\d*%?\)?|\d{0,2}\.?\d{0,2}(cm|em|ex|in|mm|pc|pt|px|%|,|\))?)$", keyword):
                         break
                 else:

included_dependencies/html5lib/serializer/htmlserializer.py

@@ -1,9 +1,6 @@
 from __future__ import absolute_import, division, unicode_literals
 from six import text_type
 
-import gettext
-_ = gettext.gettext
-
 try:
     from functools import reduce
 except ImportError:
@@ -35,7 +32,7 @@ else:
                 v = utils.surrogatePairToCodepoint(v)
             else:
                 v = ord(v)
-            if not v in encode_entity_map or k.islower():
+            if v not in encode_entity_map or k.islower():
                 # prefer < over < and similarly for &, >, etc.
                 encode_entity_map[v] = k
 
@@ -208,7 +205,7 @@ class HTMLSerializer(object):
                 if token["systemId"]:
                     if token["systemId"].find('"') >= 0:
                         if token["systemId"].find("'") >= 0:
-                            self.serializeError(_("System identifer contains both single and double quote characters"))
+                            self.serializeError("System identifer contains both single and double quote characters")
                         quote_char = "'"
                     else:
                         quote_char = '"'
@@ -220,7 +217,7 @@ class HTMLSerializer(object):
             elif type in ("Characters", "SpaceCharacters"):
                 if type == "SpaceCharacters" or in_cdata:
                     if in_cdata and token["data"].find("</") >= 0:
-                        self.serializeError(_("Unexpected </ in CDATA"))
+                        self.serializeError("Unexpected </ in CDATA")
                     yield self.encode(token["data"])
                 else:
                     yield self.encode(escape(token["data"]))
@@ -231,7 +228,7 @@ class HTMLSerializer(object):
                 if name in rcdataElements and not self.escape_rcdata:
                     in_cdata = True
                 elif in_cdata:
-                    self.serializeError(_("Unexpected child element of a CDATA element"))
+                    self.serializeError("Unexpected child element of a CDATA element")
                 for (attr_namespace, attr_name), attr_value in token["data"].items():
                     # TODO: Add namespace support here
                     k = attr_name
@@ -279,20 +276,20 @@ class HTMLSerializer(object):
                 if name in rcdataElements:
                     in_cdata = False
                 elif in_cdata:
-                    self.serializeError(_("Unexpected child element of a CDATA element"))
+                    self.serializeError("Unexpected child element of a CDATA element")
                 yield self.encodeStrict("</%s>" % name)
 
             elif type == "Comment":
                 data = token["data"]
                 if data.find("--") >= 0:
-                    self.serializeError(_("Comment contains --"))
+                    self.serializeError("Comment contains --")
                 yield self.encodeStrict("<!--%s-->" % token["data"])
 
             elif type == "Entity":
                 name = token["name"]
                 key = name + ";"
-                if not key in entities:
-                    self.serializeError(_("Entity %s not recognized" % name))
+                if key not in entities:
+                    self.serializeError("Entity %s not recognized" % name)
                 if self.resolve_entities and key not in xmlEntities:
                     data = entities[key]
                 else:

included_dependencies/html5lib/treebuilders/dom.py

@@ -158,7 +158,7 @@ def getDomBuilder(DomImplementation):
             else:
                 # HACK: allow text nodes as children of the document node
                 if hasattr(self.dom, '_child_node_types'):
-                    if not Node.TEXT_NODE in self.dom._child_node_types:
+                    if Node.TEXT_NODE not in self.dom._child_node_types:
                         self.dom._child_node_types = list(self.dom._child_node_types)
                         self.dom._child_node_types.append(Node.TEXT_NODE)
                 self.dom.appendChild(self.dom.createTextNode(data))

included_dependencies/html5lib/treewalkers/__init__.py

@@ -10,8 +10,12 @@ returning an iterator generating tokens.
 
 from __future__ import absolute_import, division, unicode_literals
 
+__all__ = ["getTreeWalker", "pprint", "dom", "etree", "genshistream", "lxmletree",
+           "pulldom"]
+
 import sys
 
+from .. import constants
 from ..utils import default_etree
 
 treeWalkerCache = {}
@@ -55,3 +59,89 @@ def getTreeWalker(treeType, implementation=None, **kwargs):
             # XXX: NEVER cache here, caching is done in the etree submodule
             return etree.getETreeModule(implementation, **kwargs).TreeWalker
     return treeWalkerCache.get(treeType)
+
+
+def concatenateCharacterTokens(tokens):
+    pendingCharacters = []
+    for token in tokens:
+        type = token["type"]
+        if type in ("Characters", "SpaceCharacters"):
+            pendingCharacters.append(token["data"])
+        else:
+            if pendingCharacters:
+                yield {"type": "Characters", "data": "".join(pendingCharacters)}
+                pendingCharacters = []
+            yield token
+    if pendingCharacters:
+        yield {"type": "Characters", "data": "".join(pendingCharacters)}
+
+
+def pprint(walker):
+    """Pretty printer for tree walkers"""
+    output = []
+    indent = 0
+    for token in concatenateCharacterTokens(walker):
+        type = token["type"]
+        if type in ("StartTag", "EmptyTag"):
+            # tag name
+            if token["namespace"] and token["namespace"] != constants.namespaces["html"]:
+                if token["namespace"] in constants.prefixes:
+                    ns = constants.prefixes[token["namespace"]]
+                else:
+                    ns = token["namespace"]
+                name = "%s %s" % (ns, token["name"])
+            else:
+                name = token["name"]
+            output.append("%s<%s>" % (" " * indent, name))
+            indent += 2
+            # attributes (sorted for consistent ordering)
+            attrs = token["data"]
+            for (namespace, localname), value in sorted(attrs.items()):
+                if namespace:
+                    if namespace in constants.prefixes:
+                        ns = constants.prefixes[namespace]
+                    else:
+                        ns = namespace
+                    name = "%s %s" % (ns, localname)
+                else:
+                    name = localname
+                output.append("%s%s=\"%s\"" % (" " * indent, name, value))
+            # self-closing
+            if type == "EmptyTag":
+                indent -= 2
+
+        elif type == "EndTag":
+            indent -= 2
+
+        elif type == "Comment":
+            output.append("%s<!-- %s -->" % (" " * indent, token["data"]))
+
+        elif type == "Doctype":
+            if token["name"]:
+                if token["publicId"]:
+                    output.append("""%s<!DOCTYPE %s "%s" "%s">""" %
+                                  (" " * indent,
+                                   token["name"],
+                                   token["publicId"],
+                                   token["systemId"] if token["systemId"] else ""))
+                elif token["systemId"]:
+                    output.append("""%s<!DOCTYPE %s "" "%s">""" %
+                                  (" " * indent,
+                                   token["name"],
+                                   token["systemId"]))
+                else:
+                    output.append("%s<!DOCTYPE %s>" % (" " * indent,
+                                                       token["name"]))
+            else:
+                output.append("%s<!DOCTYPE >" % (" " * indent,))
+
+        elif type == "Characters":
+            output.append("%s\"%s\"" % (" " * indent, token["data"]))
+
+        elif type == "SpaceCharacters":
+            assert False, "concatenateCharacterTokens should have got rid of all Space tokens"
+
+        else:
+            raise ValueError("Unknown token type, %s" % type)
+
+    return "\n".join(output)

included_dependencies/html5lib/treewalkers/_base.py

@@ -1,8 +1,8 @@
 from __future__ import absolute_import, division, unicode_literals
 from six import text_type, string_types
 
-import gettext
-_ = gettext.gettext
+__all__ = ["DOCUMENT", "DOCTYPE", "TEXT", "ELEMENT", "COMMENT", "ENTITY", "UNKNOWN",
+           "TreeWalker", "NonRecursiveTreeWalker"]
 
 from xml.dom import Node
 
@@ -58,7 +58,7 @@ class TreeWalker(object):
                "namespace": to_text(namespace),
                "data": attrs}
         if hasChildren:
-            yield self.error(_("Void element has children"))
+            yield self.error("Void element has children")
 
     def startTag(self, namespace, name, attrs):
         assert namespace is None or isinstance(namespace, string_types), type(namespace)
@@ -122,7 +122,7 @@ class TreeWalker(object):
         return {"type": "Entity", "name": text_type(name)}
 
     def unknown(self, nodeType):
-        return self.error(_("Unknown node type: ") + nodeType)
+        return self.error("Unknown node type: " + nodeType)
 
 
 class NonRecursiveTreeWalker(TreeWalker):

included_dependencies/html5lib/treewalkers/dom.py

@@ -2,9 +2,6 @@ from __future__ import absolute_import, division, unicode_literals
 
 from xml.dom import Node
 
-import gettext
-_ = gettext.gettext
-
 from . import _base
 
 

included_dependencies/html5lib/treewalkers/etree.py

@@ -7,8 +7,6 @@ except ImportError:
         from ordereddict import OrderedDict
     except ImportError:
         OrderedDict = dict
-import gettext
-_ = gettext.gettext
 
 import re
 

included_dependencies/html5lib/treewalkers/lxmletree.py

@@ -4,9 +4,6 @@ from six import text_type
 from lxml import etree
 from ..treebuilders.etree import tag_regexp
 
-from gettext import gettext
-_ = gettext
-
 from . import _base
 
 from .. import ihatexml
@@ -130,7 +127,7 @@ class TreeWalker(_base.NonRecursiveTreeWalker):
     def getNodeDetails(self, node):
         if isinstance(node, tuple):  # Text node
             node, key = node
-            assert key in ("text", "tail"), _("Text nodes are text or tail, found %s") % key
+            assert key in ("text", "tail"), "Text nodes are text or tail, found %s" % key
             return _base.TEXT, ensure_str(getattr(node, key))
 
         elif isinstance(node, Root):
@@ -169,7 +166,7 @@ class TreeWalker(_base.NonRecursiveTreeWalker):
                     attrs, len(node) > 0 or node.text)
 
     def getFirstChild(self, node):
-        assert not isinstance(node, tuple), _("Text nodes have no children")
+        assert not isinstance(node, tuple), "Text nodes have no children"
 
         assert len(node) or node.text, "Node has no children"
         if node.text:
@@ -180,7 +177,7 @@ class TreeWalker(_base.NonRecursiveTreeWalker):
     def getNextSibling(self, node):
         if isinstance(node, tuple):  # Text node
             node, key = node
-            assert key in ("text", "tail"), _("Text nodes are text or tail, found %s") % key
+            assert key in ("text", "tail"), "Text nodes are text or tail, found %s" % key
             if key == "text":
                 # XXX: we cannot use a "bool(node) and node[0] or None" construct here
                 # because node[0] might evaluate to False if it has no child element
@@ -196,7 +193,7 @@ class TreeWalker(_base.NonRecursiveTreeWalker):
     def getParentNode(self, node):
         if isinstance(node, tuple):  # Text node
             node, key = node
-            assert key in ("text", "tail"), _("Text nodes are text or tail, found %s") % key
+            assert key in ("text", "tail"), "Text nodes are text or tail, found %s" % key
             if key == "text":
                 return node
             # else: fallback to "normal" processing

included_dependencies/html5lib/utils.py

@@ -2,6 +2,8 @@ from __future__ import absolute_import, division, unicode_literals
 
 from types import ModuleType
 
+from six import text_type
+
 try:
     import xml.etree.cElementTree as default_etree
 except ImportError:
@@ -9,7 +11,26 @@ except ImportError:
 
 
 __all__ = ["default_etree", "MethodDispatcher", "isSurrogatePair",
-           "surrogatePairToCodepoint", "moduleFactoryFactory"]
+           "surrogatePairToCodepoint", "moduleFactoryFactory",
+           "supports_lone_surrogates"]
+
+
+# Platforms not supporting lone surrogates (\uD800-\uDFFF) should be
+# caught by the below test. In general this would be any platform
+# using UTF-16 as its encoding of unicode strings, such as
+# Jython. This is because UTF-16 itself is based on the use of such
+# surrogates, and there is no mechanism to further escape such
+# escapes.
+try:
+    _x = eval('"\\uD800"')
+    if not isinstance(_x, text_type):
+        # We need this with u"" because of http://bugs.jython.org/issue2039
+        _x = eval('u"\\uD800"')
+        assert isinstance(_x, text_type)
+except:
+    supports_lone_surrogates = False
+else:
+    supports_lone_surrogates = True
 
 
 class MethodDispatcher(dict):

setup.py

@@ -25,7 +25,7 @@ setup(
     # Versions should comply with PEP440.  For a discussion on single-sourcing
     # the version across setup.py and the project code, see
     # https://packaging.python.org/en/latest/single_source_version.html
-    version="2.2.4",
+    version="2.2.10",
 
     description='A tool for downloading fanfiction to eBook formats',
     long_description=long_description,

webservice/app.yaml

@@ -1,6 +1,6 @@
 # ffd-retief-hrd fanfictiondownloader fanficfare
 application: fanficfare
-version: 2-2-4
+version: 2-2-10
 runtime: python27
 api_version: 1
 threadsafe: true

webservice/index.html

@@ -46,7 +46,7 @@
           </p>
 	  <h3>Changes:</h3>
 	  <ul>
-            <li>More fixes for storiesonline.net site changes.</li>
+            <li>Updates for mediaminer.org changes.</li>
 	  </ul>
 	  <p>
 	    Questions?  Check out our 
@@ -56,7 +56,7 @@
 	    If you have any problems with this application, please
 	    report them in
 	    the <a href="http://groups.google.com/group/fanfic-downloader">FanFicFare Google Group</a>.  The
-	    <a href="http://2-2-3.fanficfare.appspot.com">previous version
+	    <a href="http://2-2-9.fanficfare.appspot.com">previous version
 	    </a> is also available for you to use if necessary.
 	  </p>
 	  <div id='error'>