Update translations, bump versions.

Add MassEffect2.in Adapter

MANIFEST.in

@@ -0,0 +1 @@
+include DESCRIPTION.rst

calibre-plugin/__init__.py

@@ -42,7 +42,7 @@ class FanFicFareBase(InterfaceActionBase):
     description         = _('UI plugin to download FanFiction stories from various sites.')
     supported_platforms = ['windows', 'osx', 'linux']
     author              = 'Jim Miller'
-    version             = (2, 2, 5)
+    version             = (2, 2, 12)
     minimum_calibre_version = (1, 48, 0)
 
     #: This field defines the GUI plugin class that contains all the code

calibre-plugin/basicinihighlighter.py

@@ -1,5 +1,5 @@
-#!/usr/bin/env python
-# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+# -*- coding: utf-8 -*-
+
 from __future__ import (unicode_literals, division,
                         print_function)
 

calibre-plugin/common_utils.py

@@ -1,5 +1,5 @@
-#!/usr/bin/env python
-# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+# -*- coding: utf-8 -*-
+
 from __future__ import (unicode_literals, division, absolute_import,
                         print_function)
 

calibre-plugin/config.py

@@ -1,5 +1,5 @@
-#!/usr/bin/env python
-# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+# -*- coding: utf-8 -*-
+
 from __future__ import (unicode_literals, division, absolute_import,
                         print_function)
 
@@ -55,6 +55,9 @@ try:
 except NameError:
     pass # load_translations() added in calibre 1.9
 
+from calibre.library.field_metadata import FieldMetadata
+field_metadata = FieldMetadata()
+
 # There are a number of things used several times that shouldn't be
 # translated.  This is just a way to make that easier by keeping them
 # out of the _() strings.
@@ -75,6 +78,8 @@ no_trans = { 'pini':'personal.ini',
              'p':'password',
              }
 
+STD_COLS_SKIP = ['size','cover','news','ondevice','path','series_sort','sort']
+
 from calibre_plugins.fanficfare_plugin.prefs \
     import (prefs, PREFS_NAMESPACE, updatecalcover_order, calcover_save_options,
             gencalcover_order, SAVE_YES, SAVE_NO)
@@ -84,7 +89,7 @@ from calibre_plugins.fanficfare_plugin.dialogs \
             EditTextDialog, IniTextDialog, RejectUrlEntry)
     
 from calibre_plugins.fanficfare_plugin.fanficfare.adapters \
-    import getConfigSections
+    import getSiteSections
 
 from calibre_plugins.fanficfare_plugin.common_utils \
     import ( KeyboardConfigDialog, PrefsViewerDialog )
@@ -254,6 +259,7 @@ class ConfigWidget(QWidget):
         prefs['checkforseriesurlid'] = self.basic_tab.checkforseriesurlid.isChecked()
         prefs['checkforurlchange'] = self.basic_tab.checkforurlchange.isChecked()
         prefs['injectseries'] = self.basic_tab.injectseries.isChecked()
+        prefs['matchtitleauth'] = self.basic_tab.matchtitleauth.isChecked()
         prefs['smarten_punctuation'] = self.basic_tab.smarten_punctuation.isChecked()
         prefs['reject_always'] = self.basic_tab.reject_always.isChecked()
 
@@ -266,6 +272,7 @@ class ConfigWidget(QWidget):
             prefs['addtolists'] = self.readinglist_tab.addtolists.isChecked()
             prefs['addtoreadlists'] = self.readinglist_tab.addtoreadlists.isChecked()
             prefs['addtolistsonread'] = self.readinglist_tab.addtolistsonread.isChecked()
+            prefs['autounnew'] = self.readinglist_tab.autounnew.isChecked()
 
         # personal.ini
         ini = self.personalini_tab.personalini
@@ -275,6 +282,8 @@ class ConfigWidget(QWidget):
             # if they've removed everything, reset to default.
             prefs['personal.ini'] = get_resources('plugin-example.ini')
 
+        prefs['cal_cols_pass_in'] = self.personalini_tab.cal_cols_pass_in.isChecked()
+            
         # Covers tab
         prefs['updatecalcover'] = calcover_save_options[unicode(self.calibrecover_tab.updatecalcover.currentText())]
         # for backward compatibility:
@@ -316,10 +325,15 @@ class ConfigWidget(QWidget):
             colsnewonly[col] = checkbox.isChecked()
         prefs['std_cols_newonly'] = colsnewonly
 
+        prefs['set_author_url'] =self.std_columns_tab.set_author_url.isChecked()
+
         # Custom Columns tab
         # error column
         prefs['errorcol'] = unicode(convert_qvariant(self.cust_columns_tab.errorcol.itemData(self.cust_columns_tab.errorcol.currentIndex())))
 
+        # metadata column
+        prefs['savemetacol'] = unicode(convert_qvariant(self.cust_columns_tab.savemetacol.itemData(self.cust_columns_tab.savemetacol.currentIndex())))
+
         # cust cols tab
         colsmap = {}
         for (col,combo) in self.cust_columns_tab.custcol_dropdowns.iteritems():
@@ -336,12 +350,14 @@ class ConfigWidget(QWidget):
         
         prefs['allow_custcol_from_ini'] = self.cust_columns_tab.allow_custcol_from_ini.isChecked()
 
-        prefs['imapserver'] = unicode(self.imap_tab.imapserver.text())
-        prefs['imapuser'] = unicode(self.imap_tab.imapuser.text())
-        prefs['imappass'] = unicode(self.imap_tab.imappass.text())
-        prefs['imapfolder'] = unicode(self.imap_tab.imapfolder.text())
+        prefs['imapserver'] = unicode(self.imap_tab.imapserver.text()).strip()
+        prefs['imapuser'] = unicode(self.imap_tab.imapuser.text()).strip()
+        prefs['imappass'] = unicode(self.imap_tab.imappass.text()).strip()
+        prefs['imapfolder'] = unicode(self.imap_tab.imapfolder.text()).strip()
         prefs['imapmarkread'] = self.imap_tab.imapmarkread.isChecked()
         prefs['imapsessionpass'] = self.imap_tab.imapsessionpass.isChecked()
+        prefs['auto_reject_from_email'] = self.imap_tab.auto_reject_from_email.isChecked()
+        prefs['download_from_email_immediately'] = self.imap_tab.download_from_email_immediately.isChecked()
         
         prefs.save_to_db()
 
@@ -508,6 +524,11 @@ class BasicTab(QWidget):
         self.injectseries.setChecked(prefs['injectseries'])
         self.l.addWidget(self.injectseries)
 
+        self.matchtitleauth = QCheckBox(_("Search by Title/Author(s) for If Story Already Exists?"),self)
+        self.matchtitleauth.setToolTip(_("When checking <i>If Story Already Exists</i> FanFicFare will first match by URL Identifier.  But if not found, it can also search existing books by Title and Author(s)."))
+        self.matchtitleauth.setChecked(prefs['matchtitleauth'])
+        self.l.addWidget(self.matchtitleauth)
+
         rej_gb = groupbox = QGroupBox(_("Reject List"))
         self.l = QVBoxLayout()
         groupbox.setLayout(self.l)
@@ -637,15 +658,25 @@ class PersonalIniTab(QWidget):
         self.ini_button.clicked.connect(self.add_ini_button)
         self.l.addWidget(self.ini_button)
         
+        label = QLabel(_("Changes will only be saved if you click 'OK' to leave Customize FanFicFare."))
+        label.setWordWrap(True)
+        self.l.addWidget(label)
+        
         self.defaults = QPushButton(_('View Defaults')+' (plugin-defaults.ini)', self)
         self.defaults.setToolTip(_("View all of the plugin's configurable settings\nand their default settings."))
         self.defaults.clicked.connect(self.show_defaults)
         self.l.addWidget(self.defaults)
 
-        label = QLabel(_("Changes will only be saved if you click 'OK' to leave Customize FanFicFare."))
-        label.setWordWrap(True)
-        self.l.addWidget(label)
+        self.cal_cols_pass_in = QCheckBox(_('Pass Calibre Columns into FanFicFare on Update/Overwrite')%no_trans,self)
+        self.cal_cols_pass_in.setToolTip(_("If checked, when updating/overwriting an existing book, FanFicFare will have the Calibre Columns available to use in replace_metadata, title_page, etc.<br>Click the button below to see the Calibre Column namess.")%no_trans)
+        self.cal_cols_pass_in.setChecked(prefs['cal_cols_pass_in'])
+        self.l.addWidget(self.cal_cols_pass_in)
         
+        self.showcalcols = QPushButton(_('Show Calibre Column Names'), self)
+        self.showcalcols.setToolTip(_("FanFicFare can pass the Calibre Columns into the download/update process.<br>This will show you the columns available by name."))
+        self.showcalcols.clicked.connect(self.show_showcalcols)
+        self.l.addWidget(self.showcalcols)
+
         self.l.insertStretch(-1)
         # let edit box fill the space.
         
@@ -671,6 +702,27 @@ class PersonalIniTab(QWidget):
         if d.result() == d.Accepted:
             self.personalini = d.get_plain_text()
                 
+    def show_showcalcols(self):
+        lines=[]#[('calibre_std_user_categories',_('User Categories'))]
+        for k,f in field_metadata.iteritems():
+            if f['name'] and k not in STD_COLS_SKIP: # only if it has a human readable name.
+                lines.append(('calibre_std_'+k,f['name']))
+            
+        for k, column in self.plugin_action.gui.library_view.model().custom_columns.iteritems():
+            if k != prefs['savemetacol']:
+                # custom always have name.
+                lines.append(('calibre_cust_'+k[1:],column['name']))
+
+        lines.sort() # sort by key.
+
+        EditTextDialog(self,
+                       '\n'.join(['%s (%s)'%(l,k) for (k,l) in lines]),
+                       icon=self.windowIcon(),
+                       title=_('Calibre Column Entry Names'),
+                       label=_('Label (entry_name)'),
+                       read_only=True,
+                       save_size_name='fff:showcalcols').exec_()
+        
 class ReadingListTab(QWidget):
 
     def __init__(self, parent_dialog, plugin_action):
@@ -731,6 +783,11 @@ class ReadingListTab(QWidget):
         self.addtolistsonread.setChecked(prefs['addtolistsonread'])
         self.l.addWidget(self.addtolistsonread)
             
+        self.autounnew = QCheckBox(_('Automatically run Remove "New" Chapter Marks when marking books "Read".'),self)
+        self.autounnew.setToolTip(_('Menu option to remove from "To Read" lists will also remove "(new)" chapter marks created by personal.ini <i>mark_new_chapters</i> setting.'))
+        self.autounnew.setChecked(prefs['autounnew'])
+        self.l.addWidget(self.autounnew)
+            
         self.l.insertStretch(-1)
         
 class CalibreCoverTab(QWidget):
@@ -878,7 +935,7 @@ class CalibreCoverTab(QWidget):
         
         self.gc_dropdowns = {}
 
-        sitelist = getConfigSections()
+        sitelist = getSiteSections()
         sitelist.sort()
         sitelist.insert(0,_("Default"))
         for site in sitelist:
@@ -1188,7 +1245,6 @@ class CustomColumnsTab(QWidget):
         self.allow_custcol_from_ini.setChecked(prefs['allow_custcol_from_ini'])
         self.l.addWidget(self.allow_custcol_from_ini)
         
-        self.l.addSpacing(5)
         label = QLabel(_("Special column:"))
         label.setWordWrap(True)
         self.l.addWidget(label)
@@ -1207,6 +1263,21 @@ class CustomColumnsTab(QWidget):
         self.errorcol.setCurrentIndex(self.errorcol.findData(prefs['errorcol']))
         horz.addWidget(self.errorcol)
         self.l.addLayout(horz)
+        
+        horz = QHBoxLayout()
+        label = QLabel(_("Saved Metadata Column:"))
+        tooltip=_("If set, FanFicFare will save a copy of all its metadata in this column when the book is downloaded or updated.<br/>The metadata from this column can later be used to update custom columns without having to request the metadata from the server again.<br/>(Long Text columns only.)")
+        label.setToolTip(tooltip)
+        horz.addWidget(label)
+        self.savemetacol = QComboBox(self)
+        self.savemetacol.setToolTip(tooltip)
+        self.savemetacol.addItem('','')
+        for key, column in custom_columns.iteritems():
+            if column['datatype'] in ('comments'):
+                self.savemetacol.addItem(column['name'],key)
+        self.savemetacol.setCurrentIndex(self.savemetacol.findData(prefs['savemetacol']))
+        horz.addWidget(self.savemetacol)
+        self.l.addLayout(horz)
 
         #print("prefs['custom_cols'] %s"%prefs['custom_cols'])
 
@@ -1255,6 +1326,17 @@ class StandardColumnsTab(QWidget):
             horz.addWidget(newonlycheck)
             
             self.l.addLayout(horz)
+            
+        self.l.addSpacing(5)
+        label = QLabel(_("Other Standard Column Options"))
+        label.setWordWrap(True)
+        self.l.addWidget(label)
+        self.l.addSpacing(5)
+
+        self.set_author_url = QCheckBox(_('Set Calibre Author URL'),self)
+        self.set_author_url.setToolTip(_("Set Calibre Author URL to Author's URL on story site."))
+        self.set_author_url.setChecked(prefs['set_author_url'])
+        self.l.addWidget(self.set_author_url)
         
         self.l.insertStretch(-1)
 
@@ -1327,6 +1409,18 @@ class ImapTab(QWidget):
         self.l.addWidget(self.imapmarkread,row,0,1,-1)
         row+=1
             
+        self.auto_reject_from_email = QCheckBox(_('Discard URLs on Reject List'),self)
+        self.auto_reject_from_email.setToolTip(_('If checked, FanFicFare will silently discard story URLs from emails that are on your Reject URL List.<br>Otherwise they will appear and you will see the normal Reject URL dialog.<br>The Emails will still be marked Read if configured to.'))
+        self.auto_reject_from_email.setChecked(prefs['auto_reject_from_email'])
+        self.l.addWidget(self.auto_reject_from_email,row,0,1,-1)
+        row+=1
+        
+        self.download_from_email_immediately = QCheckBox(_('Download from Email Immediately'),self)
+        self.download_from_email_immediately.setToolTip(_('If checked, FanFicFare will start downloading story URLs from emails immediately.<br>Otherwise the usual Download from URLs dialog will appear.'))
+        self.download_from_email_immediately.setChecked(prefs['download_from_email_immediately'])
+        self.l.addWidget(self.download_from_email_immediately,row,0,1,-1)
+        row+=1
+        
         label = QLabel(_("<b>It's safest if you create a separate email account that you use only "
                          "for your story update notices.  FanFicFare and calibre cannot guarantee that "
                          "malicious code cannot get your email password once you've entered it. "

calibre-plugin/dialogs.py

@@ -1,5 +1,5 @@
-#!/usr/bin/env python
-# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+# -*- coding: utf-8 -*-
+
 from __future__ import (unicode_literals, division,
                         print_function)
 
@@ -79,14 +79,16 @@ UPDATE=_('Update EPUB if New Chapters')
 UPDATEALWAYS=_('Update EPUB Always')
 OVERWRITE=_('Overwrite if Newer')
 OVERWRITEALWAYS=_('Overwrite Always')
-CALIBREONLY=_('Update Calibre Metadata Only')
+CALIBREONLY=_('Update Calibre Metadata from Web Site')
+CALIBREONLYSAVECOL=_('Update Calibre Metadata from Saved Metadata Column')
 collision_order=[SKIP,
                  ADDNEW,
                  UPDATE,
                  UPDATEALWAYS,
                  OVERWRITE,
                  OVERWRITEALWAYS,
-                 CALIBREONLY,]
+                 CALIBREONLY,
+                 CALIBREONLYSAVECOL,]
 
 # best idea I've had for how to deal with config/pref saving the
 # collision name in english.
@@ -97,6 +99,7 @@ SAVE_UPDATEALWAYS='Update EPUB Always'
 SAVE_OVERWRITE='Overwrite if Newer'
 SAVE_OVERWRITEALWAYS='Overwrite Always'
 SAVE_CALIBREONLY='Update Calibre Metadata Only'
+SAVE_CALIBREONLYSAVECOL='Update Calibre Metadata Only(Saved Column)'
 save_collisions={
     SKIP:SAVE_SKIP,
     ADDNEW:SAVE_ADDNEW,
@@ -112,6 +115,7 @@ save_collisions={
     SAVE_OVERWRITE:OVERWRITE,
     SAVE_OVERWRITEALWAYS:OVERWRITEALWAYS,
     SAVE_CALIBREONLY:CALIBREONLY,
+    SAVE_CALIBREONLYSAVECOL:CALIBREONLYSAVECOL,
     }
     
 anthology_collision_order=[UPDATE,
@@ -254,6 +258,7 @@ class AddNewDialog(SizePersistedDialog):
         self.l.addWidget(self.url)
 
         self.merge = self.newmerge = False
+        self.extraoptions = {}
         
         # elements to hide when doing merge.
         self.mergehide = []
@@ -303,7 +308,7 @@ class AddNewDialog(SizePersistedDialog):
         self.collision.setToolTip("CollisionToolTip")
         # add collision options
         self.set_collisions()
-        i = self.collision.findText(save_collisions[prefs['collision']])
+        i = self.collision.findText(save_collisions[self.prefs['collision']])
         if i > -1:
             self.collision.setCurrentIndex(i)
         self.collisionlabel.setBuddy(self.collision)
@@ -317,14 +322,14 @@ class AddNewDialog(SizePersistedDialog):
         horz = QHBoxLayout()
         self.updatemeta = QCheckBox(_('Update Calibre &Metadata?'),self)
         self.updatemeta.setToolTip(_("Update metadata for existing stories in Calibre from web site?\n(Columns set to 'New Only' in the column tabs will only be set for new books.)"))
-        self.updatemeta.setChecked(prefs['updatemeta'])
+        self.updatemeta.setChecked(self.prefs['updatemeta'])
         horz.addWidget(self.updatemeta)
         self.mergehide.append(self.updatemeta)
         self.mergeupdateshow.append(self.updatemeta)
 
         self.updateepubcover = QCheckBox(_('Update EPUB Cover?'),self)
         self.updateepubcover.setToolTip(_('Update book cover image from site or defaults (if found) <i>inside</i> the EPUB when EPUB is updated.'))
-        self.updateepubcover.setChecked(prefs['updateepubcover'])
+        self.updateepubcover.setChecked(self.prefs['updateepubcover'])
         horz.addWidget(self.updateepubcover)
         self.mergehide.append(self.updateepubcover)
         
@@ -354,7 +359,10 @@ class AddNewDialog(SizePersistedDialog):
                     extrapayload=None):
         # rather than mutex in fff_plugin, just bail here if it's
         # already in use.
-        if self.isVisible(): return
+        if self.isVisible():
+            if url_list_text: # add to open box.
+               self.url.setText( '\n'.join([self.get_urlstext(), url_list_text]) )
+            return
 
         try:
             self.go_signal.disconnect()
@@ -427,12 +435,19 @@ class AddNewDialog(SizePersistedDialog):
         prev=self.collision.currentText()
         self.collision.clear()
         if self.merge:
-            order = anthology_collision_order
+            order = list(anthology_collision_order)
         else:
-            order = collision_order
+            order = list(collision_order)
+            ## Remove options that aren't valid.
+            if self.fileform.currentText() != 'epub':
+                order.remove(UPDATE)
+                order.remove(UPDATEALWAYS)
+            if self.prefs['savemetacol'] == '':
+                order.remove(CALIBREONLYSAVECOL)
+
         for o in order:
-            if self.merge or self.fileform.currentText() == 'epub' or o not in [UPDATE,UPDATEALWAYS]:
-                self.collision.addItem(o)
+            self.collision.addItem(o)
+            
         i = self.collision.findText(prev)
         if i > -1:
             self.collision.setCurrentIndex(i)
@@ -783,7 +798,7 @@ class UpdateExistingDialog(SizePersistedDialog):
         self.fileform.addItem('mobi')
         self.fileform.addItem('html')
         self.fileform.addItem('txt')
-        self.fileform.setCurrentIndex(self.fileform.findText(prefs['fileform']))
+        self.fileform.setCurrentIndex(self.fileform.findText(self.prefs['fileform']))
         self.fileform.setToolTip(_('Choose output format to create.  May set default from plugin configuration.'))
         self.fileform.activated.connect(self.set_collisions)
         label.setBuddy(self.fileform)
@@ -795,7 +810,7 @@ class UpdateExistingDialog(SizePersistedDialog):
         self.collision.setToolTip(_("What sort of update to perform.  May set default from plugin configuration."))
         # add collision options
         self.set_collisions()
-        i = self.collision.findText(save_collisions[prefs['collision']])
+        i = self.collision.findText(save_collisions[self.prefs['collision']])
         if i > -1:
             self.collision.setCurrentIndex(i)
         label.setBuddy(self.collision)
@@ -803,12 +818,12 @@ class UpdateExistingDialog(SizePersistedDialog):
 
         self.updatemeta = QCheckBox(_('Update Calibre &Metadata?'),self)
         self.updatemeta.setToolTip(_("Update metadata for existing stories in Calibre from web site?\n(Columns set to 'New Only' in the column tabs will only be set for new books.)"))
-        self.updatemeta.setChecked(prefs['updatemeta'])
+        self.updatemeta.setChecked(self.prefs['updatemeta'])
         gbl.addWidget(self.updatemeta)
                 
         self.updateepubcover = QCheckBox(_('Update EPUB Cover?'),self)
         self.updateepubcover.setToolTip(_('Update book cover image from site or defaults (if found) <i>inside</i> the EPUB when EPUB is updated.'))
-        self.updateepubcover.setChecked(prefs['updateepubcover'])
+        self.updateepubcover.setChecked(self.prefs['updateepubcover'])
         gbl.addWidget(self.updateepubcover)
 
 
@@ -827,10 +842,18 @@ class UpdateExistingDialog(SizePersistedDialog):
     def set_collisions(self):
         prev=self.collision.currentText()
         self.collision.clear()
-        for o in collision_order:
-            if o not in [ADDNEW,SKIP] and \
-                    (self.fileform.currentText() == 'epub' or o not in [UPDATE,UPDATEALWAYS]):
-                self.collision.addItem(o)        
+        order = list(collision_order)
+        order.remove(ADDNEW)
+        order.remove(SKIP)
+        if self.fileform.currentText() != 'epub':
+            order.remove(UPDATE)
+            order.remove(UPDATEALWAYS)
+        if self.prefs['savemetacol'] == '':
+            order.remove(CALIBREONLYSAVECOL)
+            
+        for o in order:
+            self.collision.addItem(o)
+            
         i = self.collision.findText(prev)
         if i > -1:
             self.collision.setCurrentIndex(i)
@@ -1132,6 +1155,7 @@ class EditTextDialog(SizePersistedDialog):
 
     def __init__(self, parent, text,
                  icon=None, title=None, label=None, tooltip=None,
+                 read_only=False,
                  rejectreasons=[],reasonslabel=None,
                  save_size_name='fff:edit text dialog',
                  ):
@@ -1148,6 +1172,7 @@ class EditTextDialog(SizePersistedDialog):
         
         self.textedit = QTextEdit(self)
         self.textedit.setLineWrapMode(QTextEdit.NoWrap)
+        self.textedit.setReadOnly(read_only)
         self.textedit.setText(text)
         self.l.addWidget(self.textedit)
 

calibre-plugin/fff_plugin.py

@@ -1,5 +1,5 @@
-#!/usr/bin/env python
-# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+# -*- coding: utf-8 -*-
+
 from __future__ import (unicode_literals, division, absolute_import,
                         print_function)
 
@@ -13,24 +13,24 @@ logger = logging.getLogger(__name__)
 import time, os, copy, threading, re, platform, sys
 from StringIO import StringIO
 from functools import partial
-from datetime import datetime, time
+from datetime import datetime, time, date
 from string import Template
 import urllib
 import email
 import traceback
 
 try:
-    from PyQt5.Qt import (QApplication, QMenu, QTimer)
+    from PyQt5.Qt import (QApplication, QMenu, QTimer, QCursor, Qt)
     from PyQt5.QtCore import QBuffer
 except ImportError as e:
-    from PyQt4.Qt import (QApplication, QMenu, QTimer)
+    from PyQt4.Qt import (QApplication, QMenu, QTimer, QCursor, Qt)
     from PyQt4.QtCore import QBuffer
 
 from calibre.constants import numeric_version as calibre_version
 
 from calibre.ptempfile import PersistentTemporaryFile, PersistentTemporaryDirectory, remove_dir
 from calibre.ebooks.metadata import MetaInformation
-from calibre.ebooks.metadata.meta import get_metadata
+from calibre.ebooks.metadata.meta import get_metadata as calibre_get_metadata
 from calibre.gui2 import error_dialog, warning_dialog, question_dialog, info_dialog
 from calibre.gui2.dialogs.message_box import ViewLog
 from calibre.gui2.dialogs.confirm_delete import confirm
@@ -55,21 +55,39 @@ try:
 except:
     HAS_CALGC=False
 
-from calibre_plugins.fanficfare_plugin.common_utils import (set_plugin_icon_resources, get_icon,
-                                                            create_menu_action_unique, get_library_uuid)
+from calibre.library.field_metadata import FieldMetadata
+field_metadata = FieldMetadata()
+    
+from calibre_plugins.fanficfare_plugin.common_utils import (
+    set_plugin_icon_resources, get_icon, create_menu_action_unique,
+    get_library_uuid)
 
-from calibre_plugins.fanficfare_plugin.fanficfare import adapters, exceptions
-from calibre_plugins.fanficfare_plugin.fanficfare.epubutils import get_dcsource, get_dcsource_chaptercount, get_story_url_from_html
-from calibre_plugins.fanficfare_plugin.fanficfare.geturls import get_urls_from_page, get_urls_from_html, get_urls_from_text, get_urls_from_imap
+from calibre_plugins.fanficfare_plugin.fanficfare import (
+    adapters, exceptions)
+
+from calibre_plugins.fanficfare_plugin.fanficfare.epubutils import (
+    get_dcsource, get_dcsource_chaptercount, get_story_url_from_html,
+    reset_orig_chapters_epub)
+
+from calibre_plugins.fanficfare_plugin.fanficfare.geturls import (
+    get_urls_from_page, get_urls_from_html,get_urls_from_text,
+    get_urls_from_imap)
+
+from calibre_plugins.fanficfare_plugin.fff_util import (
+    get_fff_adapter, get_fff_config, get_fff_personalini)
+
+from calibre_plugins.fanficfare_plugin.config import (
+    permitted_values, rejecturllist, STD_COLS_SKIP)
+
+from calibre_plugins.fanficfare_plugin.prefs import (
+    prefs, SAVE_YES, SAVE_NO, SAVE_YES_IF_IMG, SAVE_YES_UNLESS_IMG)
 
-from calibre_plugins.fanficfare_plugin.fff_util import (get_fff_adapter, get_fff_config, get_fff_personalini)
-from calibre_plugins.fanficfare_plugin.config import (permitted_values, rejecturllist)
-from calibre_plugins.fanficfare_plugin.prefs import (prefs, SAVE_YES, SAVE_NO,
-                                                     SAVE_YES_IF_IMG, SAVE_YES_UNLESS_IMG)
 from calibre_plugins.fanficfare_plugin.dialogs import (
     AddNewDialog, UpdateExistingDialog,
-    LoopProgressDialog, UserPassDialog, AboutDialog, CollectURLDialog, RejectListDialog, EmailPassDialog,
+    LoopProgressDialog, UserPassDialog, AboutDialog, CollectURLDialog,
+    RejectListDialog, EmailPassDialog,
     OVERWRITE, OVERWRITEALWAYS, UPDATE, UPDATEALWAYS, ADDNEW, SKIP, CALIBREONLY,
+    CALIBREONLYSAVECOL,
     NotGoingToDownload, RejectUrlEntry )
 
 # because calibre immediately transforms html into zip and don't want
@@ -318,6 +336,12 @@ class FanFicFarePlugin(InterfaceAction):
                                                                       image='minusminus.png',
                                                                       triggered=partial(self.update_lists,add=False))
 
+            self.menu.addSeparator()
+            self.get_list_action = self.create_menu_item_ex(self.menu, _('Remove "New" Chapter Marks from Selected books'),
+                                                            unique_name='Remove "(new)" chapter marks created by personal.ini <i>mark_new_chapters</i> setting.',
+                                                            image='edit-undo.png',
+                                                            triggered=self.unnew_books)
+
             self.menu.addSeparator()
             self.get_list_action = self.create_menu_item_ex(self.menu, _('Get Story URLs from Selected Books'),
                                                             unique_name='Get URLs from Selected Books',
@@ -398,6 +422,8 @@ class FanFicFarePlugin(InterfaceAction):
                 return
 
             self.update_reading_lists(self.gui.library_view.get_selected_ids(),add)
+            if not add and prefs['autounnew']:
+                self.unnew_books()
 
     def get_urls_from_imap_menu(self):
 
@@ -421,20 +447,42 @@ class FanFicFarePlugin(InterfaceAction):
             if prefs['imapsessionpass']:
                 self.imap_pass = imap_pass
 
+        self.busy_cursor()
+        self.gui.status_bar.show_message(_('Fetching Story URLs from Email...'))
         url_list = get_urls_from_imap(prefs['imapserver'],
                                       prefs['imapuser'],
                                       imap_pass,
                                       prefs['imapfolder'],
                                       prefs['imapmarkread'],)
+        reject_list=set()
+        if prefs['auto_reject_from_email']:
+            # need to normalize for reject list.
+            reject_list = set([x for x in url_list if rejecturllist.check(adapters.getNormalStoryURLSite(x)[0])])
+        url_list = url_list - reject_list
 
-        if url_list:
-            self.add_dialog("\n".join(url_list),merge=False)
+        self.gui.status_bar.show_message(_('No Valid Story URLs Found in Unread Emails.'),3000)
+        self.restore_cursor()
+
+        if prefs['download_from_email_immediately']:
+            ## do imap fetch w/o GUI elements
+            if url_list:
+                self.prep_downloads(self.add_new_dialog.get_fff_options(),
+                                    "\n".join(url_list))
+            else:
+                self.gui.status_bar.show_message(_('Finished Fetching Story URLs from Email.'),3000)
+                
         else:
-            info_dialog(self.gui, _('Get Story URLs from Email'),
-                        _('No Valid Story URLs Found in Unread Emails.'),
-                        show=True,
-                        show_copy_button=False)
-
+            if url_list:
+                self.add_dialog("\n".join(url_list),merge=False)
+            else:            
+                msg = _('No Valid Story URLs Found in Unread Emails.')
+                if reject_list:
+                    msg = msg + '<p>'+(_('(%d Story URLs Skipped, on Rejected URL List)')%len(reject_list))+'</p>'
+                info_dialog(self.gui, _('Get Story URLs from Email'),
+                            msg,
+                            show=True,
+                            show_copy_button=False)
+    
     def get_urls_from_page_menu(self,anthology=False):
 
         urltxt = ""
@@ -452,8 +500,14 @@ class FanFicFarePlugin(InterfaceAction):
             return
         url = u"%s"%d.url.text()
 
+        self.busy_cursor()
+        self.gui.status_bar.show_message(_('Fetching Story URLs from Page...'))
+
         url_list = self.get_urls_from_page(url)
 
+        self.gui.status_bar.show_message(_('Finished Fetching Story URLs from Page.'),3000)
+        self.restore_cursor()
+        
         if url_list:
             self.add_dialog("\n".join(url_list),merge=d.anthology,anthology_url=url)
         else:
@@ -518,6 +572,70 @@ class FanFicFarePlugin(InterfaceAction):
                         show=True,
                         show_copy_button=False)
 
+    def unnew_books(self):
+        '''Get list of URLs from existing books.'''
+        if not self.is_library_view():
+            self.gui.status_bar.show_message(_('Can only UnNew books in library'),
+                                             3000)
+            return
+        
+        if not self.gui.current_view().selectionModel().selectedRows() :
+            self.gui.status_bar.show_message(_('No Selected Books to Get URLs From'),
+                                             3000)
+            return
+
+        book_list = map( partial(self.make_book_id_only),
+                         self.gui.library_view.get_selected_ids() )
+
+        tdir = PersistentTemporaryDirectory(prefix='fanficfare_')
+        LoopProgressDialog(self.gui,
+                           book_list,
+                           partial(self.get_unnew_books_loop, db=self.gui.current_db, tdir=tdir),
+                           partial(self.get_unnew_books_finish, tdir=tdir),
+                           init_label=_("UnNewing books..."),
+                           win_title=_("UnNew Books"),
+                           status_prefix=_("Books UnNewed"))
+
+    def get_unnew_books_loop(self,book,db=None,tdir=None):
+        
+        if book['calibre_id'] and db.has_format(book['calibre_id'],'EPUB',index_is_id=True):
+            tmp = PersistentTemporaryFile(prefix='%s-'%book['calibre_id'],
+                                          suffix='.epub',
+                                          dir=tdir)
+            db.copy_format_to(book['calibre_id'],'EPUB',tmp,index_is_id=True)
+            
+            unnewtmp = PersistentTemporaryFile(prefix='unnew-%s-'%book['calibre_id'],
+                                               suffix='.epub',
+                                               dir=tdir)
+            book['changed']=reset_orig_chapters_epub(tmp,unnewtmp)
+            if book['changed']:
+                db.add_format_with_hooks(book['calibre_id'],
+                                         'EPUB',
+                                         unnewtmp,
+                                         index_is_id=True)
+                if prefs['deleteotherforms']:
+                    fmts = db.formats(book['calibre_id'], index_is_id=True).split(',')
+                    for fmt in fmts:
+                        if fmt.lower() != formmapping['epub'].lower():
+                            logger.debug("deleteotherforms remove f:"+fmt)
+                            db.remove_format(book['calibre_id'], fmt, index_is_id=True)#, notify=False
+                elif prefs['autoconvert']:
+                    ## 'Convert Book'.auto_convert_auto_add doesn't convert if
+                    ## the format is already there.
+                    fmt = calibre_prefs['output_format']
+                    # delete if there, but not if the format we just made.
+                    if fmt.lower() != 'epub' and db.has_format(book['calibre_id'],fmt,index_is_id=True):
+                        logger.debug("autoconvert remove f:"+fmt)
+                        db.remove_format(book['calibre_id'], fmt, index_is_id=True)#, notify=False
+                
+    def get_unnew_books_finish(self, book_list, tdir=None):
+        remove_dir(tdir)
+        if prefs['autoconvert']:
+            changed_ids = [ x['calibre_id'] for x in book_list if x['changed'] ]
+            if changed_ids:
+                self.gui.status_bar.show_message(_('Starting auto conversion of %d books.')%(len(changed_ids)), 3000)
+                self.gui.iactions['Convert Books'].auto_convert_auto_add(changed_ids)
+
     def reject_list_urls(self):
         if self.is_library_view():
             book_list = map( partial(self.make_book_id_only),
@@ -638,6 +756,9 @@ class FanFicFarePlugin(InterfaceAction):
             remove_dir(tdir)
             return
 
+        self.busy_cursor()
+        self.gui.status_bar.show_message(_('Fetching Story URLs for Series...'))
+        
         # get list from identifiers:url/uri if present, but only if
         # it's *not* a valid story URL.
         mergeurl = self.get_story_url(db,book_id)
@@ -646,6 +767,9 @@ class FanFicFarePlugin(InterfaceAction):
 
         url_list_text = "\n".join(url_list)
 
+        self.gui.status_bar.show_message(_('Finished Fetching Story URLs for Series.'),3000)
+        self.restore_cursor()
+        
         #print("urlmapfile:%s"%urlmapfile)
 
         # AddNewDialog collects URLs, format and presents buttons.
@@ -767,10 +891,10 @@ class FanFicFarePlugin(InterfaceAction):
         # No need to do anything with perfs here, but we could.
         prefs
 
-    def make_id_searchstr(self,url):
+    def do_id_search(self,url):
         # older idents can be uri vs url and have | instead of : after
         # http, plus many sites are now switching to https.
-        return 'identifiers:"~ur(i|l):~^%s$"'%re.sub(r'https?\\\:','https?(\:|\|)',re.escape(url))
+        return self.gui.current_db.search_getting_ids('identifiers:"~ur(i|l):~^%s$"'%re.sub(r'https?\\\:','https?(\:|\|)',re.escape(url)),None)
 
     def prep_downloads(self, options, books, merge=False, extrapayload=None):
         '''Fetch metadata for stories from servers, launch BG job when done.'''
@@ -785,6 +909,7 @@ class FanFicFarePlugin(InterfaceAction):
         options['version'] = self.version
         logger.debug(self.version)
         options['personal.ini'] = get_fff_personalini()
+        options['savemetacol'] = prefs['savemetacol']
 
         #print("prep_downloads:%s"%books)
 
@@ -876,9 +1001,9 @@ class FanFicFarePlugin(InterfaceAction):
             # book has already been flagged bad for whatever reason.
             return
 
-        skip_date_update = False
-
         adapter = get_fff_adapter(url,fileform)
+        ## chapter range for title_chapter_range_pattern
+        adapter.setChaptersRange(book['begin'],book['end'])
         ## save and share cookiejar and pagecache between all
         ## downloads.
         if 'pagecache' not in options:
@@ -888,67 +1013,91 @@ class FanFicFarePlugin(InterfaceAction):
             options['cookiejar'] = adapter.get_empty_cookiejar()
         adapter.set_cookiejar(options['cookiejar'])
 
-        # reduce foreground sleep time for ffnet when few books.
-        if 'ffnetcount' in options and \
-                adapter.getConfig('tweak_fg_sleep') and \
-                adapter.getSiteDomain() == 'www.fanfiction.net':
-            minslp = float(adapter.getConfig('min_fg_sleep'))
-            maxslp = float(adapter.getConfig('max_fg_sleep'))
-            dwnlds = float(adapter.getConfig('max_fg_sleep_at_downloads'))
-            m = (maxslp-minslp) / (dwnlds-1)
-            b = minslp - m
-            slp = min(maxslp,m*float(options['ffnetcount'])+b)
-            #print("m:%s b:%s = %s"%(m,b,slp))
-            adapter.set_sleep(slp)
-
-        ## three tries, that's enough if both user/pass & is_adult needed,
-        ## or a couple tries of one or the other
-        for x in range(0,2):
-            try:
-                adapter.getStoryMetadataOnly(get_cover=False)
-            except exceptions.FailedToLogin, f:
-                logger.warn("Login Failed, Need Username/Password.")
-                userpass = UserPassDialog(self.gui,url,f)
-                userpass.exec_() # exec_ will make it act modal
-                if userpass.status:
-                    adapter.username = userpass.user.text()
-                    adapter.password = userpass.passwd.text()
-
-            except exceptions.AdultCheckRequired:
-                if question_dialog(self.gui, _('Are You an Adult?'), '<p>'+
-                                   _("%s requires that you be an adult.  Please confirm you are an adult in your locale:")%url,
-                                   show_copy_button=False):
-                    adapter.is_adult=True
-
-        # let other exceptions percolate up.
-        story = adapter.getStoryMetadataOnly(get_cover=False)
-
-        series = story.getMetadata('series')
-        if not merge and series and prefs['checkforseriesurlid']:
-            # try to find *series anthology* by *seriesUrl* identifier url or uri first.
-            searchstr = self.make_id_searchstr(story.getMetadata('seriesUrl'))
-            identicalbooks = db.search_getting_ids(searchstr, None)
-            # print("searchstr:%s"%searchstr)
-            # print("identicalbooks:%s"%identicalbooks)
-            if len(identicalbooks) > 0 and question_dialog(self.gui, _('Skip Story?'),'''
-                                                              <h3>%s</h3>
-                                                              <p>%s</p>
-                                                              <p>%s</p>
-                                                              <p>%s</p>
-                                                           '''%(
-                                                           _('Skip Anthology Story?'),
-                                                           _('"<b>%s</b>" is in series "<b><a href="%s">%s</a></b>" that you have an anthology book for.')%(story.getMetadata('title'),story.getMetadata('seriesUrl'),series[:series.index(' [')]),
-                                                           _("Click '<b>Yes</b>' to Skip."),
-                                                           _("Click '<b>No</b>' to download anyway.")),
-                                                           show_copy_button=False):
-                book['comment'] = _("Story in Series Anthology(%s).")%series
-                book['title'] = story.getMetadata('title')
-                book['author'] = [story.getMetadata('author')]
-                book['good']=False
-                book['icon']='rotate-right.png'
-                book['status'] = _('Skipped')
-                return
+        if collision in (CALIBREONLY, CALIBREONLYSAVECOL):
+            ## Getting metadata from configured column.
+            custom_columns = self.gui.library_view.model().custom_columns
+            if ( collision in (CALIBREONLYSAVECOL) and
+                 prefs['savemetacol'] != '' and 
+                 prefs['savemetacol'] in custom_columns ):
 
+                savedmeta_book_id = book['calibre_id']
+                # won't have calibre_id if update by URL vs book.
+                if not savedmeta_book_id:
+                    identicalbooks = self.do_id_search(url)
+                    if len(identicalbooks) == 1:
+                        savedmeta_book_id = identicalbooks.pop()
+                        
+                if savedmeta_book_id:
+                    label = custom_columns[prefs['savemetacol']]['label']
+                    savedmetadata = db.get_custom(savedmeta_book_id, label=label, index_is_id=True)
+                else:
+                    savedmetadata = None
+                    
+                if savedmetadata:
+                    # sets flag inside story so getStoryMetadataOnly won't hit server.
+                    adapter.setStoryMetadata(savedmetadata)
+                
+            # let other exceptions percolate up.
+            story = adapter.getStoryMetadataOnly(get_cover=False)
+        else:
+            # reduce foreground sleep time for ffnet when few books.
+            if 'ffnetcount' in options and \
+                    adapter.getConfig('tweak_fg_sleep') and \
+                    adapter.getSiteDomain() == 'www.fanfiction.net':
+                minslp = float(adapter.getConfig('min_fg_sleep'))
+                maxslp = float(adapter.getConfig('max_fg_sleep'))
+                dwnlds = float(adapter.getConfig('max_fg_sleep_at_downloads'))
+                m = (maxslp-minslp) / (dwnlds-1)
+                b = minslp - m
+                slp = min(maxslp,m*float(options['ffnetcount'])+b)
+                #print("m:%s b:%s = %s"%(m,b,slp))
+                adapter.set_sleep(slp)
+    
+            ## three tries, that's enough if both user/pass & is_adult needed,
+            ## or a couple tries of one or the other
+            for x in range(0,2):
+                try:
+                    adapter.getStoryMetadataOnly(get_cover=False)
+                except exceptions.FailedToLogin, f:
+                    logger.warn("Login Failed, Need Username/Password.")
+                    userpass = UserPassDialog(self.gui,url,f)
+                    userpass.exec_() # exec_ will make it act modal
+                    if userpass.status:
+                        adapter.username = userpass.user.text()
+                        adapter.password = userpass.passwd.text()
+    
+                except exceptions.AdultCheckRequired:
+                    if question_dialog(self.gui, _('Are You an Adult?'), '<p>'+
+                                       _("%s requires that you be an adult.  Please confirm you are an adult in your locale:")%url,
+                                       show_copy_button=False):
+                        adapter.is_adult=True
+    
+            # let other exceptions percolate up.
+            story = adapter.getStoryMetadataOnly(get_cover=False)
+    
+            series = story.getMetadata('series')
+            if not merge and series and prefs['checkforseriesurlid']:
+                # try to find *series anthology* by *seriesUrl* identifier url or uri first.
+                identicalbooks = self.do_id_search(story.getMetadata('seriesUrl'))
+                # print("identicalbooks:%s"%identicalbooks)
+                if len(identicalbooks) > 0 and question_dialog(self.gui, _('Skip Story?'),'''
+                                                                  <h3>%s</h3>
+                                                                  <p>%s</p>
+                                                                  <p>%s</p>
+                                                                  <p>%s</p>
+                                                               '''%(
+                                                               _('Skip Anthology Story?'),
+                                                               _('"<b>%s</b>" is in series "<b><a href="%s">%s</a></b>" that you have an anthology book for.')%(story.getMetadata('title'),story.getMetadata('seriesUrl'),series[:series.index(' [')]),
+                                                               _("Click '<b>Yes</b>' to Skip."),
+                                                               _("Click '<b>No</b>' to download anyway.")),
+                                                               show_copy_button=False):
+                    book['comment'] = _("Story in Series Anthology(%s).")%series
+                    book['title'] = story.getMetadata('title')
+                    book['author'] = [story.getMetadata('author')]
+                    book['good']=False
+                    book['icon']='rotate-right.png'
+                    book['status'] = _('Skipped')
+                    return
 
         ################################################################################################################################################33
 
@@ -958,7 +1107,10 @@ class FanFicFarePlugin(InterfaceAction):
 
         # all_metadata duplicates some data, but also includes extra_entries, etc.
         book['all_metadata'] = story.getAllMetadata(removeallentities=True)
-
+        if prefs['savemetacol'] != '':
+            # get metadata to save in configured column.
+            book['savemetacol'] = story.dump_html_metadata()
+        
         book['title'] = story.getMetadata("title", removeallentities=True)
         book['author_sort'] = book['author'] = story.getList("author", removeallentities=True)
         book['publisher'] = story.getMetadata("site")
@@ -985,7 +1137,7 @@ class FanFicFarePlugin(InterfaceAction):
             book['timestamp'] = None # need *something* there for calibre.
 
         if not merge:# skip all the collision code when d/ling for merging.
-            if collision in (CALIBREONLY):
+            if collision in (CALIBREONLY, CALIBREONLYSAVECOL):
                 book['icon'] = 'metadata.png'
                 book['status'] = _('Meta')
 
@@ -1003,11 +1155,9 @@ class FanFicFarePlugin(InterfaceAction):
                 logger.debug("from URL(%s)"%url)
 
                 # try to find by identifier url or uri first.
-                searchstr = self.make_id_searchstr(url)
-                identicalbooks = db.search_getting_ids(searchstr, None)
-                # print("searchstr:%s"%searchstr)
+                identicalbooks = self.do_id_search(url)
                 # print("identicalbooks:%s"%identicalbooks)
-                if len(identicalbooks) < 1:
+                if len(identicalbooks) < 1 and prefs['matchtitleauth']:
                     # find dups
                     authlist = story.getList("author", removeallentities=True)
                     mi = MetaInformation(story.getMetadata("title", removeallentities=True),
@@ -1026,7 +1176,7 @@ class FanFicFarePlugin(InterfaceAction):
                     raise NotGoingToDownload(_("More than one identical book by Identifer URL or title/author(s)--can't tell which book to update/overwrite."),"minusminus.png")
 
                 ## changed: add new book when CALIBREONLY if none found.
-                if collision == CALIBREONLY and not identicalbooks:
+                if collision in (CALIBREONLY, CALIBREONLYSAVECOL) and not identicalbooks:
                     collision = ADDNEW
                     options['collision'] = ADDNEW
 
@@ -1081,7 +1231,7 @@ class FanFicFarePlugin(InterfaceAction):
                                 return
 
             if book_id != None and collision != ADDNEW:
-                if collision in (CALIBREONLY):
+                if collision in (CALIBREONLY, CALIBREONLYSAVECOL):
                     book['comment'] = _('Metadata collected.')
                     # don't need temp file created below.
                     return
@@ -1099,9 +1249,6 @@ class FanFicFarePlugin(InterfaceAction):
                         if chaptercount == urlchaptercount:
                             if collision == UPDATE:
                                 raise NotGoingToDownload(_("Already contains %d chapters.")%chaptercount,'edit-undo.png')
-                            else:
-                                # UPDATEALWAYS
-                                skip_date_update = True
                         elif chaptercount > urlchaptercount:
                             raise NotGoingToDownload(_("Existing epub contains %d chapters, web site only has %d. Use Overwrite to force update.") % (chaptercount,urlchaptercount),'dialog_error.png')
                         elif chaptercount == 0:
@@ -1140,6 +1287,52 @@ class FanFicFarePlugin(InterfaceAction):
                     #print("calibre_series:%s [%s]"%book['calibre_series'])
 
         if book['good']: # there shouldn't be any !'good' books at this point.
+
+            ## Filling calibre_std_* and calibre_cust_* metadata
+            book['calibre_columns']={}
+            if prefs['cal_cols_pass_in']:
+                # std columns
+                mi = db.get_metadata(book['calibre_id'],index_is_id=True)
+                # book['calibre_columns']['calibre_std_identifiers']=\
+                #     {'val':', '.join(["%s:%s"%(k,v) for (k,v) in mi.get_identifiers().iteritems()]),
+                #                      'label':_('Ids')}
+                for k in mi.standard_field_keys():
+                # for k in mi:
+                    if k in STD_COLS_SKIP:
+                        continue
+                    (label,value,v,fmd) = mi.format_field_extended(k)
+                    if not label and k in field_metadata:
+                        label=field_metadata[k]['name']
+                    key='calibre_std_'+k
+                    
+                    # if k == 'user_categories':
+                    #     value=u', '.join(mi.get(k))
+                    #     label=_('User Categories')
+        
+                    if label: # only if it has a human readable name.
+                        if value is None or not book['calibre_id']:
+                            ## if existing book, populate existing calibre column
+                            ## values in metadata, else '' to hide.
+                            value=''                    
+                        book['calibre_columns'][key]={'val':value,'label':label}
+                        #logger.debug("%s(%s): %s"%(label,key,value))
+        
+                # custom columns
+                for k, column in self.gui.library_view.model().custom_columns.iteritems():
+                    if k != prefs['savemetacol']:
+                        key='calibre_cust_'+k[1:]
+                        label=column['name']
+                        value=db.get_custom(book['calibre_id'],
+                                            label=column['label'],
+                                            index_is_id=True)
+                        # custom always have name.
+                        if value is None or not book['calibre_id']:
+                            ## if existing book, populate existing calibre column
+                            ## values in metadata, else '' to hide.
+                            value=''
+                        book['calibre_columns'][key]={'val':value,'label':label}
+                        # logger.debug("%s(%s): %s"%(label,key,value))
+            
             # if still 'good', make a temp file to write the output to.
             # For HTML format users, make the filename inside the zip something reasonable.
             # For crazy long titles/authors, limit it to 200chars.
@@ -1151,17 +1344,6 @@ class FanFicFarePlugin(InterfaceAction):
             logger.debug("outfile:"+tmp.name)
             book['outfile'] = tmp.name
 
-            # cookiejar = PersistentTemporaryFile(prefix=story.formatFileName("${title}-${author}-",allowunsafefilename=False)[:100],
-            #                                     suffix='.cookiejar',
-            #                                     dir=options['tdir'])
-            # adapter.save_cookiejar(cookiejar.name)
-            # book['cookiejar'] = cookiejar.name
-            # pagecache = PersistentTemporaryFile(prefix=story.formatFileName("${title}-${author}-",allowunsafefilename=False)[:100],
-            #                                     suffix='.pagecache',
-            #                                     dir=options['tdir'])
-            # adapter.save_pagecache(pagecache.name)
-            # book['pagecache'] = pagecache.name
-
         return
 
     def start_download_job(self,book_list,
@@ -1178,7 +1360,7 @@ class FanFicFarePlugin(InterfaceAction):
 
         ## No need to BG process when CALIBREONLY!  Fake it.
         #print("options:%s"%options)
-        if options['collision'] == CALIBREONLY:
+        if options['collision'] in (CALIBREONLY, CALIBREONLYSAVECOL):
             class NotJob(object):
                 def __init__(self,result):
                     self.failed=False
@@ -1263,10 +1445,10 @@ class FanFicFarePlugin(InterfaceAction):
         logger.debug("add/update %s %s"%(book['title'],book['url']))
         mi = self.make_mi_from_book(book)
 
-        if options['collision'] != CALIBREONLY:
+        if options['collision'] not in (CALIBREONLY, CALIBREONLYSAVECOL):
             self.add_book_or_update_format(book,options,prefs,mi)
 
-        if options['collision'] == CALIBREONLY or \
+        if options['collision'] in (CALIBREONLY, CALIBREONLYSAVECOL) or \
                 ( (options['updatemeta'] or book['added']) and book['good'] ):
             try:
                 self.update_metadata(db, book['calibre_id'], book, mi, options)
@@ -1293,7 +1475,7 @@ class FanFicFarePlugin(InterfaceAction):
         failed_list = filter(lambda x : not x['good'] , book_list)
         failed_ids = [ x['calibre_id'] for x in failed_list ]
 
-        if options['collision'] != CALIBREONLY and \
+        if options['collision'] not in (CALIBREONLY, CALIBREONLYSAVECOL) and \
                 (prefs['addtolists'] or prefs['addtoreadlists']):
             self.update_reading_lists(all_ids,add=True)
 
@@ -1357,7 +1539,7 @@ class FanFicFarePlugin(InterfaceAction):
             if countpagesstats:
                 cp_plugin.count_statistics(all_ids,countpagesstats)
 
-        if prefs['autoconvert'] and options['collision'] != CALIBREONLY:
+        if prefs['autoconvert'] and options['collision'] not in (CALIBREONLY, CALIBREONLYSAVECOL):
             self.gui.status_bar.show_message(_('Starting auto conversion of %d books.')%(len(all_ids)), 3000)
             self.gui.iactions['Convert Books'].auto_convert_auto_add(all_ids)
 
@@ -1639,6 +1821,11 @@ class FanFicFarePlugin(InterfaceAction):
         #print("all_metadata: %s"%book['all_metadata'])
         custom_columns = self.gui.library_view.model().custom_columns
 
+        # save metadata to configured column
+        if 'savemetacol' in book and prefs['savemetacol'] != '' and prefs['savemetacol'] in custom_columns:
+            label = custom_columns[prefs['savemetacol']]['label']
+            self.set_custom(db, book_id, 'comment', book['savemetacol'], label=label, commit=True)
+
         #print("prefs['custom_cols'] %s"%prefs['custom_cols'])
         for col, meta in prefs['custom_cols'].iteritems():
             #print("setting %s to %s"%(col,meta))
@@ -1739,7 +1926,7 @@ class FanFicFarePlugin(InterfaceAction):
 
         # set author link if found.  All current adapters have authorUrl, except anonymous on AO3.
         # Moved down so author's already in the DB.
-        if 'authorUrl' in book['all_metadata']:
+        if 'authorUrl' in book['all_metadata'] and prefs['set_author_url']:
             authurls = book['all_metadata']['authorUrl'].split(", ")
             authorlist = [ a.replace('&',';') for a in book['author'] ]
             authorids = db.new_api.get_item_ids('authors',authorlist)
@@ -1763,7 +1950,7 @@ class FanFicFarePlugin(InterfaceAction):
             or (prefs['updatecalcover'] == SAVE_YES_IF_IMG ## yes, if image.
                 and book['all_metadata']['cover_image'] )): # in ('specific','first','default','old')
             existingepub = db.format(book_id,'EPUB',index_is_id=True, as_file=True)
-            epubmi = get_metadata(existingepub,'EPUB')
+            epubmi = calibre_get_metadata(existingepub,'EPUB')
             if epubmi.cover_data[1] is not None:
                 try:
                     db.set_cover(book_id, epubmi.cover_data[1])
@@ -2049,7 +2236,7 @@ class FanFicFarePlugin(InterfaceAction):
             existingepub = None
             if path == None and db.has_format(book_id,'EPUB',index_is_id=True):
                 existingepub = db.format(book_id,'EPUB',index_is_id=True, as_file=True)
-                mi = get_metadata(existingepub,'EPUB')
+                mi = calibre_get_metadata(existingepub,'EPUB')
                 identifiers = mi.get_identifiers()
                 if 'url' in identifiers:
                     # print("url from get_metadata:%s"%identifiers['url'].replace('|',':'))
@@ -2203,6 +2390,12 @@ class FanFicFarePlugin(InterfaceAction):
 
         return book
 
+    def busy_cursor(self):
+        QApplication.setOverrideCursor(QCursor(Qt.WaitCursor))
+
+    def restore_cursor(self):
+        QApplication.restoreOverrideCursor()
+    
 def split_text_to_urls(urls):
     # remove dups while preserving order.
     dups=set()

calibre-plugin/fff_util.py

@@ -1,5 +1,5 @@
-#!/usr/bin/env python
-# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+# -*- coding: utf-8 -*-
+
 from __future__ import (unicode_literals, division, absolute_import,
                         print_function)
 
@@ -23,12 +23,12 @@ def get_fff_personalini():
 def get_fff_config(url,fileform="epub",personalini=None):
     if not personalini:
         personalini = get_fff_personalini()
-    site='unknown'
+    sections=['unknown']
     try:
-        site = adapters.getConfigSectionFor(url)
+        sections = adapters.getConfigSectionsFor(url)
     except Exception as e:
-        logger.debug("Failed trying to get ini config for url(%s): %s, using section [%s] instead"%(url,e,site))
-    configuration = Configuration(site,fileform)
+        logger.debug("Failed trying to get ini config for url(%s): %s, using section %s instead"%(url,e,sections))
+    configuration = Configuration(sections,fileform)
     configuration.readfp(StringIO(get_resources("plugin-defaults.ini")))
     configuration.readfp(StringIO(personalini))
 

calibre-plugin/inihighlighter.py

@@ -1,5 +1,5 @@
-#!/usr/bin/env python
-# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+# -*- coding: utf-8 -*-
+
 from __future__ import (unicode_literals, division,
                         print_function)
 

calibre-plugin/jobs.py

@@ -1,5 +1,5 @@
-#!/usr/bin/env python
-# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+# -*- coding: utf-8 -*-
+
 from __future__ import (unicode_literals, division, absolute_import,
                         print_function)
 
@@ -11,6 +11,7 @@ import logging
 logger = logging.getLogger(__name__)
 
 import time, traceback
+from StringIO import StringIO
     
 from calibre.utils.ipc.server import Server
 from calibre.utils.ipc.job import ParallelJob
@@ -99,7 +100,7 @@ def do_download_for_worker(book,options,notification=lambda x,y:x):
     with fffbase:
         
         from calibre_plugins.fanficfare_plugin.dialogs import (NotGoingToDownload,
-                OVERWRITE, OVERWRITEALWAYS, UPDATE, UPDATEALWAYS, ADDNEW, SKIP, CALIBREONLY)
+                OVERWRITE, OVERWRITEALWAYS, UPDATE, UPDATEALWAYS, ADDNEW, SKIP, CALIBREONLY, CALIBREONLYSAVECOL)
         from calibre_plugins.fanficfare_plugin.fanficfare import adapters, writers, exceptions
         from calibre_plugins.fanficfare_plugin.fanficfare.epubutils import get_update_data
         
@@ -151,10 +152,12 @@ def do_download_for_worker(book,options,notification=lambda x,y:x):
             outfile = book['outfile']
     
             ## No need to download at all.  Shouldn't ever get down here.
-            if options['collision'] in (CALIBREONLY):
+            if options['collision'] in (CALIBREONLY, CALIBREONLYSAVECOL):
                 logger.info("Skipping CALIBREONLY 'update' down inside worker--this shouldn't be happening...")
                 book['comment'] = 'Metadata collected.'
                 book['all_metadata'] = story.getAllMetadata(removeallentities=True)
+                if options['savemetacol'] != '':
+                    book['savemetacol'] = story.dump_html_metadata()
                 
             ## checks were done earlier, it's new or not dup or newer--just write it.
             elif options['collision'] in (ADDNEW, SKIP, OVERWRITE, OVERWRITEALWAYS) or \
@@ -162,24 +165,20 @@ def do_download_for_worker(book,options,notification=lambda x,y:x):
     
                 # preserve logfile even on overwrite.
                 if 'epub_for_update' in book:
-                    (urlignore,
-                     chaptercountignore,
-                     oldchaptersignore,
-                     oldimgsignore,
-                     oldcoverignore,
-                     calibrebookmarkignore,
-                     # only logfile set in adapter, so others aren't used.
-                     adapter.logfile) = get_update_data(book['epub_for_update'])
-    
+                    
+                    adapter.logfile = get_update_data(book['epub_for_update'])[6]
                     # change the existing entries id to notid so
                     # write_epub writes a whole new set to indicate overwrite.
                     if adapter.logfile:
                         adapter.logfile = adapter.logfile.replace("span id","span notid")
                 
                 logger.info("write to %s"%outfile)
+                inject_cal_cols(book,story,configuration)
                 writer.writeStory(outfilename=outfile, forceOverwrite=True)
                 book['comment'] = 'Download %s completed, %s chapters.'%(options['fileform'],story.getMetadata("numChapters"))
                 book['all_metadata'] = story.getAllMetadata(removeallentities=True)
+                if options['savemetacol'] != '':
+                    book['savemetacol'] = story.dump_html_metadata()
                 
             ## checks were done earlier, just update it.
             elif 'epub_for_update' in book and options['collision'] in (UPDATE, UPDATEALWAYS):
@@ -193,13 +192,17 @@ def do_download_for_worker(book,options,notification=lambda x,y:x):
                  adapter.oldimgs,
                  adapter.oldcover,
                  adapter.calibrebookmark,
-                 adapter.logfile) = get_update_data(book['epub_for_update'])
+                 adapter.logfile,
+                 adapter.oldchaptersmap,
+                 adapter.oldchaptersdata) = get_update_data(book['epub_for_update'])[0:9]
     
                 # dup handling from fff_plugin needed for anthology updates.
                 if options['collision'] == UPDATE:
                     if chaptercount == urlchaptercount:
                         book['comment']=_("Already contains %d chapters.  Reuse as is.")%chaptercount
                         book['all_metadata'] = story.getAllMetadata(removeallentities=True)
+                        if options['savemetacol'] != '':
+                            book['savemetacol'] = story.dump_html_metadata()
                         book['outfile'] = book['epub_for_update'] # for anthology merge ops.
                         return book
     
@@ -214,12 +217,15 @@ def do_download_for_worker(book,options,notification=lambda x,y:x):
                 logger.info("Do update - epub(%d) vs url(%d)" % (chaptercount, urlchaptercount))
                 logger.info("write to %s"%outfile)
     
+                inject_cal_cols(book,story,configuration)
                 writer.writeStory(outfilename=outfile, forceOverwrite=True)
                 
                 book['comment'] = _('Update %s completed, added %s chapters for %s total.')%\
                     (options['fileform'],(urlchaptercount-chaptercount),urlchaptercount)
                 book['all_metadata'] = story.getAllMetadata(removeallentities=True)
-            
+                if options['savemetacol'] != '':
+                    book['savemetacol'] = story.dump_html_metadata()
+
             if options['smarten_punctuation'] and options['fileform'] == "epub" \
                     and calibre_version >= (0, 9, 39):
                 # for smarten punc
@@ -253,3 +259,22 @@ def do_download_for_worker(book,options,notification=lambda x,y:x):
             
         #time.sleep(10)
     return book
+
+## calibre's columns for an existing book are pased in and injected
+## into the story's metadata.  For convenience, we also add labels and
+## valid_entries for them in a special [injected] section that has
+## even less precedence than [defaults]
+def inject_cal_cols(book,story,configuration):
+    configuration.remove_section('injected')
+    if 'calibre_columns' in book:
+        injectini = ['[injected]']
+        extra_valid = []
+        for k, v in book['calibre_columns'].iteritems():
+            story.setMetadata(k,v['val'])
+            injectini.append('%s_label:%s'%(k,v['label']))
+            extra_valid.append(k)
+        if extra_valid: # if empty, there's nothing to add.
+            injectini.append("add_to_extra_valid_entries:,"+','.join(extra_valid))
+            configuration.readfp(StringIO('\n'.join(injectini)))
+            #print("added:\n%s\n"%('\n'.join(injectini)))
+    

calibre-plugin/plugin-defaults.ini

@@ -261,6 +261,11 @@ keep_summary_html:true
 ## kept.  Some sites(AO3) use <a title="Translation">Foreign language</a>
 #keep_title_attr: false
 
+## If a chapter range was given, use this pattern for the book title.
+## replace_metadata and include/exclude will be applied *after* this.
+## Set to empty value to disable.
+title_chapter_range_pattern:${title} (Ch ${first}-${last})
+
 ## Don't like the numbers at the start of chapter titles on some
 ## sites?  You can use strip_chapter_numbers to strip them off.  Just
 ## want to make them all look the same?  Strip them off, then add them
@@ -293,12 +298,31 @@ chapter_title_strip_pattern:^[0-9]+[\.: -]+
 ## etc
 #chapter_title_strip_pattern:^([0-9]+[\.: -]+)?(Chapter *[0-9]+[\.:, -]*)?
 
-## Uses a python template substitution.  The ${index} is the 'chapter'
-## number and ${title} is the chapter title, after applying
-## chapter_title_strip_pattern.  Those are the only variables available.
-## "The Beginning" => "1. The Beginning" 
+## If true, when updating an epub that already has old chapters, new
+## chapters will be marked in the TOC and chapter header by using
+## chapter_title_new_pattern and chapter_title_addnew_pattern to set the chapter.
+mark_new_chapters:false
+
+## chapter title patterns use python template substitution.  The
+## ${index} is the 'chapter' number and ${title} is the chapter title,
+## after applying chapter_title_strip_pattern.  Those are the only
+## variables available.
+
+## The basic pattern used when not using add_chapter_numbers or
+## mark_new_chapters
+chapter_title_def_pattern:${title}
+
+## Pattern used with add_chapter_numbers, but not mark_new_chapters
 chapter_title_add_pattern:${index}. ${title}
 
+## Pattern used with mark_new_chapters, but not add_chapter_numbers
+## (new) is just text and can be changed.
+chapter_title_new_pattern:(new) ${title}
+
+## Pattern used with add_chapter_numbers and mark_new_chapters
+## (new) is just text and can be changed.
+chapter_title_addnew_pattern:${index}. (new) ${title}
+
 ## Uses a python template substitution.  The ${title} is the default
 ## title of a new anthology, <series name> in the case of a series, or
 ## the first book title otherwise.  This is only applied to new
@@ -330,6 +354,74 @@ sort_ships:false
 ## User-agent
 user_agent:FFF/2.X
 
+## Added for [base_xenforoforum], but can be used with other sites,
+## too.  Limit the 'description' to the first X *characters*
+## collected.  Character count includes HTML tags, so it can be
+## non-intuitive.
+#description_limit:1000
+
+[base_efiction]
+## At the time of writing, eFiction Base adapters allow downloading
+## the whole story in bulk using the 'Print' feature. If 'bulk_load'
+## is set to 'true', both metadata and chapters can be loaded in one
+## step
+bulk_load:true
+
+[base_xenforoforum]
+## Currently only forums.spacebattles.com and forums.sufficientvelocity.com
+
+cover_exclusion_regexp:/clear.png
+
+## I saw lots of chapters name simply '1.1' etc during testing.
+strip_chapter_numbers:false
+
+## Copy title to tagsfromtitle for parsing tags.
+add_to_extra_valid_entries:,tagsfromtitle
+
+## '.NOREPL' tells the system to *not* apply title's
+## in/exclude/replace_metadata -- Only works on include_in_ lines.
+include_in_tagsfromtitle:title.NOREPL
+
+tagsfromtitle_label:Tags from Title
+
+## might want to do this, maybe not.  Will often include category, but
+## also often include non-category stuff.
+# include_in_category:tagsfromtitle 
+
+add_to_include_metadata_pre:
+# only keep tagsfromtitle with ( or [ in.
+ tagsfromtitle=~[\[\(]
+ 
+add_to_replace_metadata:
+# remove anything outside () or []
+ tagsfromtitle=>^.*?([\(\[]([^\]\)]+)[\)\]]).*?$=>\1
+# remove () []
+ tagsfromtitle=>[\(\)\[\]]=>
+# change (spaces)slash(or semicolon)(spaces) to comma
+ tagsfromtitle=> *[/;] *=>,
+ tagsfromtitle=> [xX] =>,
+ 
+# remove [] or () blocks and leading/trailing spaces/dashes/colons
+ title=>[-: ]*[\(\[]([^\]\)]+)[\)\]][-: ]*=>
+# remove 'Thread' and the next word, usually "Thread 2", "Thread
+# four", "Thread iv", etc
+ title,tagsfromtitle=>[-: ]*[Tt]hread [^ ]+[-: ]*=>
+
+add_to_extra_titlepage_entries:,tagsfromtitle
+
+## '.SPLIT' tells the system to split by ','
+add_to_include_subject_tags:,tagsfromtitle.SPLIT
+
+## base_xenforoforum reads Published and Updated datetimes from
+## Threadmarks if used, or from the posted & updated times of the
+## 'first' post if no threadmarks.
+datePublished_format:%%Y-%%m-%%d %%H:%%M:%%S
+dateUpdated_format:%%Y-%%m-%%d %%H:%%M:%%S
+
+## Only take the first X characters of the 'first' post to use as
+## the description.
+description_limit:500
+
 ## Each output format has a section that overrides [defaults]
 [html]
 
@@ -578,6 +670,8 @@ extratags: FanFiction,Testing,HTML
 ## doesn't like that.  If do_update_hook is uncommented and set true,
 ## the adapter will discard all existing chapters from the newest one
 ## on when updating to enforce accurate chapters.
+## Starting July 2015, FFF stores chapter URLs in the chapter files.
+## Stories downloaded after that shouldn't need this setting anymore.
 #do_update_hook:false
 
 ## AO3 adapter defines a few extra metadata entries.
@@ -593,6 +687,18 @@ hits_label:Hits
 collections_label:Collections
 bookmarks_label:Bookmarks
 
+## AO3 doesn't have anything it calls 'genre'.  The adapter used to be
+## hardcoded to include the site specific metadata freeformtags &
+## ao3categories in the standard metadata field genre.  By making it
+## configurable, users can change it.
+include_in_genre: freeformtags, ao3categories
+
+## AO3 uses the word 'category' differently than most sites.  The
+## adapter used to be hardcoded to include the site specific metadata
+## fandom in the standard metadata field category.  By making it
+## configurable, users can change it.
+include_in_category:fandoms
+
 ## freeformtags was previously typo'ed as freefromtags.  This way,
 ## freefromtags will still work for people who've used it.
 include_in_freefromtags:freeformtags
@@ -604,7 +710,7 @@ include_in_freefromtags:freeformtags
 #extra_subject_tags:fandoms,freeformtags,ao3categories
 
 ## AO3 chapters can include several different types of notes.  We've
-## traditional included them all in the chapter text, but this allows
+## traditionally included them all in the chapter text, but this allows
 ## you to customize which you include.  Copy this parameter to your
 ## personal.ini and list the ones you don't want.
 #exclude_notes:authorheadnotes,chaptersummary,chapterheadnotes,chapterfootnotes,authorfootnotes
@@ -754,11 +860,6 @@ extraships:Spike/Buffy
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
-## Virtually all eFiction Base adapters allow downloading the whole story in
-## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
-## metadata and chapters can be loaded in one step
-bulk_load:true
-
 [dramione.org]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -939,11 +1040,6 @@ extraships:Harry Potter/Hermione Granger
 #username:YourName
 #password:yourpassword
 
-## Virtually all eFiction Base adapters allow downloading the whole story in
-## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
-## metadata and chapters can be loaded in one step
-bulk_load:true
-
 extra_valid_entries: readings,romance
 extra_titlepage_entries: readings,romance
 readings_label: Readings
@@ -1033,6 +1129,12 @@ extra_valid_entries:size
 # don't show twitter icon.
 cover_exclusion_regexp:/res/css/bir.png
 
+[forums.spacebattles.com]
+## see [base_xenforoforum]
+
+[forums.sufficientvelocity.com]
+## see [base_xenforoforum]
+
 [grangerenchanted.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -1091,11 +1193,6 @@ extracategories:Glee RPF
 extracharacters:Darren Criss, Chris Colfer
 extraships:Darren Criss/Chris Colfer
 
-## Virtually all eFiction Base adapters allow downloading the whole story in
-## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
-## metadata and chapters can be loaded in one step
-bulk_load:true
-
 [ksarchive.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:Star Trek
@@ -1115,11 +1212,6 @@ eroticatags_label:Erotica Tags
 extra_titlepage_entries: eroticatags
 
 [lotrfanfiction.com]
-## Virtually all eFiction Base adapters allow downloading the whole story in
-## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
-## metadata and chapters can be loaded in one step
-bulk_load:true
-
 extra_valid_entries: readings
 readings_label: Readings
 
@@ -1173,6 +1265,12 @@ extracategories:NCIS
 extracategories:Buffy: The Vampire Slayer
 extracharacters:Willow
 
+[ninelives.dark-solace.org]
+## Site dedicated to these categories/characters/ships
+extracategories:The Walking Dead
+extracharacters:Carol,Daryl
+extraships:Carol/Daryl
+
 [nocturnal-light.net]
 ## Extra metadata that this adapter knows about.  See [dramione.org]
 ## for examples of how to use them.
@@ -1369,11 +1467,6 @@ extracategories:Transgender
 ## confirm they are adult for adult content.
 #is_adult:true
 
-## Virtually all eFiction Base adapters allow downloading the whole story in
-## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
-## metadata and chapters can be loaded in one step
-bulk_load:true
-
 [thehexfiles.net]
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter
@@ -1393,11 +1486,6 @@ extraships:Harry Potter/Draco Malfoy
 ## personal.ini, not defaults.ini.
 #is_adult:true
 
-## Virtually all eFiction Base adapters allow downloading the whole story in
-## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
-## metadata and chapters can be loaded in one step
-bulk_load:true
-
 ## Site dedicated to these categories/characters/ships
 extracategories:Criminal Minds
 
@@ -1407,11 +1495,6 @@ extracategories:Criminal Minds
 ## personal.ini, not defaults.ini.
 #is_adult:true
 
-## Virtually all eFiction Base adapters allow downloading the whole story in
-## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
-## metadata and chapters can be loaded in one step
-bulk_load:true
-
 extra_valid_entries: readings,challenge
 extra_titlepage_entries: readings,challenge
 challenge_label: Challenge
@@ -1765,10 +1848,27 @@ extraships:InuYasha/Kagome
 ## Site dedicated to these categories/characters/ships
 extracategories:Lord of the Rings
 
-## Virtually all eFiction Base adapters allow downloading the whole story in
-## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
-## metadata and chapters can be loaded in one step
-bulk_load:true
+[www.masseffect2.in]
+## Site dedicated to this fandom.
+extracategories: Mass Effect
+
+## Ucoz.com engine, upon which MassEffect2.in is based, imposes an unspecified limit on request frequency.
+## Reports vary from `5 requests per second' to `2 requests per second for more than 10 per minute'.
+## With default settings, a several-hours IP ban may follow, so set it higher.
+slow_down_sleep_time: 2
+
+## Whether to exclude editor signature from the bottom of chapter text.
+exclude_editor_signature: false
+
+## Stories on the site almost never have cover image, and for the stories which do,
+## this may be adjusted in `personal.ini' before downloading.
+never_make_cover: true
+
+## Titles for ratings identified by 1- or 2-letter codes from `ERATING system'
+## (`система Р.Е.Й.Т.И.Н.Г.').  MassEffect2.in and some other sites adopted it,
+## but changed titles and update them occasionally.
+rating_titles: R=RESTRICTED (16+), E=EXEMPT (18+), I=ART HOUSE, T=To every, A=IN=Иной мир, Nn=Новый мир, G=О\, Господи!
+adult_ratings: E,R
 
 [www.mediaminer.org]
 

calibre-plugin/prefs.py

@@ -1,5 +1,5 @@
-#!/usr/bin/env python
-# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+# -*- coding: utf-8 -*-
+
 from __future__ import (unicode_literals, division, absolute_import,
                         print_function)
 
@@ -51,6 +51,7 @@ PREFS_KEY_SETTINGS = 'settings'
 # take from here.
 default_prefs = {}
 default_prefs['personal.ini'] = get_resources('plugin-example.ini')
+default_prefs['cal_cols_pass_in'] = False
 default_prefs['rejecturls'] = ''
 default_prefs['rejectreasons'] = '''Sucked
 Boring
@@ -75,6 +76,7 @@ default_prefs['lookforurlinhtml'] = False
 default_prefs['checkforseriesurlid'] = True
 default_prefs['checkforurlchange'] = True
 default_prefs['injectseries'] = False
+default_prefs['matchtitleauth'] = True
 default_prefs['smarten_punctuation'] = False
 default_prefs['show_est_time'] = False
 
@@ -83,6 +85,7 @@ default_prefs['read_lists'] = ''
 default_prefs['addtolists'] = False
 default_prefs['addtoreadlists'] = False
 default_prefs['addtolistsonread'] = False
+default_prefs['autounnew'] = False
 
 default_prefs['updatecalcover'] = None
 default_prefs['gencalcover'] = SAVE_YES
@@ -98,11 +101,13 @@ default_prefs['countpagesstats'] = []
 default_prefs['wordcountmissing'] = False
 
 default_prefs['errorcol'] = ''
+default_prefs['savemetacol'] = ''
 default_prefs['custom_cols'] = {}
 default_prefs['custom_cols_newonly'] = {}
 default_prefs['allow_custcol_from_ini'] = True
 
 default_prefs['std_cols_newonly'] = {}
+default_prefs['set_author_url'] = True
 
 default_prefs['imapserver'] = ''
 default_prefs['imapuser'] = ''
@@ -110,6 +115,8 @@ default_prefs['imappass'] = ''
 default_prefs['imapsessionpass'] = False
 default_prefs['imapfolder'] = 'INBOX'
 default_prefs['imapmarkread'] = True
+default_prefs['auto_reject_from_email'] = False
+default_prefs['download_from_email_immediately'] = False
 
 def set_library_config(library_config,db):
     db.prefs.set_namespaced(PREFS_NAMESPACE,

fanficfare/adapters/__init__.py

@@ -135,6 +135,11 @@ import adapter_fanfictionjunkiesde
 import adapter_devianthearts
 import adapter_tgstorytimecom
 import adapter_itcouldhappennet
+import adapter_forumsspacebattlescom
+import adapter_forumssufficientvelocitycom
+import adapter_questionablequestingcom
+import adapter_ninelivesdarksolaceorg
+import adapter_masseffect2in
 
 ## This bit of complexity allows adapters to be added by just adding
 ## importing.  It eliminates the long if/else clauses we used to need
@@ -194,14 +199,24 @@ def getAdapter(config,url,anyurl=False):
     # No adapter found.
     raise exceptions.UnknownSite( url, [cls.getSiteDomain() for cls in __class_list] )
 
-def getConfigSections():
+def getSiteSections():
+    # doesn't include base sections. Sections rather than site DNS because of squidge/peja
     return [cls.getConfigSection() for cls in __class_list]
 
+def getConfigSections():
+    # does include base sections.
+    sections = set()
+    for cls in __class_list:
+        sections.update(cls.getConfigSections())
+    return sections
+
 def get_bulk_load_sites():
     # for now, all eFiction Base adapters are assumed to allow bulk_load.
-    return [cls.getConfigSection().replace('www.','') for cls in
-            filter( lambda x : issubclass(x,base_efiction_adapter.BaseEfictionAdapter),
-                    __class_list)]
+    sections = set()
+    for cls in filter( lambda x : issubclass(x,base_efiction_adapter.BaseEfictionAdapter),
+                       __class_list):
+        sections.update( [ x.replace('www.','') for x in cls.getConfigSections() ] )
+    return sections
 
 def getSiteExamples():
     l=[]
@@ -209,10 +224,10 @@ def getSiteExamples():
         l.append((cls.getConfigSection(),cls.getSiteExampleURLs().split()))
     return l
 
-def getConfigSectionFor(url):
+def getConfigSectionsFor(url):
     (cls,fixedurl) = getClassFor(url)
     if cls:
-        return cls.getConfigSection()
+        return cls.getConfigSections()
 
     # No adapter found.
     raise exceptions.UnknownSite( url, [cls.getSiteDomain() for cls in __class_list] )

fanficfare/adapters/adapter_archiveofourownorg.py

@@ -230,7 +230,6 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
         fandoms = a.findAll('a',{'class':"tag"})
         for fandom in fandoms:
             self.story.addToList('fandoms',fandom.string)
-            self.story.addToList('category',fandom.string)
 		
         a = metasoup.find('dd',{'class':"warning tags"})
         if a != None:
@@ -243,7 +242,6 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
             genres = a.findAll('a',{'class':"tag"})
             for genre in genres:
                 self.story.addToList('freeformtags',genre.string)
-                self.story.addToList('genre',genre.string)
                 
         a = metasoup.find('dd',{'class':"category tags"})
         if a != None:
@@ -251,7 +249,6 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
             for genre in genres:
                 if genre != "Gen":
                     self.story.addToList('ao3categories',genre.string)
-                    self.story.addToList('genre',genre.string)
 		
         a = metasoup.find('dd',{'class':"character tags"})
         if a != None:

fanficfare/adapters/adapter_dramioneorg.py

@@ -38,8 +38,9 @@ class DramioneOrgAdapter(BaseSiteAdapter):
     def __init__(self, config, url):
         BaseSiteAdapter.__init__(self, config, url)
 
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
+        self.decode = ["utf8",
+                       "Windows-1252",]
+                               # 1252 is a superset of iso-8859-1.
                                # Most sites that claim to be
                                # iso-8859-1 (and some that claim to be
                                # utf8) are really windows-1252.

fanficfare/adapters/adapter_fanfictionnet.py

@@ -104,7 +104,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
             data = self._fetchUrl(url)
             #logger.debug("\n===================\n%s\n===================\n"%data)
             soup = self.make_soup(data)
-        except urllib2.HTTPError, e:
+        except urllib2.HTTPError as e:
             if e.code == 404:
                 raise exceptions.StoryDoesNotExist(url)
             else:
@@ -135,11 +135,15 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
                                                 chapcount+1)
                 logger.debug('=Trying newer chapter: %s' % tryurl)
                 newdata = self._fetchUrl(tryurl)
-                if "not found. Please check to see you are not using an outdated url." \
-                        not in newdata:
+                if "not found. Please check to see you are not using an outdated url." not in newdata \
+                        and "This request takes too long to process, it is timed out by the server." not in newdata:
                     logger.debug('=======Found newer chapter: %s' % tryurl)
                     soup = self.make_soup(newdata)
-            except:
+            except urllib2.HTTPError as e:
+                if e.code == 503:
+                    raise e
+            except e:
+                logger.warn("Caught an exception reading URL: %s sleeptime(%s) Exception %s."%(unicode(url),sleeptime,unicode(e)))
                 pass
             
         # Find authorid and URL from... author url.

fanficfare/adapters/adapter_forumsspacebattlescom.py

@@ -0,0 +1,35 @@
+#  -*- coding: utf-8 -*-
+
+# Copyright 2015 FanFicFare team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from base_xenforoforum_adapter import BaseXenForoForumAdapter
+
+def getClass():
+    return ForumsSpacebattlesComAdapter
+
+class ForumsSpacebattlesComAdapter(BaseXenForoForumAdapter):
+
+    def __init__(self, config, url):
+        BaseXenForoForumAdapter.__init__(self, config, url)
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','fsb')
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'forums.spacebattles.com'
+

fanficfare/adapters/adapter_forumssufficientvelocitycom.py

@@ -0,0 +1,34 @@
+#  -*- coding: utf-8 -*-
+
+# Copyright 2015 FanFicFare team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from base_xenforoforum_adapter import BaseXenForoForumAdapter
+
+def getClass():
+    return ForumsSufficientVelocityComAdapter
+
+class ForumsSufficientVelocityComAdapter(BaseXenForoForumAdapter):
+
+    def __init__(self, config, url):
+        BaseXenForoForumAdapter.__init__(self, config, url)
+
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','fsv')
+
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'forums.sufficientvelocity.com'

fanficfare/adapters/adapter_masseffect2in.py

@@ -0,0 +1,719 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2015 FanFicFare team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import bs4
+import datetime
+import logging
+import re
+import urllib2
+
+from ..htmlcleanup import removeEntities, stripHTML
+from .. import exceptions as exceptions
+from base_adapter import BaseSiteAdapter, makeDate
+
+
+_logger = logging.getLogger(__name__)
+
+
+def getClass():
+    """Returns adapter class defined in this module."""
+    return MassEffect2InAdapter
+
+
+class ParsingError(Exception):
+    """Indicates an error while parsing web page content."""
+    def __init__(self, message):
+        Exception.__init__(self)
+        self.message = message
+
+    def __str__(self):
+        return self.message
+
+
+class MassEffect2InAdapter(BaseSiteAdapter):
+    """
+    Provides support for MassEffect2.in site as story source.
+    Can be used as a template for sites build upon Ucoz.com engine (until no base class extracted).
+    Specializations:
+        1) Russian content (date format, genre names, etc.);
+        2) original `E.R.A.T.I.N.G.' rating scale, used by masseffect2.in
+           and some affiliated sites, denoted with images;
+        3) editor signatures an an option to remove them.
+    """
+
+    WORD_PATTERN = re.compile(u'\w+', re.UNICODE)
+    DOCUMENT_ID_PATTERN = re.compile(u'\d+-\d+-\d+-\d+')
+    SITE_LANGUAGE = u'Russian'
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["utf8"]
+
+        self.story.setMetadata('siteabbrev', 'me2in')
+        self.story.setMetadata('storyId', self._getDocumentId(self.url))
+
+        self._setURL(self._makeDocumentUrl(self.story.getMetadata('storyId')))
+
+        self._chapters = {}
+        self._parsingConfiguration = None
+
+    # Must be @staticmethod, not @classmethod!
+    @staticmethod
+    def getSiteDomain():
+        return 'www.masseffect2.in'
+
+    @classmethod
+    def getSiteExampleURLs(cls):
+        return u' '.join([cls._makeDocumentUrl('19-1-0-1234'),
+                          cls._makeDocumentUrl('24-1-0-4321')])
+
+    def getSiteURLPattern(self):
+        return re.escape(self._makeDocumentUrl('')) + self.DOCUMENT_ID_PATTERN.pattern
+
+    def use_pagecache(self):
+        """Allows use of downloaded page cache.  It is essential for this
+        adapter, because the site does not offers chapter URL list, and many
+        pages have to be fetched and parsed repeatedly."""
+        return True
+
+    def extractChapterUrlsAndMetadata(self):
+        """Extracts chapter URLs and story metadata.  Actually downloads all
+        chapters, which is not exactly right, but necessary due to technical
+        limitations of the site."""
+
+        def followChapters(starting, forward=True):
+            if forward:
+                url = starting.getNextChapterUrl()
+            else:
+                url = starting.getPreviousChapterUrl()
+            if url:
+                url = self._makeDocumentUrl(self._getDocumentId(url))
+                following = self._makeChapter(url)
+                # Do not follow links to related, but different stories (prequels or sequels).
+                if not following.isFromStory(starting.getHeading()):
+                    return
+                if forward:
+                    yield following
+                for chapter in followChapters(following, forward):
+                    yield chapter
+                if not forward:
+                    yield following
+
+        try:
+            startingChapter = self._makeChapter(self.url)
+        except urllib2.HTTPError, error:
+            if error.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            raise
+
+        # We only have one date for each chapter and assume the oldest one
+        # to be publication date and the most recent one to be update date.
+        datePublished = datetime.datetime.max
+        dateUpdated = datetime.datetime.min
+        wordCount = 0
+        # We aim at counting chapters, not chapter parts.
+        chapterCount = 0
+        storyInProgress = False
+
+        chapters = \
+            list(followChapters(startingChapter, forward=False)) + \
+            [startingChapter] + \
+            list(followChapters(startingChapter, forward=True))
+
+        headings = [chapter.getHeading() for chapter in chapters]
+        largestCommonPrefix = _getLargestCommonPrefix(*headings)
+        prefixLength = len(largestCommonPrefix)
+        storyTitleEnd, chapterTitleStart = prefixLength, prefixLength
+        match = re.search(u'[:\.\s]*(?P<chapter>глава\s+)?$', largestCommonPrefix, re.IGNORECASE | re.UNICODE)
+        if match:
+            storyTitleEnd -= len(match.group())
+            label = match.group('chapter')
+            if label:
+                chapterTitleStart -= len(label)
+        storyTitle = largestCommonPrefix[:storyTitleEnd]
+        self.story.setMetadata('title', storyTitle)
+
+        garbagePattern = re.compile(u'(?P<start>^)?[:\.\s]*(?(start)|$)', re.UNICODE)
+        indexPattern = re.compile(u'(?:глава\s)?(?:(?<!\d)(?P<index>\d{1,3})(?=\D|$))', re.IGNORECASE | re.UNICODE)
+
+        for chapter in chapters:
+            url = chapter.getUrl()
+            self._chapters[url] = chapter
+            _logger.debug(u"Processing chapter `%s'.", url)
+
+            try:
+                authorName = chapter.getAuthorName()
+                if authorName:
+                    self.story.extendList('author', [authorName])
+                    authorId = chapter.getAuthorId()
+                    if authorId:
+                        authorUrl = 'http://%s/index/%s' % (self.getSiteDomain(), authorId)
+                    else:
+                        authorId = u''
+                        authorUrl = u''
+                    self.story.extendList('authorId', [authorId])
+                    self.story.extendList('authorUrl', [authorUrl])
+
+                if not self.story.getMetadata('rating'):
+                    ratingTitle = chapter.getRatingTitle()
+                    if ratingTitle:
+                        self.story.setMetadata('rating', ratingTitle)
+
+                if not self.story.getMetadata('description'):
+                    summary = chapter.getSummary()
+                    if summary:
+                        self.story.setMetadata('description', summary)
+
+                datePublished = min(datePublished, chapter.getDate())
+                dateUpdated = max(dateUpdated, chapter.getDate())
+
+                self.story.extendList('genre', chapter.getGenres())
+                self.story.extendList('characters', chapter.getCharacters())
+                self.story.extendList('ships', chapter.getPairings())
+
+                wordCount += self._getWordCount(chapter.getTextElement())
+
+                # Chapter status usually represents the story status, so we want the last chapter status.
+                # Some chapters may have no status attribute.
+                chapterInProgress = chapter.isInProgress()
+                if chapterInProgress is not None:
+                    storyInProgress = chapterInProgress
+
+                # If any chapter is adult, consider the whole story adult.
+                if chapter.isAdult():
+                    self.story.setMetadata('is_adult', True)
+                    warning = chapter.getWarning()
+                    if warning:
+                        self.story.extendList('warnings', [warning])
+
+                chapterTitle = re.sub(garbagePattern, u'', chapter.getHeading()[chapterTitleStart:])
+
+                match = re.search(indexPattern, chapterTitle)
+                if match:
+                    index = int(match.group('index'))
+                    chapterCount = max(chapterCount, index)
+                else:
+                    chapterCount += 1
+
+                self.chapterUrls.append((chapterTitle, url))
+            except ParsingError, error:
+                raise exceptions.FailedToDownload(u"Failed to download chapter `%s': %s" % (url, error))
+
+        # Some metadata are handled separately due to format conversions.
+        self.story.setMetadata('status', 'In Progress' if storyInProgress else 'Completed')
+        self.story.setMetadata('datePublished', datePublished)
+        self.story.setMetadata('dateUpdated', dateUpdated)
+        self.story.setMetadata('numWords', str(wordCount))
+        self.story.setMetadata('numChapters', chapterCount)
+
+        # Site-specific metadata.
+        self.story.setMetadata('language', self.SITE_LANGUAGE)
+
+    def getChapterText(self, url):
+        """Grabs the text for an individual chapter."""
+        if url not in self._chapters:
+            raise exceptions.FailedToDownload(u"No chapter `%s' present!" % url)
+        chapter = self._chapters[url]
+        return self.utf8FromSoup(url, chapter.getTextElement())
+
+    def _makeChapter(self, url):
+        """Creates a chapter object given a URL."""
+        document = self.make_soup(self._fetchUrl(url))
+        chapter = Chapter(self._getParsingConfiguration(), url, document)
+        return chapter
+
+    def _getWordCount(self, element):
+        """Returns word count in plain text extracted from chapter body."""
+        text = stripHTML(element)
+        count = len(re.findall(self.WORD_PATTERN, text))
+        return count
+
+    def _getParsingConfiguration(self):
+        if not self._parsingConfiguration:
+            self._parsingConfiguration = {}
+
+            adultRatings = self.getConfigList('adult_ratings')
+            if not adultRatings:
+                raise exceptions.PersonalIniFailed(
+                    u"Missing `adult_ratings' setting", u"MassEffect2.in", u"?")
+            adultRatings = set(adultRatings)
+            self._parsingConfiguration['adultRatings'] = adultRatings
+
+            ratingTitleDescriptions = self.getConfigList('rating_titles')
+            if ratingTitleDescriptions:
+                ratingTitles = {}
+                for ratingDescription in ratingTitleDescriptions:
+                    parts = ratingDescription.split(u'=')
+                    if len(parts) < 2:
+                        _logger.warning(
+                            u"Invalid `rating_titles' setting, missing `=' in `%s'."
+                            % ratingDescription)
+                        continue
+                    labels = parts[:-1]
+                    title = parts[-1]
+                    for label in labels:
+                        ratingTitles[label] = title
+                        # Duplicate label aliasing in adult rating set.
+                        if label in adultRatings:
+                            adultRatings.add(*labels)
+                self._parsingConfiguration['adultRatings'] = list(adultRatings)
+                self._parsingConfiguration['ratingTitles'] = ratingTitles
+            else:
+                raise exceptions.PersonalIniFailed(
+                    u"Missing `rating_titles' setting", u"MassEffect2.in", u"?")
+
+            self._parsingConfiguration['excludeEditorSignature'] = \
+                self.getConfig('exclude_editor_signature', False)
+
+        return self._parsingConfiguration
+
+    def _getDocumentId(self, url):
+        """Extract document ID from MassEffect2.in URL."""
+        match = re.search(self.DOCUMENT_ID_PATTERN, url)
+        if not match:
+            raise ValueError(u"Failed to extract document ID from `'" % url)
+        documentId = url[match.start():match.end()]
+        return documentId
+
+    @classmethod
+    def _makeDocumentUrl(cls, documentId):
+        """Make a chapter URL given a document ID."""
+        return 'http://%s/publ/%s' % (cls.getSiteDomain(), documentId)
+
+
+class Chapter(object):
+    """Represents a lazily-parsed chapter of a story."""
+    def __init__(self, configuration, url, document):
+        self._configuration = configuration
+        self._url = url
+        self._document = document
+        # Lazy-loaded:
+        self._heading = None
+        self._date = None
+        self._author = None
+        self._attributes = None
+        self._textElement = None
+        self._infoBar = None
+
+    def getHeading(self):
+        return self._extractHeading()
+
+    def getSummary(self):
+        attributes = self.__getAttributes()
+        if 'summary' in attributes:
+            return attributes['summary']
+
+    def getAuthorId(self):
+        author = self._getAuthor()
+        if author:
+            return author['id']
+
+    def getAuthorName(self):
+        author = self._getAuthor()
+        if author:
+            return author['name']
+
+    def getDate(self):
+        return self.__getDate()
+
+    def getRatingTitle(self):
+        attributes = self.__getAttributes()
+        if 'rating' in attributes:
+            return attributes['rating']['title']
+
+    def isAdult(self):
+        attributes = self.__getAttributes()
+        if 'rating' in attributes and attributes['rating']['isAdult']:
+            return True
+        if 'warning' in attributes:
+            return True
+        return False
+
+    def getWarning(self):
+        attributes = self.__getAttributes()
+        if 'warning' in attributes:
+            return attributes['warning']
+
+    def getCharacters(self):
+        return self._getListAttribute('characters')
+
+    def getPairings(self):
+        return self._getListAttribute('pairings')
+
+    def getGenres(self):
+        return self._getListAttribute('genres')
+
+    def isInProgress(self):
+        attributes = self.__getAttributes()
+        if 'isInProgress' in attributes:
+            return attributes['isInProgress']
+
+    def getUrl(self):
+        return self._url
+
+    def getTextElement(self):
+        return self._getTextElement()
+
+    def getPreviousChapterUrl(self):
+        """Download chapters following `Previous chapter' links.
+        Returns a list of chapters' URLs."""
+        return self._getSiblingChapterUrl({'class': 'fl tal'})
+
+    def getNextChapterUrl(self):
+        """Download chapters following `Next chapter' links.
+        Returns a list of chapters' URLs."""
+        return self._getSiblingChapterUrl({'class': 'tar fr'})
+
+    def isFromStory(self, storyTitle, prefixThreshold=-1):
+        """Check if this chapter is from a story different from the given one.
+        Prefix threshold specifies how long common story title prefix shall be
+        for chapters from one story: negative value means implementation-defined
+        optimum, zero inhibits the check, and positive value adjusts threshold."""
+
+        def getFirstWord(string):
+            match = re.search(u'^\s*\w+', string, re.UNICODE)
+            return string[match.start():match.end()]
+
+        thisStoryTitle = self.getHeading()
+        if prefixThreshold != 0:
+            if prefixThreshold < 0:
+                prefixThreshold = min(
+                    len(getFirstWord(storyTitle)), len(getFirstWord(thisStoryTitle)))
+            else:
+                prefixThreshold = min(
+                    prefixThreshold, len(storyTitle), len(thisStoryTitle))
+            result = len(_getLargestCommonPrefix(storyTitle, thisStoryTitle)) >= prefixThreshold
+            return result
+        else:
+            return storyTitle != thisStoryTitle
+
+    def _getListAttribute(self, name):
+        """Return an attribute value as a list or an empty list if the attribute is absent."""
+        attributes = self.__getAttributes()
+        if name in attributes:
+            return attributes[name]
+        return []
+
+    def _extractHeading(self):
+        """Extract header text from the document."""
+        return stripHTML(
+            self._document.find('div', {'class': 'eTitle'}).string)
+
+    def __getHeading(self):
+        """Lazily parse and return heading."""
+        if not self._heading:
+            self._heading = self._extractHeading()
+        return self._heading
+
+    def _getAuthor(self):
+        """Lazily parse and return author's information."""
+        if not self._author:
+            self._author = self._parseAuthor()
+        return self._author
+
+    def _parseAuthor(self):
+        """Locate and parse chapter author's information to a dictionary with author's `id' and `name'."""
+        try:
+            authorLink = self._getInfoBarElement() \
+                .find('i', {'class': 'icon-user'}) \
+                .findNextSibling('a')
+        except AttributeError:
+            raise ParsingError(u'Failed to locate author link.')
+        match = re.search(u'(8-\d+)', authorLink['onclick'])
+        if not match:
+            raise ParsingError(u'Failed to extract author ID.')
+        authorId = match.group(0)
+        authorName = stripHTML(authorLink.text)
+        return {
+            'id': authorId,
+            'name': authorName
+        }
+
+    def __getDate(self):
+        """Lazily parse chapter date."""
+        if not self._date:
+            self._date = self._parseDate()
+        return self._date
+
+    def _parseDate(self):
+        """Locate and parse chapter date."""
+        try:
+            dateText = self._getInfoBarElement() \
+                .find('i', {'class': 'icon-eye'}) \
+                .findPreviousSibling(text=True) \
+                .strip(u'| \n')
+        except AttributeError:
+            raise ParsingError(u'Failed to locate date.')
+        date = makeDate(dateText, '%d.%m.%Y')
+        return date
+
+    def _getInfoBarElement(self):
+        """Locate informational bar element, containing chapter date and author, on the page."""
+        if not self._infoBar:
+            self._infoBar = self._document.find('td', {'class': 'eDetails2'})
+            if not self._infoBar:
+                raise ParsingError(u'No informational bar found.')
+        return self._infoBar
+
+    def __getAttributes(self):
+        """Lazily parse attributes."""
+        if not self._attributes:
+            self._attributes = self._parseAttributes()
+        return self._attributes
+
+    def _parseAttributes(self):
+        """Parse chapter attribute block and return it as a dictionary with standard entries."""
+
+        attributes = {}
+        attributesText = u''
+        try:
+            starter = self._document \
+                .find('div', {'class': 'comm-div'}) \
+                .findNextSibling('div', {'class': 'cb'})
+            bound = starter.findNextSibling('div', {'class': 'cb'})
+
+            def processElement(element):
+                """Return textual representation an *inline* element of chapter attribute block."""
+                result = u''
+                if isinstance(element, bs4.Tag):
+                    if element.name in ('b', 'strong', 'font', 'br'):
+                        result += u"\n"
+                    if element.name == 's':
+                        result += u"<s>%s</s>" % stripHTML(element)
+                    else:
+                        result += stripHTML(element)
+                else:
+                    result += removeEntities(element)
+                return result
+
+            elements = starter.nextSiblingGenerator()
+            for element in elements:
+                if isinstance(element, bs4.Tag):
+                    if element == bound:
+                        break
+                    else:
+                        if element.name in ('div', 'p'):
+                            attributesText += u"\n"
+                            for child in element.childGenerator():
+                                attributesText += processElement(child)
+                            continue
+                attributesText += processElement(element)
+
+            elements = starter.nextGenerator()
+            for element in elements:
+                if isinstance(element, bs4.Tag):
+                    if element == bound:
+                        break
+                    elif element.name == 'img':
+                        rating = self._parseRatingFromImage(element)
+                        if rating:
+                            attributes['rating'] = rating
+                            break
+        except AttributeError or TypeError:
+            raise ParsingError(u'Failed to locate and collect attributes.')
+
+        separators = u"\r\n :;."
+        freestandingText = u''
+        for line in attributesText.split(u'\n'):
+            if line.count(u':') != 1:
+                freestandingText += line
+                continue
+            key, value = line.split(u':', 1)
+            key = key.strip(separators).lower()
+            value = value.strip().strip(separators)
+            parsed = self._parseAttribute(key, value)
+            for parsedKey, parsedValue in parsed.iteritems():
+                attributes[parsedKey] = parsedValue
+
+        freestandingText = freestandingText.strip()
+        if 'summary' not in attributes and freestandingText:
+            attributes['summary'] = freestandingText
+
+        if 'rating' not in attributes:
+            _logger.warning(u"Failed to locate or recognize rating for `%s'!", self.getUrl())
+
+        return attributes
+
+    # Most, but not all, URLs of rating icons match this.
+    RATING_LABEL_PATTERN = re.compile(u'/(?P<rating>[ERATINnG]+)\.png$')
+
+    def _parseRatingFromImage(self, element):
+        """Given an image element, try to parse story rating from it."""
+        # Although deprecated, `has_key()' is required here.
+        if not element.has_attr('src'):
+            return
+        source = element['src']
+        if 'REITiNG' in source:
+            match = re.search(self.RATING_LABEL_PATTERN, source)
+            if not match:
+                return
+            label = match.group('rating')
+            if label in self._configuration['ratingTitles']:
+                return {
+                    'label': label,
+                    'title': self._configuration['ratingTitles'][label],
+                    'isAdult': label in self._configuration['adultRatings']
+                }
+            else:
+                _logger.warning(u"No title found for rating label `%s'!" % label)
+        # TODO: conduct a research on such abnormal URLs.
+        elif source == 'http://www.masseffect2.in/_fr/10/1360399.png':
+            label = 'Nn'
+            return {
+                'label': 'Nn',
+                'title': self._configuration['ratingTitles'][label],
+                'isAdult': label in self._configuration['adultRatings']
+            }
+
+    # Various `et cetera' and `et al' forms in Russian texts.
+    # Intended to be used with whole strings!
+    ETC_PATTERN = re.compile(
+        u'''[и&]\s(?:
+              (?:т\.?\s?[пд]?\.?)|
+              (?:др(?:угие|\.)?)|
+              (?:пр(?:очие|\.)?)|
+              # Note: identically looking letters `K' and `o'
+              # below are from Latin and Cyrillic alphabets.
+              (?:ко(?:мпания)?|[KК][oо°])
+            )$
+        ''',
+        re.IGNORECASE + re.UNICODE + re.VERBOSE)
+
+    # `Author's Notes' and its variants in Russian.
+    ANNOTATION_PATTERN = re.compile(u'аннотация|описание|(?:(?:за|при)мечание\s)?(?:от\s)?автора', re.UNICODE)
+
+    def _parseAttribute(self, key, value):
+        """
+        Parse a single a single record in chapter attributes for chapter metadata.
+        Return a dictionary of canonical attributes and values (i. e. multiple attributes may be discovered).
+        """
+
+        def refineCharacter(name):
+            """Refines character name from stop-words and distortions."""
+            strippedName = name.strip()
+            nameOnly = re.sub(self.ETC_PATTERN, u'', strippedName)
+            # TODO: extract canonical name (even ME-specific?).
+            canonicalName = nameOnly
+            return canonicalName
+
+        if re.match(u'жанры?', key, re.UNICODE):
+            genres = filter(bool, map(unicode.strip, re.split(u'[,;/]', value)))
+            return {'genres': genres}
+        elif key == u'статус':
+            isInProgress = value == u'в процессе'
+            return {'isInProgress': isInProgress}
+        elif key == u'персонажи':
+            participants = map(refineCharacter, re.split(u'[,;]', value))
+            characters = []
+            pairings = []
+            for participant in participants:
+                if u'/' in participant:
+                    pairings.append(participant)
+                else:
+                    characters.append(participant)
+            return {
+                'characters': characters,
+                'pairings': pairings
+            }
+        elif key == u'предупреждение':
+            return {'warning': value}
+        elif re.match(self.ANNOTATION_PATTERN, key):
+            if not value.endswith(u'.'):
+                value += u'.'
+            # Capitalize would make value[1:] lowercase, which we don't want.
+            value = value[:1].upper() + value[1:]
+            return {'summary': value}
+        else:
+            _logger.info(u"Unrecognized attribute `%s' ignored.", key)
+            return {}
+
+    def _getTextElement(self):
+        """Locate chapter body text element on the page."""
+        if not self._textElement:
+            self._textElement = self.__collectTextElements()
+        return self._textElement
+
+    def __collectTextElements(self):
+        """Return all elements containing parts of chapter text (which may be
+        <p>aragraphs, <div>isions or plain text nodes) under a single root."""
+        starter = self._document.find('div', {'id': u'article'})
+        if starter is None:
+            # FIXME: This will occur if the method is called more than once.
+            # The reason is elements appended to `root' are removed from the document.
+            # BS 4.4 implements cloning via `copy.copy()', but supporting it for BS 4.3
+            # would be error-prone (due to relying on BS internals) and is not needed.
+            if self._textElement:
+                _logger.debug(u"You may not call this function more than once!")
+            raise ParsingError(u'Failed to locate text.')
+        collection = [starter]
+        for element in starter.nextSiblingGenerator():
+            if element is None:
+                break
+            if isinstance(element, bs4.Tag) and element.name == 'tr':
+                break
+            collection.append(element)
+        root = bs4.Tag(name='td')
+        for element in collection:
+            root.append(element)
+
+        if self._configuration['excludeEditorSignature']:
+            root = self._excludeEditorSignature(root)
+
+        return root
+
+    def _getSiblingChapterUrl(self, selector):
+        """Locate a link to a sibling chapter, either previous or next one, and return its URL."""
+        block = self._document \
+            .find('td', {'class': 'eDetails1'}) \
+            .find('div', selector)
+        if not block:
+            return
+        link = block.find('a')
+        if not link:
+            return
+        return link['href']
+
+    # Editor signature always starts with something like this.
+    SIGNED_PATTERN = re.compile(u'отредактирова(?:но|ла?)[:.\s]', re.IGNORECASE + re.UNICODE)
+
+    def _excludeEditorSignature(self, root):
+        """Exclude editor signature from within `root' element."""
+        for textNode in root.findAll(text=True):
+            if re.match(self.SIGNED_PATTERN, textNode.string):
+                editorLink = textNode.findNext('a')
+                if editorLink:
+                    editorLink.extract()
+                # Seldom editor link has inner formatting, which is sibling DOM-wise.
+                editorName = textNode.findNext('i')
+                if editorName:
+                    editorName.extract()
+                textNode.extract()
+                # We could try removing container element, but there is a risk
+                # of removing text ending with it.  Better play safe here.
+                break
+        return root
+
+
+def _getLargestCommonPrefix(*args):
+    """Returns largest common prefix of all unicode arguments, ignoring case.
+    :rtype : unicode
+    """
+    from itertools import takewhile, izip
+    toLower = lambda xs: map(lambda x: x.lower(), xs)
+    allSame = lambda xs: len(set(toLower(xs))) == 1
+    return u''.join([i[0] for i in takewhile(allSame, izip(*args))])

fanficfare/adapters/adapter_mediaminerorg.py

@@ -22,7 +22,6 @@ import re
 import urllib
 import urllib2
 
-from .. import BeautifulSoup as bs
 from ..htmlcleanup import stripHTML
 from .. import exceptions as exceptions
 
@@ -42,7 +41,12 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
         # get storyId from url--url validation guarantees query correct
         m = re.match(self.getSiteURLPattern(),url)
         if m:
-            self.story.setMetadata('storyId',m.group('id'))
+            if m.group('id'):
+                self.story.setMetadata('storyId',m.group('id'))
+            elif m.group('id2'):
+                self.story.setMetadata('storyId',m.group('id2'))
+            elif m.group('id3'):
+                self.story.setMetadata('storyId',m.group('id2'))
             
             # normalized story URL.
             self._setURL('http://' + self.getSiteDomain() + '/fanfic/view_st.php/'+self.story.getMetadata('storyId'))
@@ -62,8 +66,17 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
     def getSiteURLPattern(self):
         ##  http://www.mediaminer.org/fanfic/view_st.php/76882
         ##  http://www.mediaminer.org/fanfic/view_ch.php/167618/594087#fic_c
+        ##  http://www.mediaminer.org/fanfic/view_ch.php?submit=View+Chapter&id=105816&cid=357151
+        ##  http://www.mediaminer.org/fanfic/view_ch.php?cid=612153&submit=View+Chapter&id=171668
         return re.escape("http://"+self.getSiteDomain())+\
-            "/fanfic/view_(st|ch)\.php/"+r"(?P<id>\d+)(/\d+(#fic_c)?)?$"
+            r"/fanfic/view_(st|ch)\.php"+\
+            r"(/(?P<id>\d+)(/\d+(#fic_c)?)?/?|"+\
+            r"\?((submit=View(\+| )Chapter|id=(?P<id2>\d+)|cid=\d+)&?)+)"
+
+    # Override stripURLParameters so the id parameter won't get stripped
+    @classmethod
+    def stripURLParameters(cls, url):
+        return url
 
     def extractChapterUrlsAndMetadata(self):
 
@@ -71,7 +84,7 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
         logger.debug("URL: "+url)
 
         try:
-            data = self._fetchUrl(url)
+            data = self._fetchUrl(url+'/') # trailing / gets 'chapter list' page even for one-shots.
         except urllib2.HTTPError, e:
             if e.code == 404:
                 raise exceptions.StoryDoesNotExist(self.url)
@@ -79,7 +92,7 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
                 raise e
 
         # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
+        soup = self.make_soup(data)
 
         # [ A - All Readers ], strip '[' ']'
         ## Above title because we remove the smtxt font to get title.
@@ -106,18 +119,12 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
         title = soup.find('td',{'class':'ffh'})
         for font in title.findAll('font'):
             font.extract() # removes 'font' tags from inside the td.        
-        if title.has_key('colspan'):
+        if title.has_attr('colspan'):
             titlet = stripHTML(title)
         else:
             ## No colspan, it's part chapter title--even if it's a one-shot.
             titlet = ':'.join(stripHTML(title).split(':')[:-1]) # strip trailing 'Chapter X' or chapter title
         self.story.setMetadata('title',titlet)
-        ## The story title is difficult to reliably parse from the
-        ## story pages.  Getting it from the author page is, but costs
-        ## another fetch.
-        # authsoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
-        # titlea = authsoup.find('a',{'href':'/fanfic/view_st.php/'+self.story.getMetadata('storyId')})
-        # self.story.setMetadata('title',titlea.text)
 
         # save date from first for later.
         firstdate=None
@@ -137,7 +144,9 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
                 # save date from first for later.
                 if not firstdate:
                     firstdate = m.group(3)
-                self.chapterUrls.append((chapter,'http://'+self.host+'/fanfic/view_ch.php/'+self.story.getMetadata('storyId')+'/'+option['value']))
+                # http://www.mediaminer.org/fanfic/view_ch.php?cid=376587&submit=View+Chapter&id=105816
+                # self.chapterUrls.append((chapter,'http://'+self.host+'/fanfic/view_ch.php/'+self.story.getMetadata('storyId')+'/'+option['value']))
+                self.chapterUrls.append((chapter,'http://'+self.host+'/fanfic/view_ch.php?submit=View Chapter&id='+self.story.getMetadata('storyId')+'&cid='+option['value']))
         self.story.setMetadata('numChapters',len(self.chapterUrls))
 
         # category
@@ -193,38 +202,37 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
         logger.debug('Getting chapter text from: %s' % url)
 
         data=self._fetchUrl(url)
-        soup = bs.BeautifulStoneSoup(data,
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        soup = self.make_soup(data)
 
-        anchor = soup.find('a',{'name':'fic_c'})
+        header = soup.find('div',{'class':'post-meta clearfix '})
+        # print("data:%s"%data)
 
-        if None == anchor:
+        chapter=self.make_soup('<div class="story"></div>').find('div')
+
+        if None == header:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
 
         ## find divs with align=left, those are paragraphs in newer stories.
-        divlist = anchor.findAllNext('div',{'align':'left'})
+        divlist = header.findAllNext('div',{'align':'left'})
         if divlist:
             for div in divlist:
                 div.name='p' # convert to <p> mediaminer uses div with
                              # a margin for paragraphs.
-                anchor.append(div) # cheat!  stuff all the content
-                                   # divs into anchor just as a
-                                   # holder.
+                chapter.append(div)
                 del div['style']
                 del div['align']
-            anchor.name='div'
-            return self.utf8FromSoup(url,anchor)
+            return self.utf8FromSoup(url,chapter)
         
         else:
             logger.debug('Using kludgey text find for older mediaminer story.')
             ## Some older mediaminer stories are unparsable with BeautifulSoup.
             ## Really nasty formatting.  Sooo... Cheat!  Parse it ourselves a bit first.
             ## Story stuff falls between:
-            data = "<div id='HERE'>" + data[data.find('<a name="fic_c">'):] +"</div>"
-            soup = bs.BeautifulStoneSoup(data,
-                                         selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+            data = "<div id='HERE'>" + data[data.find('<div class="adWrap">'):data.find('<div class="addthis_sharing_toolbox">')] +"</div>"
+            soup = self.make_soup(data)
             for tag in soup.findAll('td',{'class':'ffh'}) + \
                     soup.findAll('div',{'class':'acl'}) + \
+                    soup.findAll('div',{'class':'adWrap'}) + \
                     soup.findAll('div',{'class':'footer smtxt'}) + \
                     soup.findAll('table',{'class':'tbbrdr'}):
                 tag.extract() # remove tag from soup.

fanficfare/adapters/adapter_nhamagicalworldsus.py

@@ -16,222 +16,24 @@
 #
 
 # Software: eFiction
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return NHAMagicalWorldsUsAdapter
+from base_efiction_adapter import BaseEfictionAdapter
 
 # Class name has to be unique.  Our convention is camel case the
 # sitename with Adapter at the end.  www is skipped.
-class NHAMagicalWorldsUsAdapter(BaseSiteAdapter):
+class NHAMagicalWorldsUsAdapter(BaseEfictionAdapter):
 
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        
-		
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','nha')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = " %d/%m/%y"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
+    @staticmethod
     def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
         return 'nha.magical-worlds.us'
 
     @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-        url = self.url
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-            
-        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
-        if m != None:
-            if self.is_adult or self.getConfig("is_adult"):
-                # We tried the default and still got a warning, so
-                # let's pull the warning number from the 'continue'
-                # link and reload data.
-                addurl = m.group(1)
-                # correct stupid & error in url.
-                addurl = addurl.replace("&","&")
-                url = self.url+'&index=1'+addurl
-                logger.debug("URL 2nd try: "+url)
-
-                try:
-                    data = self._fetchUrl(url)
-                except urllib2.HTTPError, e:
-                    if e.code == 404:
-                        raise exceptions.StoryDoesNotExist(self.url)
-                    else:
-                        raise e    
-            else:
-                raise exceptions.AdultCheckRequired(self.url)
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-            
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
-		
-        try:
-            # in case link points somewhere other than the first chapter
-            a = soup.findAll('option')[1]['value']
-            self.story.setMetadata('storyId',a.split('=',)[1])
-            url = 'http://'+self.host+'/'+a
-            soup = bs.BeautifulSoup(self._fetchUrl(url))
-        except:
-            pass
-		
-        for info in asoup.findAll('table', {'width' : '100%', 'bordercolor' : re.compile(r'#')}):
-            a = info.find('a')
-            if 'viewstory.php?sid='+self.story.getMetadata('storyId') == a['href'] or \
-                    ('viewstory.php?sid='+self.story.getMetadata('storyId')+'&') in a['href']:
-                self.story.setMetadata('title',stripHTML(a))
-                break
-		
-
-        # Find the chapters:
-        chapters=soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+'&chapter=\d+$'))
-        if len(chapters) == 0:
-            self.chapterUrls.append((self.story.getMetadata('title'),url))
-        else:
-            for chapter in chapters:
-                # just in case there's tags, like <i> in chapter titles.
-                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d):
-            try:
-                return d.name
-            except:
-                return ""
-				
-        cats = info.findAll('a',href=re.compile('categories.php'))
-        for cat in cats:
-            self.story.addToList('category',cat.string)
-					
-        a = info.find('a', href=re.compile(r'viewuser.php'))
-        val = a.nextSibling
-        svalue = ""
-        while not defaultGetattr(val) == 'br':
-            val = val.nextSibling
-        val = val.nextSibling
-        while not defaultGetattr(val) == 'br':
-            svalue += unicode(val)
-            val = val.nextSibling
-        self.setDescription(url,svalue)
+    def getSiteAbbrev(self):
+        return 'nha'
         
-        #does not provide convenient way to get word count
-        labels = info.findAll('i')
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = stripHTML(labelspan)
-
-            if 'Rating' in label:
-                self.story.setMetadata('rating', value.split(' -')[0])
-
-            if 'Genres' in label:
-                genres = value.string.split(', ')
-                for genre in genres:
-                    if 'None' not in genre:
-                        self.story.addToList('genre',genre.split(' -')[0])
-            
-            if 'Characters' in label:
-                chars = value.string.split(', ')
-                for char in chars:
-                    if 'None' not in char:
-                        self.story.addToList('characters',char.split(' -')[0])
-
-            if 'Warnings' in label:
-                warnings = value.string.split(', ')
-                for warning in warnings:
-                    if 'None' not in warning:
-                        self.story.addToList('warnings',warning.split(' -')[0])
-
-            if 'Completed' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(value.split(' -')[0], self.dateformat))
-            
-            if 'Updated' in label:
-                # there's a stray [ at the end.
-                #value = value[0:-1]
-                self.story.setMetadata('dateUpdated', makeDate(value.split(' -')[0], self.dateformat))
-
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-		
-        data = self._fetchUrl(url)
-
-        soup = bs.BeautifulSoup(data, selfClosingTags=('br','hr','span','center')) # some chapters seem to be hanging up on those tags, so it is safer to close them
-        
-        story = soup.find('div', {"id" : "story"})
-
-        if None == story:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    @classmethod
+    def getDateFormat(self):
+        return "%d/%m/%y" 
+                           
+def getClass():
+    return NHAMagicalWorldsUsAdapter
     
-        return self.utf8FromSoup(url,story)

fanficfare/adapters/adapter_ninelivesdarksolaceorg.py

@@ -0,0 +1,38 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2015 Fanficdownloader team, 2015 FanFicFare team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Software: eFiction
+import re
+from base_efiction_adapter import BaseEfictionAdapter
+
+class NineLivesDarkSolaceAdapter(BaseEfictionAdapter):
+
+    @staticmethod
+    def getSiteDomain():
+        return 'ninelives.dark-solace.org'
+
+    @classmethod
+    def getSiteAbbrev(self):
+        return '9lvs'
+        
+    @classmethod
+    def getDateFormat(self):
+        return "%B %d, %Y" 
+                           
+def getClass():
+    return NineLivesDarkSolaceAdapter
+    

fanficfare/adapters/adapter_portkeyorg.py

@@ -258,6 +258,7 @@ class PortkeyOrgAdapter(BaseSiteAdapter): # XXX
         data = self._fetchUrl(url)
 
         data = data.replace("HTML>","div>")
+        data = data.replace("html>","div>")
         
         soup = bs.BeautifulSoup(data)
 

fanficfare/adapters/adapter_questionablequestingcom.py

@@ -0,0 +1,35 @@
+#  -*- coding: utf-8 -*-
+
+# Copyright 2015 FanFicFare team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from base_xenforoforum_adapter import BaseXenForoForumAdapter
+
+def getClass():
+    return QuestionablequestingComAdapter
+
+class QuestionablequestingComAdapter(BaseXenForoForumAdapter):
+
+    def __init__(self, config, url):
+        BaseXenForoForumAdapter.__init__(self, config, url)
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','qq')
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'questionablequesting.com'
+

fanficfare/adapters/adapter_spikeluvercom.py

@@ -106,7 +106,9 @@ class SpikeluverComAdapter(BaseSiteAdapter):
 
         listbox_tag = soup.find('div', {'class': 'listbox'})
         for span_tag in listbox_tag('span'):
-            key = span_tag.string.strip(' :')
+            key = span_tag.string
+            if key:
+                key = key.strip(' :')
             try:
                 value = stripHTML(span_tag.nextSibling)
             # This can happen with some fancy markup in the summary. Just
@@ -135,8 +137,10 @@ class SpikeluverComAdapter(BaseSiteAdapter):
                         contents.append(sibling)
 
                 # Remove the preceding break line tag and other crud
-                contents.pop()
-                contents.pop()
+                if contents:
+                    contents.pop()
+                if contents:
+                    contents.pop()
                 self.story.setMetadata('description', ''.join(contents))
 
             elif key == 'Rated':

fanficfare/adapters/adapter_storiesonlinenet.py

@@ -114,7 +114,7 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
         return True
 
     ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
+    def doExtractChapterUrlsAndMetadata(self, get_cover=True):
 
         # index=1 makes sure we see the story chapter index.  Some
         # sites skip that for one-chapter stories.
@@ -259,7 +259,7 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
                 logger.debug("Retrieving Universe - have page")
                 if universe_soup:
                     logger.debug("Retrieving Universe - looking for name")
-                    universe_name = universe_soup.find('h1', {'id' : 'ptitle'}).text.partition(' —')[0]
+                    universe_name = universe_soup.find('h1', {'id' : 'ptitle'}).text.partition('—')[0]
                     logger.debug("Universes name: '{0}'".format(universe_name))
 
                 self.story.setMetadata('universeUrl',universeUrl)
@@ -301,7 +301,21 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
 
             if 'Updated' in label:
                 self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-#
+
+        # Some books have a cover in the index page.
+        # Samples are:
+        #     http://storiesonline.net/s/11999
+        #     http://storiesonline.net/s/10823
+        if get_cover:
+            logger.debug("Looking for the cover image...")
+            cover_url = ""
+            img = soup.find('img')
+            if img:
+                cover_url=img['src']
+            logger.debug("cover_url: %s"%cover_url)
+            if cover_url:
+                self.setCoverImage(url,cover_url)
+
         status = lc4.find('span', {'class' : 'ab'})
         if  status != None:
             self.story.setMetadata('status', 'In-Progress')

fanficfare/adapters/base_adapter.py

@@ -104,6 +104,8 @@ class BaseSiteAdapter(Configurable):
         self.chapterFirst = None
         self.chapterLast = None
         self.oldchapters = None
+        self.oldchaptersmap = None
+        self.oldchaptersdata = None
         self.oldimgs = None
         self.oldcover = None # (data of existing cover html, data of existing cover image)
         self.calibrebookmark = None
@@ -140,28 +142,6 @@ class BaseSiteAdapter(Configurable):
         '''
         self.get_cookiejar().load(filename, ignore_discard=True, ignore_expires=True)
         
-    # def save_cookiejar(self,filename):
-    #     '''
-    #     Assumed to be a FileCookieJar if self.cookiejar set.
-    #     Takes file *name*.
-    #     '''
-    #     self.get_cookiejar().save(filename, ignore_discard=True, ignore_expires=True)
-
-    # def save_pagecache(self,filename):
-    #     '''
-    #     Writes pickle of pagecache to file *name*
-    #     '''
-    #     with open(filename, 'wb') as f:
-    #         pickle.dump(self.get_pagecache(),
-    #                     f,protocol=pickle.HIGHEST_PROTOCOL)
-        
-    # def load_pagecache(self,filename):
-    #     '''
-    #     Reads pickle of pagecache from file *name*
-    #     '''
-    #     with open(filename, 'rb') as f:
-    #         self.set_pagecache(pickle.load(f))
-
     def get_pagecache(self):
         return self.pagecache
     
@@ -185,9 +165,9 @@ class BaseSiteAdapter(Configurable):
         else:
             return None
 
-    def _set_to_pagecache(self,cachekey,data):
+    def _set_to_pagecache(self,cachekey,data,redirectedurl):
         if self.use_pagecache():
-            self.get_pagecache()[cachekey] = data
+            self.get_pagecache()[cachekey] = (data,redirectedurl)
 
     def use_pagecache(self):
         '''
@@ -257,7 +237,8 @@ class BaseSiteAdapter(Configurable):
         cachekey=self._get_cachekey(url, parameters, headers)
         if usecache and self._has_cachekey(cachekey):
             logger.debug("#####################################\npagecache HIT: %s"%cachekey)
-            return self._get_from_pagecache(cachekey)
+            data,redirecturl = self._get_from_pagecache(cachekey)
+            return data
         
         logger.debug("#####################################\npagecache MISS: %s"%cachekey)
         self.do_sleep(extrasleep)
@@ -272,13 +253,23 @@ class BaseSiteAdapter(Configurable):
                          data=urllib.urlencode(parameters),
                          headers=headers)
         data = self._decode(self.opener.open(req,None,float(self.getConfig('connect_timeout',30.0))).read())
-        self._set_to_pagecache(cachekey,data)
+        self._set_to_pagecache(cachekey,data,url)
         return data
 
     def _fetchUrlRaw(self, url,
                      parameters=None,
                      extrasleep=None,
                      usecache=True):
+        
+        return self._fetchUrlRawOpened(url,
+                                       parameters,
+                                       extrasleep,
+                                       usecache)[0]
+        
+    def _fetchUrlRawOpened(self, url,
+                           parameters=None,
+                           extrasleep=None,
+                           usecache=True):
         '''
         When should cache be cleared or not used? logins...
         
@@ -289,16 +280,25 @@ class BaseSiteAdapter(Configurable):
         cachekey=self._get_cachekey(url, parameters)
         if usecache and self._has_cachekey(cachekey):
             logger.debug("#####################################\npagecache HIT: %s"%cachekey)
-            return self._get_from_pagecache(cachekey)
+            data,redirecturl = self._get_from_pagecache(cachekey)
+            class FakeOpened:
+                def __init__(self,data,url):
+                    self.data=data
+                    self.url=url
+                def geturl(self): return self.url
+                def read(self): return self.data
+            return (data,FakeOpened(data,redirecturl))
         
         logger.debug("#####################################\npagecache MISS: %s"%cachekey)
         self.do_sleep(extrasleep)
         if parameters != None:
-            data = self.opener.open(url.replace(' ','%20'),urllib.urlencode(parameters),float(self.getConfig('connect_timeout',30.0))).read()
+            opened = self.opener.open(url.replace(' ','%20'),urllib.urlencode(parameters),float(self.getConfig('connect_timeout',30.0)))
         else:
-            data = self.opener.open(url.replace(' ','%20'),None,float(self.getConfig('connect_timeout',30.0))).read()
-        self._set_to_pagecache(cachekey,data)
-        return data
+            opened = self.opener.open(url.replace(' ','%20'),None,float(self.getConfig('connect_timeout',30.0)))
+        data = opened.read()
+        self._set_to_pagecache(cachekey,data,opened.url)
+        
+        return (data,opened)
 
     def set_sleep(self,val):
         logger.debug("\n===========\n set sleep time %s\n==========="%val)
@@ -312,20 +312,30 @@ class BaseSiteAdapter(Configurable):
         elif self.getConfig('slow_down_sleep_time'):
             time.sleep(float(self.getConfig('slow_down_sleep_time')))
         
-    # parameters is a dict()
     def _fetchUrl(self, url,
                   parameters=None,
                   usecache=True,
                   extrasleep=None):
+        return self._fetchUrlOpened(url,
+                                    parameters,
+                                    usecache,
+                                    extrasleep)[0]
+
+    # parameters is a dict()
+    def _fetchUrlOpened(self, url,
+                        parameters=None,
+                        usecache=True,
+                        extrasleep=None):
 
         excpt=None
         for sleeptime in [0, 0.5, 4, 9]:
             time.sleep(sleeptime)	
             try:
-                return self._decode(self._fetchUrlRaw(url,
+                (data,opened)=self._fetchUrlRawOpened(url,
                                                       parameters=parameters,
                                                       usecache=usecache,
-                                                      extrasleep=extrasleep))
+                                                      extrasleep=extrasleep)
+                return (self._decode(data),opened)
             except u2.HTTPError, he:
                 excpt=he
                 if he.code == 404:
@@ -345,6 +355,7 @@ class BaseSiteAdapter(Configurable):
             self.chapterFirst=int(first)-1
         if last:
             self.chapterLast=int(last)-1
+        self.story.set_chapters_range(first,last)
     
     # Does the download the first time it's called.
     def getStory(self):
@@ -352,21 +363,40 @@ class BaseSiteAdapter(Configurable):
             self.getStoryMetadataOnly(get_cover=True)
 
             for index, (title,url) in enumerate(self.chapterUrls):
+                newchap = False
                 if (self.chapterFirst!=None and index < self.chapterFirst) or \
                         (self.chapterLast!=None and index > self.chapterLast):
                     self.story.addChapter(url,
                                           removeEntities(title),
                                           None)
                 else:
-                    if self.oldchapters and index < len(self.oldchapters):
+                    data = None
+                    if self.oldchaptersmap:
+                        if url in self.oldchaptersmap:
+                            data = self.utf8FromSoup(None,
+                                                     self.oldchaptersmap[url],
+                                                     partial(cachedfetch,self._fetchUrlRaw,self.oldimgs))
+                    elif self.oldchapters and index < len(self.oldchapters):
                         data = self.utf8FromSoup(None,
                                                  self.oldchapters[index],
                                                  partial(cachedfetch,self._fetchUrlRaw,self.oldimgs))
-                    else:
+
+                    # if already marked new -- ie, origtitle and title don't match
+                    # logger.debug("self.oldchaptersdata[url]:%s"%(self.oldchaptersdata[url]))
+                    newchap = (self.oldchaptersdata is not None and
+                               url in self.oldchaptersdata and (
+                            self.oldchaptersdata[url]['chapterorigtitle'] !=
+                            self.oldchaptersdata[url]['chaptertitle']) )
+                    
+                    if not data:
                         data = self.getChapterText(url)
+                        # if had to fetch and has existing chapters
+                        newchap = bool(self.oldchapters or self.oldchaptersmap)
+                        
                     self.story.addChapter(url,
                                           removeEntities(title),
-                                          removeEntities(data))
+                                          removeEntities(data),
+                                          newchap)
             self.storyDone = True
             
             # include image, but no cover from story, add default_cover_image cover.
@@ -399,11 +429,24 @@ class BaseSiteAdapter(Configurable):
             self.doExtractChapterUrlsAndMetadata(get_cover=get_cover)
             
             if not self.story.getMetadataRaw('dateUpdated'):
-                self.story.setMetadata('dateUpdated',self.story.getMetadataRaw('datePublished'))
+                if self.story.getMetadataRaw('datePublished'):
+                    self.story.setMetadata('dateUpdated',self.story.getMetadataRaw('datePublished'))
+                else:
+                    self.story.setMetadata('dateUpdated',self.story.getMetadataRaw('dateCreated'))                
 
             self.metadataDone = True
         return self.story
 
+    def setStoryMetadata(self,metahtml):
+        if metahtml:
+            self.story.load_html_metadata(metahtml)
+            self.metadataDone = True
+            if not self.story.getMetadataRaw('dateUpdated'):
+                if self.story.getMetadataRaw('datePublished'):
+                    self.story.setMetadata('dateUpdated',self.story.getMetadataRaw('datePublished'))
+                else:
+                    self.story.setMetadata('dateUpdated',self.story.getMetadataRaw('dateCreated'))                
+    
     def hookForUpdates(self,chaptercount):
         "Usually not needed."
         return chaptercount
@@ -420,6 +463,11 @@ class BaseSiteAdapter(Configurable):
         "Only needs to be overriden if != site domain."
         return cls.getSiteDomain()
     
+    @classmethod
+    def getConfigSections(cls):
+        "Only needs to be overriden if has additional ini sections."
+        return [cls.getConfigSection()]
+    
     @classmethod
     def stripURLParameters(cls,url):
         "Only needs to be overriden if URL contains more than one parameter"
@@ -469,6 +517,13 @@ class BaseSiteAdapter(Configurable):
 
     def setDescription(self,url,svalue):
         #print("\n\nsvalue:\n%s\n"%svalue)
+        strval = u"%s"%svalue # works for either soup or string
+        if self.hasConfig('description_limit'):
+            limit = int(self.getConfig('description_limit'))
+            if limit and len(strval) > limit:
+                svalue = strval[:limit]
+
+        #print(u"[[[[[\n\n%s\n\n]]]]]]]]"%svalue) # works for either soup or string
         if self.getConfig('keep_summary_html'):
             if isinstance(svalue,basestring):
                 # bs4/html5lib add html, header and body tags, which

fanficfare/adapters/base_efiction_adapter.py

@@ -70,6 +70,11 @@ class BaseEfictionAdapter(BaseSiteAdapter):
         self.triedAcceptWarnings = False
         self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
 
+    @classmethod
+    def getConfigSections(cls):
+        "Only needs to be overriden if has additional ini sections."
+        return ['base_efiction',cls.getConfigSection()]
+    
     @classmethod
     def getAcceptDomains(cls):
         return [cls.getSiteDomain(),'www.' + cls.getSiteDomain()]
@@ -264,7 +269,6 @@ class BaseEfictionAdapter(BaseSiteAdapter):
                 else:
                     super(NameOfMyAdapter, self).handleMetadata(key, value)
         """
-        # logger.debug("metadata: '%s' == '%s'" % (key, value))
         if value == 'None':
             return
         elif key == 'Summary':
@@ -287,7 +291,7 @@ class BaseEfictionAdapter(BaseSiteAdapter):
                 self.story.addToList('challenge', val)
         elif key == 'Chapters':
             self.story.setMetadata('numChapters', int(value))
-        elif key == 'Rating':
+        elif key == 'Rating' or key == 'Rated':
             self.story.setMetadata('rating', value)
         elif key == 'Word count':
             self.story.setMetadata('numWords', value)

fanficfare/adapters/base_xenforoforum_adapter.py

@@ -0,0 +1,229 @@
+#  -*- coding: utf-8 -*-
+
+# Copyright 2015 FanFicFare team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+logger = logging.getLogger(__name__)
+
+class BaseXenForoForumAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["utf8",
+                       "Windows-1252"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+							   
+							   
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])        
+        
+        # get storyId from url--url validation guarantees query correct
+        m = re.match(self.getSiteURLPattern(),url)
+        if m:
+            self.story.setMetadata('storyId',m.group('id'))
+
+            # normalized story URL.
+            self._setURL(self.getURLPrefix() + '/'+m.group('tp')+'/'+self.story.getMetadata('storyId')+'/')
+        else:
+            raise exceptions.InvalidStoryURL(url,
+                                             self.getSiteDomain(),
+                                             self.getSiteExampleURLs())
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','fsb')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%b %d, %Y at %I:%M %p"
+            
+    @classmethod
+    def getConfigSections(cls):
+        "Only needs to be overriden if has additional ini sections."
+        return ['base_xenforoforum',cls.getConfigSection()]
+    
+    @classmethod
+    def getURLPrefix(cls):
+        # The site domain.  Does have www here, if it uses it.
+        return 'https://' + cls.getSiteDomain() 
+
+    @classmethod
+    def getSiteExampleURLs(cls):
+        return cls.getURLPrefix()+"/threads/some-story-name.123456/"
+
+    def getSiteURLPattern(self):
+        return r"https?://"+re.escape(self.getSiteDomain())+r"/(?P<tp>threads|posts)/(.+\.)?(?P<id>\d+)/?"
+        
+    def use_pagecache(self):
+        '''
+        adapters that will work with the page cache need to implement
+        this and change it to True.
+        '''
+        return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        useurl = self.url
+        logger.info("url: "+useurl)
+
+        try:
+            (data,opened) = self._fetchUrlOpened(useurl)
+            useurl = opened.geturl()
+            logger.info("use useurl: "+useurl)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = self.make_soup(data)
+
+        a = soup.find('h3',{'class':'userText'}).find('a')
+        self.story.addToList('authorId',a['href'].split('/')[1])
+        self.story.addToList('authorUrl',self.getURLPrefix()+'/'+a['href'])
+        self.story.addToList('author',a.text)
+
+        h1 = soup.find('div',{'class':'titleBar'}).h1
+        self.story.setMetadata('title',stripHTML(h1))
+        
+        if '#' in useurl:
+            anchorid = useurl.split('#')[1]
+            soup = soup.find('li',id=anchorid)
+        else:
+            # try threadmarks if no '#' in , require at least 2.
+            threadmarksa = soup.find('a',{'class':'threadmarksTrigger'})
+            if threadmarksa:
+                soupmarks = self.make_soup(self._fetchUrl(self.getURLPrefix()+'/'+threadmarksa['href']))
+                markas = soupmarks.find('ol',{'class':'overlayScroll'}).find_all('a')
+                if len(markas) > 1:
+                    for (atag,url,name) in [ (x,x['href'],stripHTML(x)) for x in markas ]:
+                        date = self.make_date(atag.find_next_sibling('div',{'class':'extra'}))
+                        if not self.story.getMetadataRaw('datePublished') or date < self.story.getMetadataRaw('datePublished'):
+                            self.story.setMetadata('datePublished', date)
+                        if not self.story.getMetadataRaw('dateUpdated') or date > self.story.getMetadataRaw('dateUpdated'):
+                            self.story.setMetadata('dateUpdated', date)
+                            
+                        self.chapterUrls.append((name,self.getURLPrefix()+'/'+url))
+                        
+            soup = soup.find('li',{'class':'message'}) # limit first post for date stuff below. ('#' posts above)
+                
+        # Now go hunting for the 'chapter list'.
+        bq = soup.find('blockquote') # assume first posting contains TOC urls.
+        
+        bq.name='div'
+
+        for iframe in bq.find_all('iframe'):
+            iframe.extract() # calibre book reader & editor don't like iframes to youtube.
+
+        for qdiv in bq.find_all('div',{'class':'quoteExpand'}):
+            qdiv.extract() # Remove <div class="quoteExpand">click to expand</div>
+            
+        self.setDescription(useurl,bq)
+
+        # otherwise, use first post links--include first post since
+        # that's often also the first chapter.
+        if not self.chapterUrls:
+            self.chapterUrls.append(("First Post",useurl))
+            for (url,name) in [ (x['href'],stripHTML(x)) for x in bq.find_all('a') ]:
+                logger.debug("found chapurl:%s"%url)
+                if not url.startswith('http'):
+                    url = self.getURLPrefix()+'/'+url
+    
+                if ( url.startswith(self.getURLPrefix()) or
+                     url.startswith('http://'+self.getSiteDomain()) or
+                     url.startswith('https://'+self.getSiteDomain()) ) and ('/posts/' in url or '/threads/' in url):
+                    # brute force way to deal with SB's http->https change when hardcoded http urls.
+                    url = url.replace('http://'+self.getSiteDomain(),self.getURLPrefix())
+                    logger.debug("used chapurl:%s"%(url))
+                    self.chapterUrls.append((name,url))
+                    if url == useurl and 'First Post' == self.chapterUrls[0][0]:
+                        # remove "First Post" if included in list.
+                        logger.debug("delete dup 'First Post' chapter: %s %s"%self.chapterUrls[0])
+                        del self.chapterUrls[0]
+                        
+            # Didn't use threadmarks, so take created/updated dates
+            # from the 'first' posting created and updated.
+            date = self.make_date(soup.find('a',{'class':'datePermalink'}))
+            if date:
+                self.story.setMetadata('datePublished', date)
+                self.story.setMetadata('dateUpdated', date) # updated overwritten below if found.
+        
+            date = self.make_date(soup.find('div',{'class':'editDate'}))
+            if date:
+                self.story.setMetadata('dateUpdated', date) 
+            
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+    def make_date(self,parenttag): # forums use a BS thing where dates
+                                  # can appear different if recent.
+        datestr=None
+        try:
+            datetag = parenttag.find('span',{'class':'DateTime'})
+            if datetag:
+                datestr = datetag['title']
+            else:
+                datetag = parenttag.find('abbr',{'class':'DateTime'})
+                if datetag:
+                    datestr="%s at %s"%(datetag['data-datestring'],datetag['data-timestring'])
+            # Apr 24, 2015 at 4:39 AM
+            # May 1, 2015 at 5:47 AM
+            datestr = re.sub(r' (\d[^\d])',r' 0\1',datestr) # add leading 0 for single digit day & hours.
+            return makeDate(datestr, self.dateformat)
+        except:
+            logger.debug('No date found in %s'%parenttag)
+            return None
+        
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+        logger.debug('Getting chapter text from: %s' % url)
+
+        origurl = url
+        (data,opened) = self._fetchUrlOpened(url)
+        url = opened.geturl()
+        if '#' in origurl and '#' not in url:
+            url = url + origurl[origurl.index('#'):]
+        logger.debug("chapter URL redirected to: %s"%url)
+
+        soup = self.make_soup(data)
+
+        if '#' in url:
+            anchorid = url.split('#')[1]
+            soup = soup.find('li',id=anchorid)
+        bq = soup.find('blockquote')
+
+        bq.name='div'
+
+        for iframe in bq.find_all('iframe'):
+            iframe.extract() # calibre book reader & editor don't like iframes to youtube.
+
+        for qdiv in bq.find_all('div',{'class':'quoteExpand'}):
+            qdiv.extract() # Remove <div class="quoteExpand">click to expand</div>
+
+        return self.utf8FromSoup(url,bq)

fanficfare/cli.py

@@ -16,7 +16,7 @@
 #
 
 from optparse import OptionParser
-from os.path import expanduser, isfile, join, dirname
+from os.path import expanduser, join, dirname
 from os import access, R_OK
 from subprocess import call
 import ConfigParser
@@ -41,12 +41,14 @@ try:
     # running under calibre
     from calibre_plugins.fanfictiondownloader_plugin.fanficfare import adapters, writers, exceptions
     from calibre_plugins.fanfictiondownloader_plugin.fanficfare.configurable import Configuration
-    from calibre_plugins.fanfictiondownloader_plugin.fanficfare.epubutils import get_dcsource_chaptercount, get_update_data
+    from calibre_plugins.fanfictiondownloader_plugin.fanficfare.epubutils import (
+        get_dcsource_chaptercount, get_update_data, reset_orig_chapters_epub)
     from calibre_plugins.fanfictiondownloader_plugin.fanficfare.geturls import get_urls_from_page
 except ImportError:
     from fanficfare import adapters, writers, exceptions
     from fanficfare.configurable import Configuration
-    from fanficfare.epubutils import get_dcsource_chaptercount, get_update_data
+    from fanficfare.epubutils import (
+        get_dcsource_chaptercount, get_update_data, reset_orig_chapters_epub)
     from fanficfare.geturls import get_urls_from_page
 
 
@@ -87,6 +89,9 @@ def main(argv=None, parser=None, passed_defaultsini=None, passed_personalini=Non
     parser.add_option('-u', '--update-epub',
                       action='store_true', dest='update',
                       help='Update an existing epub with new chapters, give epub filename instead of storyurl.', )
+    parser.add_option('--unnew',
+                      action='store_true', dest='unnew',
+                      help='Remove (new) chapter marks left by mark_new_chapters setting.', )
     parser.add_option('--update-cover',
                       action='store_true', dest='updatecover',
                       help='Update cover in an existing epub, otherwise existing cover (if any) is used on update.  Only valid with --update-epub.', )
@@ -129,6 +134,9 @@ def main(argv=None, parser=None, passed_defaultsini=None, passed_personalini=Non
     if options.update and options.format != 'epub':
         parser.error('-u/--update-epub only works with epub')
 
+    if options.unnew and options.format != 'epub':
+        parser.error('--unnew only works with epub')
+
     # for passing in a file list
     if options.infile:
         urls=[]
@@ -168,6 +176,12 @@ def do_download(arg,
     # Attempt to update an existing epub.
     chaptercount = None
     output_filename = None
+
+    if options.unnew:
+        # remove mark_new_chapters marks
+        reset_orig_chapters_epub(arg,arg)
+        return
+    
     if options.update:
         try:
             url, chaptercount = get_dcsource_chaptercount(arg)
@@ -184,7 +198,7 @@ def do_download(arg,
         url = arg
         
     try:
-        configuration = Configuration(adapters.getConfigSectionFor(url), options.format)
+        configuration = Configuration(adapters.getConfigSectionsFor(url), options.format)
     except exceptions.UnknownSite, e:
         if options.list or options.normalize:
             # list for page doesn't have to be a supported site.
@@ -200,25 +214,18 @@ def do_download(arg,
     if passed_defaultsini:
         configuration.readfp(passed_defaultsini)
 
-    if isfile(join(dirname(__file__), 'defaults.ini')):
-        conflist.append(join(dirname(__file__), 'defaults.ini'))
-        
-    if isfile(join(homepath, 'defaults.ini')):
-        conflist.append(join(homepath, 'defaults.ini'))
-    if isfile(join(homepath2, 'defaults.ini')):
-        conflist.append(join(homepath2, 'defaults.ini'))
-    if isfile('defaults.ini'):
-        conflist.append('defaults.ini')
+    # don't need to check existance for our selves.
+    conflist.append(join(dirname(__file__), 'defaults.ini'))
+    conflist.append(join(homepath, 'defaults.ini'))
+    conflist.append(join(homepath2, 'defaults.ini'))
+    conflist.append('defaults.ini')
 
     if passed_personalini:
         configuration.readfp(passed_personalini)
 
-    if isfile(join(homepath, 'personal.ini')):
-        conflist.append(join(homepath, 'personal.ini'))
-    if isfile(join(homepath2, 'personal.ini')):
-        conflist.append(join(homepath2, 'personal.ini'))
-    if isfile('personal.ini'):
-        conflist.append('personal.ini')
+    conflist.append(join(homepath, 'personal.ini'))
+    conflist.append(join(homepath2, 'personal.ini'))
+    conflist.append('personal.ini')
 
     if options.configfile:
         conflist.extend(options.configfile)
@@ -318,7 +325,15 @@ def do_download(arg,
                 # update now handled by pre-populating the old
                 # images and chapters in the adapter rather than
                 # merging epubs.
-                url, chaptercount, adapter.oldchapters, adapter.oldimgs, adapter.oldcover, adapter.calibrebookmark, adapter.logfile = get_update_data(output_filename)
+                (url,
+                 chaptercount,
+                 adapter.oldchapters,
+                 adapter.oldimgs,
+                 adapter.oldcover,
+                 adapter.calibrebookmark,
+                 adapter.logfile,
+                 adapter.oldchaptersmap,
+                 adapter.oldchaptersdata) = (get_update_data(output_filename))[0:9]
 
                 print 'Do update - epub(%d) vs url(%d)' % (chaptercount, urlchaptercount)
 

fanficfare/configurable.py

@@ -17,6 +17,7 @@
 
 import ConfigParser, re
 import exceptions
+import codecs
 from ConfigParser import DEFAULTSECT, MissingSectionHeaderError, ParsingError
 
 # All of the writers(epub,html,txt) and adapters(ffnet,twlt,etc)
@@ -41,15 +42,48 @@ def re_compile(regex,line):
         return re.compile(regex)
     except Exception, e: 
         raise exceptions.RegularExpresssionFailed(e,regex,line)
-    
+
+# fall back labels.
+titleLabels = {
+    'category':'Category',
+    'genre':'Genre',
+    'language':'Language',
+    'status':'Status',
+    'series':'Series',
+    'characters':'Characters',
+    'ships':'Relationships',
+    'datePublished':'Published',
+    'dateUpdated':'Updated',
+    'dateCreated':'Packaged',
+    'rating':'Rating',
+    'warnings':'Warnings',
+    'numChapters':'Chapters',
+    'numWords':'Words',
+    'site':'Site',
+    'storyId':'Story ID',
+    'authorId':'Author ID',
+    'extratags':'Extra Tags',
+    'title':'Title',
+    'storyUrl':'Story URL',
+    'description':'Summary',
+    'author':'Author',
+    'authorUrl':'Author URL',
+    'formatname':'File Format',
+    'formatext':'File Extension',
+    'siteabbrev':'Site Abbrev',
+    'version':'Downloader Version'
+    }
+
 formatsections = ['html','txt','epub','mobi']
 othersections = ['defaults','overrides']
 
 def get_valid_sections():
-    sites = adapters.getConfigSections()
+    sites = adapters.getConfigSections() 
     sitesections = list(othersections)
     for section in sites:
         sitesections.append(section)
+        # also allows [www.base_efiction] and [www.base_forum]. Not
+        # likely to matter.
         if section.startswith('www.'):
             # add w/o www if has www
             sitesections.append(section[4:])
@@ -99,6 +133,7 @@ def get_valid_set_options():
                'replace_hr':(None,None,boollist),
                'sort_ships':(None,None,boollist),
                'strip_chapter_numbers':(None,None,boollist),
+               'mark_new_chapters':(None,None,boollist),
                'titlepage_use_table':(None,None,boollist),
                
                'use_ssl_unverified_context':(None,None,boollist),
@@ -111,7 +146,8 @@ def get_valid_set_options():
                
                'fix_fimf_blockquotes':(['fimfiction.net'],None,boollist),
                'fail_on_password':(['fimfiction.net'],None,boollist),
-               'do_update_hook':(['fimfiction.net'],None,boollist),
+               'do_update_hook':(['fimfiction.net',
+                                  'archiveofourown.org'],None,boollist),
 
                'force_login':(['phoenixsong.net'],None,boollist),
                'non_breaking_spaces':(['fictionmania.tv'],None,boollist),
@@ -170,6 +206,7 @@ def get_valid_keywords():
     return list(['(in|ex)clude_metadata_(pre|post)',
                  'add_chapter_numbers',
                  'add_genre_when_multi_category',
+                 'adult_ratings',
                  'allow_unsafe_filename',
                  'always_overwrite',
                  'anthology_tags',
@@ -178,8 +215,12 @@ def get_valid_keywords():
                  'bulk_load',
                  'chapter_end',
                  'chapter_start',
-                 'chapter_title_add_pattern',
                  'chapter_title_strip_pattern',
+                 'chapter_title_def_pattern',
+                 'chapter_title_add_pattern',
+                 'chapter_title_new_pattern',
+                 'chapter_title_addnew_pattern',
+                 'mark_new_chapters',
                  'check_next_chapter',
                  'skip_author_cover',
                  'collect_series',
@@ -192,8 +233,10 @@ def get_valid_keywords():
                  'datePublished_format',
                  'dateUpdated_format',
                  'default_cover_image',
+                 'description_limit',
                  'do_update_hook',
                  'exclude_notes',
+                 'exclude_editor_signature',
                  'extra_logpage_entries',
                  'extra_subject_tags',
                  'extra_titlepage_entries',
@@ -245,6 +288,7 @@ def get_valid_keywords():
                  'output_filename_safepattern',
                  'password',
                  'post_process_cmd',
+                 'rating_titles',
                  'remove_transparency',
                  'replace_br_with_p',
                  'replace_hr',
@@ -299,13 +343,20 @@ def make_generate_cover_settings(param):
 
 class Configuration(ConfigParser.SafeConfigParser):
 
-    def __init__(self, site, fileform):
+    def __init__(self, sections, fileform):
+        site = sections[-1] # first section is site DN.
         ConfigParser.SafeConfigParser.__init__(self)
 
         self.linenos=dict() # key by section or section,key -> lineno
         
-        self.sectionslist = ['defaults']
+        ## [injected] section has even less priority than [defaults]
+        self.sectionslist = ['defaults','injected']
 
+        ## add other sections (not including site DN) after defaults,
+        ## but before site-specific.
+        for section in sections[:-1]:
+            self.addConfigSection(section)
+        
         if site.startswith("www."):
             sitewith = site
             sitewithout = site.replace("www.","")
@@ -315,8 +366,13 @@ class Configuration(ConfigParser.SafeConfigParser):
         
         self.addConfigSection(sitewith)
         self.addConfigSection(sitewithout)
+        
         if fileform:
             self.addConfigSection(fileform)
+            ## add other sections:fileform (not including site DN)
+            ## after fileform, but before site-specific:fileform.
+            for section in sections[:-1]:
+                self.addConfigSection(section+":"+fileform)
             self.addConfigSection(sitewith+":"+fileform)
             self.addConfigSection(sitewithout+":"+fileform)
         self.addConfigSection("overrides")
@@ -404,6 +460,30 @@ class Configuration(ConfigParser.SafeConfigParser):
         else:
             return self.linenos.get(section,None)
     
+    ## Copied from Python 2.7 library so as to make read utf8.
+    def read(self, filenames):
+        """Read and parse a filename or a list of filenames.
+        Files that cannot be opened are silently ignored; this is
+        designed so that you can specify a list of potential
+        configuration file locations (e.g. current directory, user's
+        home directory, systemwide directory), and all existing
+        configuration files in the list will be read.  A single
+        filename may also be given.
+        Return list of successfully read files.
+        """
+        if isinstance(filenames, basestring):
+            filenames = [filenames]
+        read_ok = []
+        for filename in filenames:
+            try:
+                fp = codecs.open(filename,encoding='utf-8')
+            except IOError:
+                continue
+            self._read(fp, filename)
+            fp.close()
+            read_ok.append(filename)
+        return read_ok
+    
     ## Copied from Python 2.7 library so as to make it save linenos too.
     #
     # Regular expressions for parsing section headers and options.
@@ -606,3 +686,13 @@ class Configurable(object):
 
     def get_config_list(self, sections, key):
         return self.configuration.get_config_list(sections,key)
+
+    def get_label(self, entry):
+        if self.hasConfig(entry+"_label"):
+            label=self.getConfig(entry+"_label")
+        elif entry in titleLabels:
+            label=titleLabels[entry]
+        else:
+            label=entry.title()
+        return label
+

fanficfare/defaults.ini

@@ -268,6 +268,11 @@ keep_summary_html:true
 ## kept.  Some sites(AO3) use <a title="Translation">Foreign language</a>
 #keep_title_attr: false
 
+## If a chapter range was given, use this pattern for the book title.
+## replace_metadata and include/exclude will be applied *after* this.
+## Set to empty value to disable.
+title_chapter_range_pattern:${title} (Ch ${first}-${last})
+
 ## Don't like the numbers at the start of chapter titles on some
 ## sites?  You can use strip_chapter_numbers to strip them off.  Just
 ## want to make them all look the same?  Strip them off, then add them
@@ -300,12 +305,31 @@ chapter_title_strip_pattern:^[0-9]+[\.: -]+
 ## etc
 #chapter_title_strip_pattern:^([0-9]+[\.: -]+)?(Chapter *[0-9]+[\.:, -]*)?
 
-## Uses a python template substitution.  The ${index} is the 'chapter'
-## number and ${title} is the chapter title, after applying
-## chapter_title_strip_pattern.  Those are the only variables available.
-## "The Beginning" => "1. The Beginning" 
+## If true, when updating an epub that already has old chapters, new
+## chapters will be marked in the TOC and chapter header by using
+## chapter_title_new_pattern and chapter_title_addnew_pattern to set the chapter.
+mark_new_chapters:false
+
+## chapter title patterns use python template substitution.  The
+## ${index} is the 'chapter' number and ${title} is the chapter title,
+## after applying chapter_title_strip_pattern.  Those are the only
+## variables available.
+
+## The basic pattern used when not using add_chapter_numbers or
+## mark_new_chapters
+chapter_title_def_pattern:${title}
+
+## Pattern used with add_chapter_numbers, but not mark_new_chapters
 chapter_title_add_pattern:${index}. ${title}
 
+## Pattern used with mark_new_chapters, but not add_chapter_numbers
+## (new) is just text and can be changed.
+chapter_title_new_pattern:(new) ${title}
+
+## Pattern used with add_chapter_numbers and mark_new_chapters
+## (new) is just text and can be changed.
+chapter_title_addnew_pattern:${index}. (new) ${title}
+
 ## Reorder ships so b/a and c/b/a become a/b and a/b/c. Only separates
 ## on '/', so use replace_metadata to change separator first if
 ## needed. Something like: ships=>[ ]*(/|&amp;|&)[ ]*=>/ You can use
@@ -327,6 +351,74 @@ sort_ships:false
 ## User-agent
 user_agent:FFF/2.X
 
+## Added for [base_xenforoforum], but can be used with other sites,
+## too.  Limit the 'description' to the first X *characters*
+## collected.  Character count includes HTML tags, so it can be
+## non-intuitive.
+#description_limit:1000
+
+[base_efiction]
+## At the time of writing, eFiction Base adapters allow downloading
+## the whole story in bulk using the 'Print' feature. If 'bulk_load'
+## is set to 'true', both metadata and chapters can be loaded in one
+## step
+bulk_load:true
+
+[base_xenforoforum]
+## Currently only forums.spacebattles.com and forums.sufficientvelocity.com
+
+cover_exclusion_regexp:/clear.png
+
+## I saw lots of chapters name simply '1.1' etc during testing.
+strip_chapter_numbers:false
+
+## Copy title to tagsfromtitle for parsing tags.
+add_to_extra_valid_entries:,tagsfromtitle
+
+## '.NOREPL' tells the system to *not* apply title's
+## in/exclude/replace_metadata -- Only works on include_in_ lines.
+include_in_tagsfromtitle:title.NOREPL
+
+tagsfromtitle_label:Tags from Title
+
+## might want to do this, maybe not.  Will often include category, but
+## also often include non-category stuff.
+# include_in_category:tagsfromtitle 
+
+add_to_include_metadata_pre:
+# only keep tagsfromtitle with ( or [ in.
+ tagsfromtitle=~[\[\(]
+ 
+add_to_replace_metadata:
+# remove anything outside () or []
+ tagsfromtitle=>^.*?([\(\[]([^\]\)]+)[\)\]]).*?$=>\1
+# remove () []
+ tagsfromtitle=>[\(\)\[\]]=>
+# change (spaces)slash(or semicolon)(spaces) to comma
+ tagsfromtitle=> *[/;] *=>,
+ tagsfromtitle=> [xX] =>,
+ 
+# remove [] or () blocks and leading/trailing spaces/dashes/colons
+ title=>[-: ]*[\(\[]([^\]\)]+)[\)\]][-: ]*=>
+# remove 'Thread' and the next word, usually "Thread 2", "Thread
+# four", "Thread iv", etc
+ title,tagsfromtitle=>[-: ]*[Tt]hread [^ ]+[-: ]*=>
+
+add_to_extra_titlepage_entries:,tagsfromtitle
+
+## '.SPLIT' tells the system to split by ','
+add_to_include_subject_tags:,tagsfromtitle.SPLIT
+
+## base_xenforoforum reads Published and Updated datetimes from
+## Threadmarks if used, or from the posted & updated times of the
+## 'first' post if no threadmarks.
+datePublished_format:%%Y-%%m-%%d %%H:%%M:%%S
+dateUpdated_format:%%Y-%%m-%%d %%H:%%M:%%S
+
+## Only take the first X characters of the 'first' post to use as
+## the description.
+description_limit:500
+
 ## Each output format has a section that overrides [defaults]
 [html]
 
@@ -582,6 +674,8 @@ extratags: FanFiction,Testing,HTML
 ## doesn't like that.  If do_update_hook is uncommented and set true,
 ## the adapter will discard all existing chapters from the newest one
 ## on when updating to enforce accurate chapters.
+## Starting July 2015, FFF stores chapter URLs in the chapter files.
+## Stories downloaded after that shouldn't need this setting anymore.
 #do_update_hook:false
 
 ## AO3 adapter defines a few extra metadata entries.
@@ -597,6 +691,18 @@ hits_label:Hits
 collections_label:Collections
 bookmarks_label:Bookmarks
 
+## AO3 doesn't have anything it calls 'genre'.  The adapter used to be
+## hardcoded to include the site specific metadata freeformtags &
+## ao3categories in the standard metadata field genre.  By making it
+## configurable, users can change it.
+include_in_genre: freeformtags, ao3categories
+
+## AO3 uses the word 'category' differently than most sites.  The
+## adapter used to be hardcoded to include the site specific metadata
+## fandom in the standard metadata field category.  By making it
+## configurable, users can change it.
+include_in_category:fandoms
+
 ## freeformtags was previously typo'ed as freefromtags.  This way,
 ## freefromtags will still work for people who've used it.
 include_in_freefromtags:freeformtags
@@ -608,7 +714,7 @@ include_in_freefromtags:freeformtags
 #extra_subject_tags:fandoms,freeformtags,ao3categories
 
 ## AO3 chapters can include several different types of notes.  We've
-## traditional included them all in the chapter text, but this allows
+## traditionally included them all in the chapter text, but this allows
 ## you to customize which you include.  Copy this parameter to your
 ## personal.ini and list the ones you don't want.
 #exclude_notes:authorheadnotes,chaptersummary,chapterheadnotes,chapterfootnotes,authorfootnotes
@@ -758,11 +864,6 @@ extraships:Spike/Buffy
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
-## Virtually all eFiction Base adapters allow downloading the whole story in
-## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
-## metadata and chapters can be loaded in one step
-bulk_load:true
-
 [dramione.org]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -925,11 +1026,6 @@ extraships:Harry Potter/Hermione Granger
 #username:YourName
 #password:yourpassword
 
-## Virtually all eFiction Base adapters allow downloading the whole story in
-## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
-## metadata and chapters can be loaded in one step
-bulk_load:true
-
 extra_valid_entries: readings,romance
 extra_titlepage_entries: readings,romance
 readings_label: Readings
@@ -1019,6 +1115,12 @@ extra_valid_entries:size
 # don't show twitter icon.
 cover_exclusion_regexp:/res/css/bir.png
 
+[forums.spacebattles.com]
+## see [base_xenforoforum]
+
+[forums.sufficientvelocity.com]
+## see [base_xenforoforum]
+
 [grangerenchanted.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -1077,11 +1179,6 @@ extracategories:Glee RPF
 extracharacters:Darren Criss, Chris Colfer
 extraships:Darren Criss/Chris Colfer
 
-## Virtually all eFiction Base adapters allow downloading the whole story in
-## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
-## metadata and chapters can be loaded in one step
-bulk_load:true
-
 [ksarchive.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:Star Trek
@@ -1101,11 +1198,6 @@ eroticatags_label:Erotica Tags
 extra_titlepage_entries: eroticatags
 
 [lotrfanfiction.com]
-## Virtually all eFiction Base adapters allow downloading the whole story in
-## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
-## metadata and chapters can be loaded in one step
-bulk_load:true
-
 extra_valid_entries: readings
 readings_label: Readings
 
@@ -1159,6 +1251,12 @@ extracategories:NCIS
 extracategories:Buffy: The Vampire Slayer
 extracharacters:Willow
 
+[ninelives.dark-solace.org]
+## Site dedicated to these categories/characters/ships
+extracategories:The Walking Dead
+extracharacters:Carol,Daryl
+extraships:Carol/Daryl
+
 [nocturnal-light.net]
 ## Extra metadata that this adapter knows about.  See [dramione.org]
 ## for examples of how to use them.
@@ -1355,11 +1453,6 @@ extracategories:Transgender
 ## confirm they are adult for adult content.
 #is_adult:true
 
-## Virtually all eFiction Base adapters allow downloading the whole story in
-## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
-## metadata and chapters can be loaded in one step
-bulk_load:true
-
 [thehexfiles.net]
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter
@@ -1379,11 +1472,6 @@ extraships:Harry Potter/Draco Malfoy
 ## personal.ini, not defaults.ini.
 #is_adult:true
 
-## Virtually all eFiction Base adapters allow downloading the whole story in
-## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
-## metadata and chapters can be loaded in one step
-bulk_load:true
-
 ## Site dedicated to these categories/characters/ships
 extracategories:Criminal Minds
 
@@ -1393,11 +1481,6 @@ extracategories:Criminal Minds
 ## personal.ini, not defaults.ini.
 #is_adult:true
 
-## Virtually all eFiction Base adapters allow downloading the whole story in
-## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
-## metadata and chapters can be loaded in one step
-bulk_load:true
-
 extra_valid_entries: readings,challenge
 extra_titlepage_entries: readings,challenge
 challenge_label: Challenge
@@ -1466,6 +1549,8 @@ extra_titlepage_entries:readings,awards
 awards_label:Awards
 readings_label:Readings
 
+## some sites include images that we don't ever want becoming the
+## cover image.  This lets you exclude them.
 cover_exclusion_regexp:art/.*Awards.jpg
 
 [voracity2.e-fic.com]
@@ -1743,10 +1828,27 @@ extraships:InuYasha/Kagome
 ## Site dedicated to these categories/characters/ships
 extracategories:Lord of the Rings
 
-## Virtually all eFiction Base adapters allow downloading the whole story in
-## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
-## metadata and chapters can be loaded in one step
-bulk_load:true
+[www.masseffect2.in]
+## Site dedicated to this fandom.
+extracategories: Mass Effect
+
+## Ucoz.com engine, upon which MassEffect2.in is based, imposes an unspecified limit on request frequency.
+## Reports vary from `5 requests per second' to `2 requests per second for more than 10 per minute'.
+## With default settings, a several-hours IP ban may follow, so set it higher.
+slow_down_sleep_time: 2
+
+## Whether to exclude editor signature from the bottom of chapter text.
+exclude_editor_signature: false
+
+## Stories on the site almost never have cover image, and for the stories which do,
+## this may be adjusted in `personal.ini' before downloading.
+never_make_cover: true
+
+## Titles for ratings identified by 1- or 2-letter codes from `ERATING system'
+## (`система Р.Е.Й.Т.И.Н.Г.').  MassEffect2.in and some other sites adopted it,
+## but changed titles and update them occasionally.
+rating_titles: R=RESTRICTED (16+), E=EXEMPT (18+), I=ART HOUSE, T=To every, A=IN=Иной мир, Nn=Новый мир, G=О\, Господи!
+adult_ratings: E,R
 
 [www.mediaminer.org]
 

fanficfare/epubutils.py

@@ -1,18 +1,17 @@
-#!/usr/bin/env python
-# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
-from __future__ import (unicode_literals, division, absolute_import,
-                        print_function)
+# -*- coding: utf-8 -*-
 
 __license__   = 'GPL v3'
-__copyright__ = '2014, Jim Miller'
+__copyright__ = '2015, Jim Miller'
 __docformat__ = 'restructuredtext en'
 
 import logging
 logger = logging.getLogger(__name__)
 
 import re, os, traceback
-from zipfile import ZipFile
+from collections import defaultdict
+from zipfile import ZipFile, ZIP_STORED, ZIP_DEFLATED
 from xml.dom.minidom import parseString
+from StringIO import StringIO
 
 import bs4 as bs
 
@@ -90,7 +89,9 @@ def get_update_data(inputio,
 
     filecount = 0
     soups = [] # list of xhmtl blocks
+    urlsoups = {} # map of xhtml blocks by url
     images = {} # dict() longdesc->data
+    datamaps = defaultdict(dict) # map of data maps by url
     if getfilecount:
         # spin through the manifest--only place there are item tags.
         for item in contentdom.getElementsByTagName("item"):
@@ -124,20 +125,45 @@ def get_update_data(inputio,
                                 logger.warn("Image %s not found!\n(originally:%s)"%(newsrc,longdesc))
                                 logger.warn("Exception: %s"%(unicode(e)))
                                 traceback.print_exc()
-                        soup = soup.find('body')
+                        bodysoup = soup.find('body')
                         # ffdl epubs have chapter title h3
-                        h3 = soup.find('h3')
+                        h3 = bodysoup.find('h3')
                         if h3:
                             h3.extract()
                         # TtH epubs have chapter title h2
-                        h2 = soup.find('h2')
+                        h2 = bodysoup.find('h2')
                         if h2:
                             h2.extract()
 
-                        for skip in soup.findAll(attrs={'class':'skip_on_ffdl_update'}):
+                        for skip in bodysoup.findAll(attrs={'class':'skip_on_ffdl_update'}):
                             skip.extract()
+
+                        ## <meta name="chapterurl" content="${url}"></meta>
+                        #print("look for meta chapurl")
+                        currenturl = None
+                        chapurl = soup.find('meta',{'name':'chapterurl'})
+                        if chapurl:
+                            if chapurl['content'] not in urlsoups: # keep first found if more than one.
+                            #print("Found chapurl['content']:%s"%chapurl['content'])
+                                currenturl = chapurl['content']
+                                urlsoups[chapurl['content']] = bodysoup
+                        else:
+                            # for older pre-meta.  Only temp.
+                            chapa = bodysoup.find('a',{'class':'chapterurl'})
+                            if chapa and chapa['href'] not in urlsoups: # keep first found if more than one.
+                                urlsoups[chapa['href']] = bodysoup
+                                currenturl = chapa['href']
+                                chapa.extract()
+
+                        chapterorigtitle = soup.find('meta',{'name':'chapterorigtitle'})
+                        if chapterorigtitle:
+                            datamaps[currenturl]['chapterorigtitle'] = chapterorigtitle['content']
                             
-                        soups.append(soup)
+                        chaptertitle = soup.find('meta',{'name':'chaptertitle'})
+                        if chaptertitle:
+                            datamaps[currenturl]['chaptertitle'] = chaptertitle['content']
+                            
+                        soups.append(bodysoup)
                         
                     filecount+=1
 
@@ -148,7 +174,8 @@ def get_update_data(inputio,
                     
     #for k in images.keys():
         #print("\tlongdesc:%s\n\tData len:%s\n"%(k,len(images[k])))
-    return (source,filecount,soups,images,oldcover,calibrebookmark,logfile)
+    # print("datamaps:%s"%datamaps)
+    return (source,filecount,soups,images,oldcover,calibrebookmark,logfile,urlsoups,datamaps)
 
 def get_path_part(n):
     relpath = os.path.dirname(n)
@@ -192,3 +219,105 @@ def get_story_url_from_html(inputio,_is_good_url=None):
                 if _is_good_url == None or _is_good_url(ahref):
                     return ahref
     return None
+
+def reset_orig_chapters_epub(inputio,outfile):
+    inputepub = ZipFile(inputio, 'r') # works equally well with a path or a blob
+
+    ## build zip in memory in case updating in place(CLI).
+    zipio = StringIO()
+    
+    ## Write mimetype file, must be first and uncompressed.
+    ## Older versions of python(2.4/5) don't allow you to specify
+    ## compression by individual file.
+    ## Overwrite if existing output file.
+    outputepub = ZipFile(zipio, 'w', compression=ZIP_STORED)
+    outputepub.debug = 3
+    outputepub.writestr("mimetype", "application/epub+zip")
+    outputepub.close()
+
+    ## Re-open file for content.
+    outputepub = ZipFile(zipio, "a", compression=ZIP_DEFLATED)
+    outputepub.debug = 3
+
+    changed = False
+    
+    tocncxdom = parseString(inputepub.read('toc.ncx'))
+    ## spin through file contents.
+    for zf in inputepub.namelist():
+        if zf not in ['mimetype','toc.ncx'] :
+            entrychanged = False
+            data = inputepub.read(zf)
+            # if isinstance(data,unicode):
+            #     logger.debug("\n\n\ndata is unicode\n\n\n")
+            if re.match(r'.*/file\d+\.xhtml',zf):
+                data = data.decode('utf-8')
+                soup = bs.BeautifulSoup(data,"html5lib")
+                
+                chapterorigtitle = None
+                tag = soup.find('meta',{'name':'chapterorigtitle'})
+                if tag:
+                    chapterorigtitle = tag['content']
+
+                # toctitle is separate for add_chapter_numbers:toconly users.
+                chaptertoctitle = None
+                tag = soup.find('meta',{'name':'chaptertoctitle'})
+                if tag:
+                    chaptertoctitle = tag['content']
+                elif chapterorigtitle:
+                    chaptertoctitle = chapterorigtitle
+                    
+                chaptertitle = None
+                tag = soup.find('meta',{'name':'chaptertitle'})
+                if tag:
+                    chaptertitle = tag['content']
+
+                if chaptertitle and chapterorigtitle and chapterorigtitle != chaptertitle:
+                    origdata = data
+                    # print("\n%s\n%s\n"%(chapterorigtitle,chaptertitle))
+                    data = data.replace(u'<meta name="chaptertitle" content="'+chaptertitle+u'"></meta>',
+                                        u'<meta name="chaptertitle" content="'+chapterorigtitle+u'"></meta>')
+                    data = data.replace(u'<title>'+chaptertitle+u'</title>',u'<title>'+chapterorigtitle+u'</title>')
+                    data = data.replace(u'<h3>'+chaptertitle+u'</h3>',u'<h3>'+chapterorigtitle+u'</h3>')
+
+                    entrychanged = ( origdata != data )
+                    changed = changed or entrychanged
+                    
+                    if entrychanged:
+                        ## go after the TOC entry, too.
+                        # <navPoint id="file0005" playOrder="6">
+                        #   <navLabel>
+                        #     <text>5. (new) Chapter 4</text>
+                        #   </navLabel>
+                        #   <content src="OEBPS/file0005.xhtml"/>
+                        # </navPoint>
+                        for contenttag in tocncxdom.getElementsByTagName("content"):
+                            if contenttag.getAttribute('src') == zf:
+                                texttag = contenttag.parentNode.getElementsByTagName('navLabel')[0].getElementsByTagName('text')[0]
+                                texttag.childNodes[0].replaceWholeText(chaptertoctitle)
+                                # logger.debug("text label:%s"%texttag.toxml())
+                                continue
+                    
+                outputepub.writestr(zf,data.encode('utf-8'))
+            else:
+                # possibly binary data, thus no .encode().
+                outputepub.writestr(zf,data)
+
+    outputepub.writestr('toc.ncx',tocncxdom.toxml(encoding='utf-8'))
+    outputepub.close()
+    # declares all the files created by Windows.  otherwise, when
+    # it runs in appengine, windows unzips the files as 000 perms.
+    for zf in outputepub.filelist:
+        zf.create_system = 0
+        
+    # only *actually* write if changed.
+    if changed:
+        if isinstance(outfile,basestring):
+            with open(outfile,"wb") as outputio:
+                outputio.write(zipio.getvalue())
+        else:
+            outfile.write(zipio.getvalue())
+
+    inputepub.close()
+    zipio.close()
+            
+    return changed

fanficfare/geturls.py

@@ -15,12 +15,15 @@
 # limitations under the License.
 #
 
-import re
-import urlparse
-import urllib2 as u2
-
-import imaplib
+import collections
 import email
+import imaplib
+import re
+import urllib2 as u2
+import urlparse
+
+import logging
+logger = logging.getLogger(__name__)
 
 from BeautifulSoup import BeautifulSoup 
 from gziphttp import GZipProcessor
@@ -74,28 +77,25 @@ def get_urls_from_page(url,configuration=None,normalize=False):
     return get_urls_from_html(data,url,configuration,normalize,restrictsearch)
 
 def get_urls_from_html(data,url=None,configuration=None,normalize=False,restrictsearch=None):
+    urls = collections.OrderedDict()
 
-    normalized = [] # normalized url
-    retlist = [] # orig urls.
-    
     if not configuration:
         configuration = Configuration("test1.com","EPUB")
 
     soup = BeautifulSoup(data)
     if restrictsearch:
         soup = soup.find(*restrictsearch)
-        #print("restrict search:%s"%soup)
+        #logger.debug("restrict search:%s"%soup)
     
     for a in soup.findAll('a'):
         if a.has_key('href'):
-            #print("a['href']:%s"%a['href'])
+            #logger.debug("a['href']:%s"%a['href'])
             href = form_url(url,a['href'])
-            #print("1 urlhref:%s"%href)
+            #logger.debug("1 urlhref:%s"%href)
             # this (should) catch normal story links, some javascript
             # 'are you old enough' links, and 'Report This' links.
-            # The 'normalized' set prevents duplicates.
             if 'story.php' in a['href']:
-                #print("trying:%s"%a['href'])
+                #logger.debug("trying:%s"%a['href'])
                 m = re.search(r"(?P<sid>(view)?story\.php\?(sid|psid|no|story|stid)=\d+)",a['href'])
                 if m != None:
                     href = form_url(a['href'] if '//' in a['href'] else url,
@@ -103,34 +103,32 @@ def get_urls_from_html(data,url=None,configuration=None,normalize=False,restrict
                     
             try:
                 href = href.replace('&index=1','')
-                #print("2 urlhref:%s"%href)
+                #logger.debug("2 urlhref:%s"%href)
                 adapter = adapters.getAdapter(configuration,href)
-                #print("found adapter")
-                if adapter.story.getMetadata('storyUrl') not in normalized:
-                    normalized.append(adapter.story.getMetadata('storyUrl'))
-                    retlist.append(href)
+                #logger.debug("found adapter")
+                if adapter.story.getMetadata('storyUrl') not in urls:
+                    urls[adapter.story.getMetadata('storyUrl')] = [href]
+                else:
+                    urls[adapter.story.getMetadata('storyUrl')].append(href)
             except Exception, e:
-                #print e
+                #logger.debug e
                 pass
 
-    if normalize:
-        return normalized
-    else:
-        return retlist
+    # Simply return the longest URL with the assumption that it contains the
+    # most user readable metadata, if not normalized
+    return urls.keys() if normalize else [max(value, key=len) for key, value in urls.items()]
+
 
 def get_urls_from_text(data,configuration=None,normalize=False):
-
-    normalized = [] # normalized url
-    retlist = [] # orig urls.
+    urls = collections.OrderedDict()
     data=unicode(data)
-    
+
     if not configuration:
         configuration = Configuration("test1.com","EPUB")
     
     for href in re.findall('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+', data):
         # this (should) catch normal story links, some javascript
         # 'are you old enough' links, and 'Report This' links.
-        # The 'normalized' set prevents duplicates.
         if 'story.php' in href:
             m = re.search(r"(?P<sid>(view)?story\.php\?(sid|psid|no|story|stid)=\d+)",href)
             if m != None:
@@ -138,16 +136,17 @@ def get_urls_from_text(data,configuration=None,normalize=False):
         try:
             href = href.replace('&index=1','')
             adapter = adapters.getAdapter(configuration,href)
-            if adapter.story.getMetadata('storyUrl') not in normalized:
-                normalized.append(adapter.story.getMetadata('storyUrl'))
-                retlist.append(href)
+            if adapter.story.getMetadata('storyUrl') not in urls:
+                urls[adapter.story.getMetadata('storyUrl')] = [href]
+            else:
+                urls[adapter.story.getMetadata('storyUrl')].append(href)
         except:
             pass
 
-    if normalize:
-        return normalized
-    else:
-        return retlist
+    # Simply return the longest URL with the assumption that it contains the
+    # most user readable metadata, if not normalized
+    return urls.keys() if normalize else [max(value, key=len) for key, value in urls.items()]
+
 
 def form_url(parenturl,url):
      url = url.strip() # ran across an image with a space in the
@@ -177,7 +176,8 @@ def form_url(parenturl,url):
      return returl
        
 def get_urls_from_imap(srv,user,passwd,folder,markread=True):
-    
+
+    logger.debug("get_urls_from_imap srv:(%s)"%srv)
     mail = imaplib.IMAP4_SSL(srv)
     mail.login(user, passwd)
     mail.list()
@@ -186,8 +186,8 @@ def get_urls_from_imap(srv,user,passwd,folder,markread=True):
     
     result, data = mail.uid('search', None, "UNSEEN")
     
-    #print("result:%s"%result)
-    #print("data:%s"%data)
+    #logger.debug("result:%s"%result)
+    #logger.debug("data:%s"%data)
     urls=set()
     
     #latest_email_uid = data[0].split()[-1]
@@ -195,8 +195,8 @@ def get_urls_from_imap(srv,user,passwd,folder,markread=True):
 
         result, data = mail.uid('fetch', email_uid, '(BODY.PEEK[])') #RFC822
     
-        #print("result:%s"%result)
-        #print("data:%s"%data)
+        #logger.debug("result:%s"%result)
+        #logger.debug("data:%s"%data)
     
         raw_email = data[0][1]
         
@@ -205,28 +205,28 @@ def get_urls_from_imap(srv,user,passwd,folder,markread=True):
     
         email_message = email.message_from_string(raw_email)
      
-        #print "To:%s"%email_message['To']
-        #print "From:%s"%email_message['From']
-        #print "Subject:%s"%email_message['Subject']
+        #logger.debug "To:%s"%email_message['To']
+        #logger.debug "From:%s"%email_message['From']
+        #logger.debug "Subject:%s"%email_message['Subject']
     
-    #    print("payload:%s"%email_message.get_payload())
+    #    logger.debug("payload:%s"%email_message.get_payload())
 
         urllist=[]
         for part in email_message.walk():
             try:
-            #print("part mime:%s"%part.get_content_type())
+            #logger.debug("part mime:%s"%part.get_content_type())
                 if part.get_content_type() == 'text/plain':
                     urllist.extend(get_urls_from_text(part.get_payload(decode=True)))
                 if part.get_content_type() == 'text/html':
                     urllist.extend(get_urls_from_html(part.get_payload(decode=True)))
             except Exception as e:
-                print("Failed to read email content: %s"%e)
-        #print "urls:%s"%get_urls_from_text(get_first_text_block(email_message))
+                logger.error("Failed to read email content: %s"%e)
+        #logger.debug "urls:%s"%get_urls_from_text(get_first_text_block(email_message))
 
         if urllist and markread:
             #obj.store(data[0].replace(' ',','),'+FLAGS','\Seen')
             r,d = mail.uid('store',email_uid,'+FLAGS','(\\SEEN)')
-            #print("seen result:%s->%s"%(email_uid,r))
+            #logger.debug("seen result:%s->%s"%(email_uid,r))
                 
         [ urls.add(x) for x in urllist ]
     

fanficfare/htmlcleanup.py

@@ -81,10 +81,10 @@ def removeEntities(text):
     
     try:
         t = text.decode('utf-8')
-    except UnicodeEncodeError, e:
+    except (UnicodeEncodeError,UnicodeDecodeError), e:
         try:
             t = text.encode ('ascii', 'xmlcharrefreplace') 
-        except UnicodeEncodeError, e:
+        except (UnicodeEncodeError,UnicodeDecodeError), e:
             t = text
     text = t 
     # replace numeric versions of [&<>] with named versions,

fanficfare/story.py

@@ -16,18 +16,25 @@
 #
 
 import os, re
+from collections import namedtuple
 import urlparse
 import string
+import json
+import datetime
 from math import floor
 from functools import partial
 import logging
 logger = logging.getLogger(__name__)
 import urlparse as up
 
+import bs4
+
 import exceptions
 from htmlcleanup import conditionalRemoveEntities, removeAllEntities
 from configurable import Configurable, re_compile
 
+Chapter = namedtuple('Chapter', 'url title html origtitle toctitle new')
+
 SPACE_REPLACE=u'\s'
 SPLIT_META=u'\,'
 
@@ -408,7 +415,9 @@ class Story(Configurable):
         except:
             self.metadata = {'version':'4.4'}
         self.in_ex_cludes = {}
-        self.chapters = [] # chapters will be tuples of (title,html)
+        self.chapters = [] # chapters will be namedtuple of Chapter(url,title,html,etc)
+        self.chapter_first = None
+        self.chapter_last = None
         self.imgurls = []
         self.imgtuples = []
 
@@ -437,8 +446,12 @@ class Story(Configurable):
                 iel = []
                 self.in_ex_cludes[ie] = set_in_ex_clude(ies)
 
+    def set_chapters_range(self,first=None,last=None):
+        self.chapter_first=first
+        self.chapter_last=last
+                
     def join_list(self, key, vallist):
-        return self.getConfig("join_string_"+key,u", ").replace(SPACE_REPLACE,' ').join(map(unicode, vallist))
+        return self.getConfig("join_string_"+key,u", ").replace(SPACE_REPLACE,' ').join(map(unicode, [ x for x in vallist if x is not None ]))
 
     def setMetadata(self, key, value, condremoveentities=True):
 
@@ -461,7 +474,7 @@ class Story(Configurable):
 
         if key == 'dateUpdated' and value:
             # Last Update tags for Bill.
-            self.addToList('lastupdate',value.strftime("Last Update Year/Month: %Y/%m"))
+            self.addToList('lastupdate',value.strftime("Last Update Year/Month: %Y/%m"),clear=True)
             self.addToList('lastupdate',value.strftime("Last Update: %Y/%m/%d"))
 
 
@@ -544,6 +557,69 @@ class Story(Configurable):
         else:
             return self.join_list(key,retlist)
 
+    # for saving an html-ified copy of metadata.
+    def dump_html_metadata(self):
+        lines=[]
+        for k,v in sorted(self.metadata.iteritems()):
+            classes=['metadata']
+            if isinstance(v, (datetime.date, datetime.datetime, datetime.time)):
+                classes.append("datetime")
+                val = v.isoformat()
+            elif isinstance(v,list):
+                classes.append("list")
+                if '' in v:
+                    v.remove('')
+                if None in v:
+                    v.remove(None)
+                #logger.debug("k:%s v:%s"%(k,v))
+                # force ints/floats to strings.
+                val = "<ul>\n<li>%s</li>\n</ul>" % "</li>\n<li>".join([ "%s"%x for x in v ])
+            elif isinstance(v, (int)):
+                classes.append("int")
+                val = v
+            else:
+                val = v
+
+            # don't include items passed in for calibre cols, etc.
+            if not k.startswith('calibre_') and k not in ['output_css']:
+                lines.append("<p><span class='label'>%s</span>: <div class='%s' id='%s'>%s</div><p>\n"%(
+                        self.get_label(k),
+                        " ".join(classes),
+                        k,val))
+        return "\n".join(lines)
+
+    # for loading an html-ified copy of metadata.
+    def load_html_metadata(self,data):
+        soup = bs4.BeautifulSoup(data,'html5lib')
+        for tag in soup.find_all('div','metadata'):
+            val = None
+            if 'datetime' in tag['class']:
+                v = tag.string
+                try:
+                    val = datetime.datetime.strptime(v, '%Y-%m-%dT%H:%M:%S.%f')
+                except ValueError:
+                    try:
+                        val = datetime.datetime.strptime(v, '%Y-%m-%dT%H:%M:%S')
+                    except ValueError:
+                        try:
+                            val = datetime.datetime.strptime(v, '%Y-%m-%d')
+                        except ValueError:
+                            pass
+            elif 'list' in tag['class']:
+                val = []
+                for i in tag.find_all('li'):
+                    val.append(i.string)
+            elif 'int' in tag['class']:
+                val = int(tag.string)
+            else:
+                val = unicode("\n".join([ unicode(c) for c in tag.contents ]))
+                
+            #logger.debug("key(%s)=val(%s)"%(tag['id'],val))
+            if val:
+                self.metadata[tag['id']]=val
+                
+        # self.metadata = json.loads(s, object_hook=datetime_decoder)
+        
     def getMetadataRaw(self,key):
         if self.isValidMetaEntry(key) and self.metadata.has_key(key):
             return self.metadata[key]
@@ -574,6 +650,14 @@ class Story(Configurable):
                 if key in ("datePublished","dateUpdated"):
                     value = value.strftime(self.getConfig(key+"_format","%Y-%m-%d"))
 
+                if key == "title" and (self.chapter_first or self.chapter_last) and self.getConfig("title_chapter_range_pattern"):
+                    first = self.chapter_first or "1"
+                    last = self.chapter_last or self.getMetadata("numChapters")
+                    templ = string.Template(self.getConfig("title_chapter_range_pattern"))
+                    value = templ.substitute({'title':value,
+                                              'first':commaGroups(first),
+                                              'last':commaGroups(last)})
+
             if doreplacements:
                 value=self.doReplacements(value,key)
             if removeallentities and value != None:
@@ -671,9 +755,6 @@ class Story(Configurable):
         if not value in self.metadata[listname]:
             self.metadata[listname].append(value)
 
-        if listname == 'category' and self.getConfig('add_genre_when_multi_category') and len(self.metadata[listname]) > 1:
-            self.addToList('genre',self.getConfig('add_genre_when_multi_category'))
-
     def isList(self,listname):
         'Everything set with an include_in_* is considered a list.'
         return self.isListType(listname) or \
@@ -693,8 +774,12 @@ class Story(Configurable):
         # includelist prevents infinite recursion of include_in_'s
         if self.hasConfig("include_in_"+listname) and listname not in includelist:
             for k in self.getConfigList("include_in_"+listname):
+                ldorepl = doreplacements
+                if k.endswith('.NOREPL'):
+                    k = k[:-len('.NOREPL')]
+                    ldorepl = False
                 retlist.extend(self.getList(k,removeallentities=False,
-                                            doreplacements=doreplacements,includelist=includelist+[listname]))
+                                            doreplacements=ldorepl,includelist=includelist+[listname]))
         else:
 
             if not self.isList(listname):
@@ -702,6 +787,8 @@ class Story(Configurable):
                                             doreplacements=doreplacements)]
             else:
                 retlist = self.getMetadataRaw(listname)
+                if retlist is None:
+                    retlist = []
 
         if retlist:
             if doreplacements:
@@ -715,6 +802,13 @@ class Story(Configurable):
 
             retlist = filter( lambda x : x!=None and x!='' ,retlist)
 
+        if listname == 'genre' and self.getConfig('add_genre_when_multi_category') and len(self.getList('category',
+                                                                                                        removeallentities=False,
+                                                                                                        # to avoid inf loops if genre/cat substs
+                                                                                                        doreplacements=False
+                                                                                                        )) > 1:
+            retlist.append(self.getConfig('add_genre_when_multi_category'))
+
         # reorder ships so b/a and c/b/a become a/b and a/b/c.  Only on '/',
         # use replace_metadata to change separator first if needed.
         # ships=>[ ]*(/|&amp;|&)[ ]*=>/
@@ -740,7 +834,16 @@ class Story(Configurable):
 
         # metadata all go into dc:subject tags, but only if they are configured.
         for (name,value) in self.getAllMetadata(removeallentities=removeallentities,keeplists=True).iteritems():
-            if name in tags_list:
+            if name+'.SPLIT' in tags_list:
+                flist=[]
+                if isinstance(value,list):
+                    for tag in value:
+                        flist.extend(tag.split(','))
+                else:
+                    flist.extend(value)
+                for tag in flist:
+                    subjectset.add(tag)
+            elif name in tags_list:
                 if isinstance(value,list):
                     for tag in value:
                         subjectset.add(tag)
@@ -754,24 +857,63 @@ class Story(Configurable):
 
         return list(subjectset | set(self.getConfigList("extratags")))
 
-    def addChapter(self, url, title, html):
+    def addChapter(self, url, title, html, newchap=False):
+        # logger.debug("addChapter(%s,%s)"%(url,newchap))
         if self.getConfig('strip_chapter_numbers') and \
                 self.getConfig('chapter_title_strip_pattern'):
             title = re.sub(self.getConfig('chapter_title_strip_pattern'),"",title)
-        self.chapters.append( (url,title,html) )
+        self.chapters.append( Chapter(url,title,html,title,title,newchap) )
 
     def getChapters(self,fortoc=False):
-        "Chapters will be tuples of (title,html)"
+        "Chapters will be Chapter namedtuples"
         retval = []
-        ## only add numbers if more than one chapter.
-        if len(self.chapters) > 1 and \
-                (self.getConfig('add_chapter_numbers') == "true" \
-                     or (self.getConfig('add_chapter_numbers') == "toconly" and fortoc)) \
-                     and self.getConfig('chapter_title_add_pattern'):
-            for index, (url,title,html) in enumerate(self.chapters):
-                retval.append( (url,
-                                string.Template(self.getConfig('chapter_title_add_pattern')).substitute({'index':index+1,'title':title}),
-                                html) )
+
+        ## only add numbers if more than one chapter.  Ditto (new) marks.
+        if len(self.chapters) > 1:
+            addnums = ( self.getConfig('add_chapter_numbers') == "true" 
+                        or (self.getConfig('add_chapter_numbers') == "toconly" and fortoc) )
+            
+            marknew = self.getConfig('mark_new_chapters')=='true'
+
+            defpattern = self.getConfig('chapter_title_def_pattern','${title}') # default val in case of missing defaults.ini
+            if addnums and marknew:
+                pattern = self.getConfig('chapter_title_add_pattern')
+                newpattern = self.getConfig('chapter_title_addnew_pattern')
+            elif addnums:
+                pattern = self.getConfig('chapter_title_add_pattern')
+                newpattern = pattern
+            elif marknew:
+                pattern = defpattern
+                newpattern = self.getConfig('chapter_title_new_pattern')
+            else:
+                pattern = defpattern
+                newpattern = pattern
+
+            if self.getConfig('add_chapter_numbers') in ["true","toconly"]:
+                tocpattern = self.getConfig('chapter_title_add_pattern')
+            else:
+                tocpattern = defpattern
+
+            # logger.debug("Patterns: (%s)(%s)"%(pattern,newpattern))
+            templ = string.Template(pattern)
+            newtempl = string.Template(newpattern)
+            toctempl = string.Template(tocpattern)            
+            
+            for index, chap in enumerate(self.chapters):
+                if chap.new:
+                    usetempl = newtempl
+                else:
+                    usetempl = templ
+                # logger.debug("chap.url, chap.new: (%s)(%s)"%(chap.url,chap.new))
+                retval.append( Chapter(chap.url,
+                                       # 'new'
+                                       usetempl.substitute({'index':index+1,'title':chap.title}),
+                                       chap.html,
+                                       # 'orig'
+                                       templ.substitute({'index':index+1,'title':chap.title}),
+                                       # 'toc'
+                                       toctempl.substitute({'index':index+1,'title':chap.title}),
+                                       chap.new) )
         else:
             retval = self.chapters
 
@@ -838,6 +980,7 @@ class Story(Configurable):
                 #print("\n===========\nparsedUrl.path:%s\ntoppath:%s\nimgurl:%s\n\n"%(parsedUrl.path,toppath,imgurl))
 
         # apply coverexclusion to explicit covers, too.  Primarily for ffnet imageu.
+        #print("[[[[[\n\n %s %s \n\n]]]]]]]"%(imgurl,coverexclusion))
         if cover and coverexclusion and re.search(coverexclusion,imgurl):
             return (None,None)
 
@@ -929,4 +1072,3 @@ def commaGroups(s):
         groups.append(s[-3:])
         s = s[:-3]
     return s + ','.join(reversed(groups))
-

fanficfare/writers/base_writer.py

@@ -45,36 +45,6 @@ class BaseStoryWriter(Configurable):
         self.adapter = adapter
         self.story = adapter.getStoryMetadataOnly() # only cache the metadata initially.
         
-        # fall back labels.
-        self.titleLabels = {
-            'category':'Category',
-            'genre':'Genre',
-            'language':'Language',
-            'status':'Status',
-            'series':'Series',
-            'characters':'Characters',
-            'ships':'Relationships',
-            'datePublished':'Published',
-            'dateUpdated':'Updated',
-            'dateCreated':'Packaged',
-            'rating':'Rating',
-            'warnings':'Warnings',
-            'numChapters':'Chapters',
-            'numWords':'Words',
-            'site':'Site',
-            'storyId':'Story ID',
-            'authorId':'Author ID',
-            'extratags':'Extra Tags',
-            'title':'Title',
-            'storyUrl':'Story URL',
-            'description':'Summary',
-            'author':'Author',
-            'authorUrl':'Author URL',
-            'formatname':'File Format',
-            'formatext':'File Extension',
-            'siteabbrev':'Site Abbrev',
-            'version':'Downloader Version'
-            }
         self.story.setMetadata('formatname',self.getFormatName())
         self.story.setMetadata('formatext',self.getFormatExt())
 
@@ -135,15 +105,16 @@ class BaseStoryWriter(Configurable):
                             TEMPLATE=WIDE_ENTRY
                         else:
                             TEMPLATE=ENTRY
-                            
-                        if self.hasConfig(entry+"_label"):
-                            label=self.getConfig(entry+"_label")
-                        elif entry in self.titleLabels:
-                            logger.debug("Using fallback label for %s_label"%entry)
-                            label=self.titleLabels[entry]
-                        else:
-                            label="%s"%entry.title()
-                            logger.debug("No known label for %s, fallback to '%s'"%(entry,label))
+
+                        label=self.get_label(entry)
+                        # if self.hasConfig(entry+"_label"):
+                        #     label=self.getConfig(entry+"_label")
+                        # elif entry in self.titleLabels:
+                        #     logger.debug("Using fallback label for %s_label"%entry)
+                        #     label=self.titleLabels[entry]
+                        # else:
+                        #     label="%s"%entry.title()
+                        #     logger.debug("No known label for %s, fallback to '%s'"%(entry,label))
 
                         # If the label for the title entry is empty, use the
                         # 'no title' option if there is one.
@@ -177,12 +148,12 @@ class BaseStoryWriter(Configurable):
             
             self._write(out,START.substitute(self.story.getAllMetadata()))
 
-            for index, (url,title,html) in enumerate(self.story.getChapters(fortoc=True)):
-                if html:
-                    self._write(out,ENTRY.substitute({'chapter':title,
+            for index, chap in enumerate(self.story.getChapters(fortoc=True)):
+                if chap.html:
+                    self._write(out,ENTRY.substitute({'chapter':chap.title,
                                                       'number':index+1,
                                                       'index':"%04d"%(index+1),
-                                                      'url':url}))
+                                                      'url':chap.url}))
 
             self._write(out,END.substitute(self.story.getAllMetadata()))
 

fanficfare/writers/writer_epub.py

@@ -28,7 +28,7 @@ import re
 from xml.dom.minidom import parse, parseString, getDOMImplementation
 
 from base_writer import *
-from ..htmlcleanup import stripHTML
+from ..htmlcleanup import stripHTML,removeEntities
 
 logger = logging.getLogger(__name__)
 
@@ -133,6 +133,10 @@ ${value}<br />
 <head>
 <title>${chapter}</title>
 <link href="stylesheet.css" type="text/css" charset="UTF-8" rel="stylesheet"/>
+<meta name="chapterurl" content="${url}"></meta>
+<meta name="chapterorigtitle" content="${origchapter}"></meta>
+<meta name="chaptertoctitle" content="${tocchapter}"></meta>
+<meta name="chaptertitle" content="${chapter}"></meta>
 </head>
 <body>
 <h3>${chapter}</h3>
@@ -258,14 +262,15 @@ div { margin: 0pt; padding: 0pt; }
             if self.isValidMetaEntry(entry):
                 val = self.story.getMetadata(entry)
                 if val and ( entry not in oldvalues or val != oldvalues[entry] ):
-                    if self.hasConfig(entry+"_label"):
-                        label=self.getConfig(entry+"_label")
-                    elif entry in self.titleLabels:
-                        logger.debug("Using fallback label for %s_label"%entry)
-                        label=self.titleLabels[entry]
-                    else:
-                        label="%s"%entry.title()
-                        logger.debug("No known label for %s, fallback to '%s'"%(entry,label))
+                    label=self.get_label(entry)
+                    # if self.hasConfig(entry+"_label"):
+                    #     label=self.getConfig(entry+"_label")
+                    # elif entry in self.titleLabels:
+                    #     logger.debug("Using fallback label for %s_label"%entry)
+                    #     label=self.titleLabels[entry]
+                    # else:
+                    #     label="%s"%entry.title()
+                    #     logger.debug("No known label for %s, fallback to '%s'"%(entry,label))
 
                     retval = retval + ENTRY.substitute({'id':entry,
                                                         'label':label,
@@ -501,13 +506,13 @@ div { margin: 0pt; padding: 0pt; }
             items.append(("log_page","OEBPS/log_page.xhtml","application/xhtml+xml","Update Log"))
             itemrefs.append("log_page")
             
-        for index, (url,title,html) in enumerate(self.story.getChapters(fortoc=True)):
-            if html:
+        for index, chap in enumerate(self.story.getChapters(fortoc=True)):
+            if chap.html:
                 i=index+1
                 items.append(("file%04d"%i,
                               "OEBPS/file%04d.xhtml"%i,
                               "application/xhtml+xml",
-                              title))
+                              chap.title))
                 itemrefs.append("file%04d"%i)
 
         manifest = contentdom.createElement("manifest")
@@ -649,19 +654,21 @@ div { margin: 0pt; padding: 0pt; }
         else:
             CHAPTER_END = self.EPUB_CHAPTER_END
         
-        for index, (url,title,html) in enumerate(self.story.getChapters()):
-            if html:
-                logger.debug('Writing chapter text for: %s' % title)
-                vals={'url':url, 'chapter':title, 'index':"%04d"%(index+1), 'number':index+1}
-                fullhtml = CHAPTER_START.substitute(vals) + html + CHAPTER_END.substitute(vals)
+        for index, chap in enumerate(self.story.getChapters()): # (url,title,html)
+            if chap.html:
+                logger.debug('Writing chapter text for: %s' % chap.title)
+                vals={'url':removeEntities(chap.url),
+                      'chapter':chap.title,
+                      'origchapter':chap.origtitle,
+                      'tocchapter':chap.toctitle,
+                      'index':"%04d"%(index+1),
+                      'number':index+1}
+                fullhtml = CHAPTER_START.substitute(vals) + \
+                    chap.html + CHAPTER_END.substitute(vals)
                 # ffnet(& maybe others) gives the whole chapter text
                 # as one line.  This causes problems for nook(at
                 # least) when the chapter size starts getting big
                 # (200k+)
-                #fullhtml = fullhtml.replace('</p>','</p>\n').replace('<br />','<br />\n')
-                # The replaces above added tons of extra newlines
-                # during *each* epub update.  The regexp version adds
-                # only one and removes any extra.
                 fullhtml = re.sub(r'(</p>|<br ?/>)\n*',r'\1\n',fullhtml)
 
                 outputepub.writestr("OEBPS/file%04d.xhtml"%(index+1),fullhtml.encode('utf-8'))

fanficfare/writers/writer_html.py

@@ -128,12 +128,12 @@ ${output_css}
         else:
             CHAPTER_END = self.HTML_CHAPTER_END
         
-        for index, (url,title,html) in enumerate(self.story.getChapters()):
-            if html:
-                logging.debug('Writing chapter text for: %s' % title)
-                vals={'url':url, 'chapter':title, 'index':"%04d"%(index+1), 'number':index+1}
+        for index, chap in enumerate(self.story.getChapters()):
+            if chap.html:
+                logging.debug('Writing chapter text for: %s' % chap.title)
+                vals={'url':chap.url, 'chapter':chap.title, 'index':"%04d"%(index+1), 'number':index+1}
                 self._write(out,CHAPTER_START.substitute(vals))
-                self._write(out,html)
+                self._write(out,chap.html)
                 self._write(out,CHAPTER_END.substitute(vals))
 
         self._write(out,FILE_END.substitute(self.story.getAllMetadata()))

fanficfare/writers/writer_mobi.py

@@ -161,11 +161,11 @@ ${value}<br />
         else:
             CHAPTER_END = self.MOBI_CHAPTER_END
         
-        for index, (url,title,html) in enumerate(self.story.getChapters()):
-            if html:
-                logger.debug('Writing chapter text for: %s' % title)
-                vals={'url':url, 'chapter':title, 'index':"%04d"%(index+1), 'number':index+1}
-                fullhtml = CHAPTER_START.substitute(vals) + html + CHAPTER_END.substitute(vals)
+        for index, chap in enumerate(self.story.getChapters()):
+            if chap.html:
+                logger.debug('Writing chapter text for: %s' % chap.title)
+                vals={'url':chap.url, 'chapter':chap.title, 'index':"%04d"%(index+1), 'number':index+1}
+                fullhtml = CHAPTER_START.substitute(vals) + chap.html + CHAPTER_END.substitute(vals)
                 # ffnet(& maybe others) gives the whole chapter text
                 # as one line.  This causes problems for nook(at
                 # least) when the chapter size starts getting big

fanficfare/writers/writer_txt.py

@@ -154,12 +154,12 @@ End file.
         else:
             CHAPTER_END = self.TEXT_CHAPTER_END
         
-        for index, (url, title,html) in enumerate(self.story.getChapters()):
-            if html:
-                logging.debug('Writing chapter text for: %s' % title)
-                vals={'url':url, 'chapter':title, 'index':"%04d"%(index+1), 'number':index+1}
+        for index, chap in enumerate(self.story.getChapters()):
+            if chap.html:
+                logging.debug('Writing chapter text for: %s' % chap.title)
+                vals={'url':chap.url, 'chapter':chap.title, 'index':"%04d"%(index+1), 'number':index+1}
                 self._write(out,self.lineends(self.wraplines(removeAllEntities(CHAPTER_START.substitute(vals)))))
-                self._write(out,self.lineends(html2text(html,wrap_width=self.wrap_width)))
+                self._write(out,self.lineends(html2text(chap.html,wrap_width=self.wrap_width)))
                 self._write(out,self.lineends(self.wraplines(removeAllEntities(CHAPTER_END.substitute(vals)))))
 
         self._write(out,self.lineends(self.wraplines(FILE_END.substitute(self.story.getAllMetadata()))))

included_dependencies/html5lib/__init__.py

@@ -20,4 +20,4 @@ from .serializer import serialize
 
 __all__ = ["HTMLParser", "parse", "parseFragment", "getTreeBuilder",
            "getTreeWalker", "serialize"]
-__version__ = "0.999"
+__version__ = "0.99999"

included_dependencies/html5lib/constants.py

@@ -1,292 +1,290 @@
 from __future__ import absolute_import, division, unicode_literals
 
 import string
-import gettext
-_ = gettext.gettext
 
 EOF = None
 
 E = {
     "null-character":
-        _("Null character in input stream, replaced with U+FFFD."),
+        "Null character in input stream, replaced with U+FFFD.",
     "invalid-codepoint":
-        _("Invalid codepoint in stream."),
+        "Invalid codepoint in stream.",
     "incorrectly-placed-solidus":
-        _("Solidus (/) incorrectly placed in tag."),
+        "Solidus (/) incorrectly placed in tag.",
     "incorrect-cr-newline-entity":
-        _("Incorrect CR newline entity, replaced with LF."),
+        "Incorrect CR newline entity, replaced with LF.",
     "illegal-windows-1252-entity":
-        _("Entity used with illegal number (windows-1252 reference)."),
+        "Entity used with illegal number (windows-1252 reference).",
     "cant-convert-numeric-entity":
-        _("Numeric entity couldn't be converted to character "
-          "(codepoint U+%(charAsInt)08x)."),
+        "Numeric entity couldn't be converted to character "
+        "(codepoint U+%(charAsInt)08x).",
     "illegal-codepoint-for-numeric-entity":
-        _("Numeric entity represents an illegal codepoint: "
-          "U+%(charAsInt)08x."),
+        "Numeric entity represents an illegal codepoint: "
+        "U+%(charAsInt)08x.",
     "numeric-entity-without-semicolon":
-        _("Numeric entity didn't end with ';'."),
+        "Numeric entity didn't end with ';'.",
     "expected-numeric-entity-but-got-eof":
-        _("Numeric entity expected. Got end of file instead."),
+        "Numeric entity expected. Got end of file instead.",
     "expected-numeric-entity":
-        _("Numeric entity expected but none found."),
+        "Numeric entity expected but none found.",
     "named-entity-without-semicolon":
-        _("Named entity didn't end with ';'."),
+        "Named entity didn't end with ';'.",
     "expected-named-entity":
-        _("Named entity expected. Got none."),
+        "Named entity expected. Got none.",
     "attributes-in-end-tag":
-        _("End tag contains unexpected attributes."),
+        "End tag contains unexpected attributes.",
     'self-closing-flag-on-end-tag':
-        _("End tag contains unexpected self-closing flag."),
+        "End tag contains unexpected self-closing flag.",
     "expected-tag-name-but-got-right-bracket":
-        _("Expected tag name. Got '>' instead."),
+        "Expected tag name. Got '>' instead.",
     "expected-tag-name-but-got-question-mark":
-        _("Expected tag name. Got '?' instead. (HTML doesn't "
-          "support processing instructions.)"),
+        "Expected tag name. Got '?' instead. (HTML doesn't "
+        "support processing instructions.)",
     "expected-tag-name":
-        _("Expected tag name. Got something else instead"),
+        "Expected tag name. Got something else instead",
     "expected-closing-tag-but-got-right-bracket":
-        _("Expected closing tag. Got '>' instead. Ignoring '</>'."),
+        "Expected closing tag. Got '>' instead. Ignoring '</>'.",
     "expected-closing-tag-but-got-eof":
-        _("Expected closing tag. Unexpected end of file."),
+        "Expected closing tag. Unexpected end of file.",
     "expected-closing-tag-but-got-char":
-        _("Expected closing tag. Unexpected character '%(data)s' found."),
+        "Expected closing tag. Unexpected character '%(data)s' found.",
     "eof-in-tag-name":
-        _("Unexpected end of file in the tag name."),
+        "Unexpected end of file in the tag name.",
     "expected-attribute-name-but-got-eof":
-        _("Unexpected end of file. Expected attribute name instead."),
+        "Unexpected end of file. Expected attribute name instead.",
     "eof-in-attribute-name":
-        _("Unexpected end of file in attribute name."),
+        "Unexpected end of file in attribute name.",
     "invalid-character-in-attribute-name":
-        _("Invalid character in attribute name"),
+        "Invalid character in attribute name",
     "duplicate-attribute":
-        _("Dropped duplicate attribute on tag."),
+        "Dropped duplicate attribute on tag.",
     "expected-end-of-tag-name-but-got-eof":
-        _("Unexpected end of file. Expected = or end of tag."),
+        "Unexpected end of file. Expected = or end of tag.",
     "expected-attribute-value-but-got-eof":
-        _("Unexpected end of file. Expected attribute value."),
+        "Unexpected end of file. Expected attribute value.",
     "expected-attribute-value-but-got-right-bracket":
-        _("Expected attribute value. Got '>' instead."),
+        "Expected attribute value. Got '>' instead.",
     'equals-in-unquoted-attribute-value':
-        _("Unexpected = in unquoted attribute"),
+        "Unexpected = in unquoted attribute",
     'unexpected-character-in-unquoted-attribute-value':
-        _("Unexpected character in unquoted attribute"),
+        "Unexpected character in unquoted attribute",
     "invalid-character-after-attribute-name":
-        _("Unexpected character after attribute name."),
+        "Unexpected character after attribute name.",
     "unexpected-character-after-attribute-value":
-        _("Unexpected character after attribute value."),
+        "Unexpected character after attribute value.",
     "eof-in-attribute-value-double-quote":
-        _("Unexpected end of file in attribute value (\")."),
+        "Unexpected end of file in attribute value (\").",
     "eof-in-attribute-value-single-quote":
-        _("Unexpected end of file in attribute value (')."),
+        "Unexpected end of file in attribute value (').",
     "eof-in-attribute-value-no-quotes":
-        _("Unexpected end of file in attribute value."),
+        "Unexpected end of file in attribute value.",
     "unexpected-EOF-after-solidus-in-tag":
-        _("Unexpected end of file in tag. Expected >"),
+        "Unexpected end of file in tag. Expected >",
     "unexpected-character-after-solidus-in-tag":
-        _("Unexpected character after / in tag. Expected >"),
+        "Unexpected character after / in tag. Expected >",
     "expected-dashes-or-doctype":
-        _("Expected '--' or 'DOCTYPE'. Not found."),
+        "Expected '--' or 'DOCTYPE'. Not found.",
     "unexpected-bang-after-double-dash-in-comment":
-        _("Unexpected ! after -- in comment"),
+        "Unexpected ! after -- in comment",
     "unexpected-space-after-double-dash-in-comment":
-        _("Unexpected space after -- in comment"),
+        "Unexpected space after -- in comment",
     "incorrect-comment":
-        _("Incorrect comment."),
+        "Incorrect comment.",
     "eof-in-comment":
-        _("Unexpected end of file in comment."),
+        "Unexpected end of file in comment.",
     "eof-in-comment-end-dash":
-        _("Unexpected end of file in comment (-)"),
+        "Unexpected end of file in comment (-)",
     "unexpected-dash-after-double-dash-in-comment":
-        _("Unexpected '-' after '--' found in comment."),
+        "Unexpected '-' after '--' found in comment.",
     "eof-in-comment-double-dash":
-        _("Unexpected end of file in comment (--)."),
+        "Unexpected end of file in comment (--).",
     "eof-in-comment-end-space-state":
-        _("Unexpected end of file in comment."),
+        "Unexpected end of file in comment.",
     "eof-in-comment-end-bang-state":
-        _("Unexpected end of file in comment."),
+        "Unexpected end of file in comment.",
     "unexpected-char-in-comment":
-        _("Unexpected character in comment found."),
+        "Unexpected character in comment found.",
     "need-space-after-doctype":
-        _("No space after literal string 'DOCTYPE'."),
+        "No space after literal string 'DOCTYPE'.",
     "expected-doctype-name-but-got-right-bracket":
-        _("Unexpected > character. Expected DOCTYPE name."),
+        "Unexpected > character. Expected DOCTYPE name.",
     "expected-doctype-name-but-got-eof":
-        _("Unexpected end of file. Expected DOCTYPE name."),
+        "Unexpected end of file. Expected DOCTYPE name.",
     "eof-in-doctype-name":
-        _("Unexpected end of file in DOCTYPE name."),
+        "Unexpected end of file in DOCTYPE name.",
     "eof-in-doctype":
-        _("Unexpected end of file in DOCTYPE."),
+        "Unexpected end of file in DOCTYPE.",
     "expected-space-or-right-bracket-in-doctype":
-        _("Expected space or '>'. Got '%(data)s'"),
+        "Expected space or '>'. Got '%(data)s'",
     "unexpected-end-of-doctype":
-        _("Unexpected end of DOCTYPE."),
+        "Unexpected end of DOCTYPE.",
     "unexpected-char-in-doctype":
-        _("Unexpected character in DOCTYPE."),
+        "Unexpected character in DOCTYPE.",
     "eof-in-innerhtml":
-        _("XXX innerHTML EOF"),
+        "XXX innerHTML EOF",
     "unexpected-doctype":
-        _("Unexpected DOCTYPE. Ignored."),
+        "Unexpected DOCTYPE. Ignored.",
     "non-html-root":
-        _("html needs to be the first start tag."),
+        "html needs to be the first start tag.",
     "expected-doctype-but-got-eof":
-        _("Unexpected End of file. Expected DOCTYPE."),
+        "Unexpected End of file. Expected DOCTYPE.",
     "unknown-doctype":
-        _("Erroneous DOCTYPE."),
+        "Erroneous DOCTYPE.",
     "expected-doctype-but-got-chars":
-        _("Unexpected non-space characters. Expected DOCTYPE."),
+        "Unexpected non-space characters. Expected DOCTYPE.",
     "expected-doctype-but-got-start-tag":
-        _("Unexpected start tag (%(name)s). Expected DOCTYPE."),
+        "Unexpected start tag (%(name)s). Expected DOCTYPE.",
     "expected-doctype-but-got-end-tag":
-        _("Unexpected end tag (%(name)s). Expected DOCTYPE."),
+        "Unexpected end tag (%(name)s). Expected DOCTYPE.",
     "end-tag-after-implied-root":
-        _("Unexpected end tag (%(name)s) after the (implied) root element."),
+        "Unexpected end tag (%(name)s) after the (implied) root element.",
     "expected-named-closing-tag-but-got-eof":
-        _("Unexpected end of file. Expected end tag (%(name)s)."),
+        "Unexpected end of file. Expected end tag (%(name)s).",
     "two-heads-are-not-better-than-one":
-        _("Unexpected start tag head in existing head. Ignored."),
+        "Unexpected start tag head in existing head. Ignored.",
     "unexpected-end-tag":
-        _("Unexpected end tag (%(name)s). Ignored."),
+        "Unexpected end tag (%(name)s). Ignored.",
     "unexpected-start-tag-out-of-my-head":
-        _("Unexpected start tag (%(name)s) that can be in head. Moved."),
+        "Unexpected start tag (%(name)s) that can be in head. Moved.",
     "unexpected-start-tag":
-        _("Unexpected start tag (%(name)s)."),
+        "Unexpected start tag (%(name)s).",
     "missing-end-tag":
-        _("Missing end tag (%(name)s)."),
+        "Missing end tag (%(name)s).",
     "missing-end-tags":
-        _("Missing end tags (%(name)s)."),
+        "Missing end tags (%(name)s).",
     "unexpected-start-tag-implies-end-tag":
-        _("Unexpected start tag (%(startName)s) "
-          "implies end tag (%(endName)s)."),
+        "Unexpected start tag (%(startName)s) "
+        "implies end tag (%(endName)s).",
     "unexpected-start-tag-treated-as":
-        _("Unexpected start tag (%(originalName)s). Treated as %(newName)s."),
+        "Unexpected start tag (%(originalName)s). Treated as %(newName)s.",
     "deprecated-tag":
-        _("Unexpected start tag %(name)s. Don't use it!"),
+        "Unexpected start tag %(name)s. Don't use it!",
     "unexpected-start-tag-ignored":
-        _("Unexpected start tag %(name)s. Ignored."),
+        "Unexpected start tag %(name)s. Ignored.",
     "expected-one-end-tag-but-got-another":
-        _("Unexpected end tag (%(gotName)s). "
-          "Missing end tag (%(expectedName)s)."),
+        "Unexpected end tag (%(gotName)s). "
+        "Missing end tag (%(expectedName)s).",
     "end-tag-too-early":
-        _("End tag (%(name)s) seen too early. Expected other end tag."),
+        "End tag (%(name)s) seen too early. Expected other end tag.",
     "end-tag-too-early-named":
-        _("Unexpected end tag (%(gotName)s). Expected end tag (%(expectedName)s)."),
+        "Unexpected end tag (%(gotName)s). Expected end tag (%(expectedName)s).",
     "end-tag-too-early-ignored":
-        _("End tag (%(name)s) seen too early. Ignored."),
+        "End tag (%(name)s) seen too early. Ignored.",
     "adoption-agency-1.1":
-        _("End tag (%(name)s) violates step 1, "
-          "paragraph 1 of the adoption agency algorithm."),
+        "End tag (%(name)s) violates step 1, "
+        "paragraph 1 of the adoption agency algorithm.",
     "adoption-agency-1.2":
-        _("End tag (%(name)s) violates step 1, "
-          "paragraph 2 of the adoption agency algorithm."),
+        "End tag (%(name)s) violates step 1, "
+        "paragraph 2 of the adoption agency algorithm.",
     "adoption-agency-1.3":
-        _("End tag (%(name)s) violates step 1, "
-          "paragraph 3 of the adoption agency algorithm."),
+        "End tag (%(name)s) violates step 1, "
+        "paragraph 3 of the adoption agency algorithm.",
     "adoption-agency-4.4":
-        _("End tag (%(name)s) violates step 4, "
-          "paragraph 4 of the adoption agency algorithm."),
+        "End tag (%(name)s) violates step 4, "
+        "paragraph 4 of the adoption agency algorithm.",
     "unexpected-end-tag-treated-as":
-        _("Unexpected end tag (%(originalName)s). Treated as %(newName)s."),
+        "Unexpected end tag (%(originalName)s). Treated as %(newName)s.",
     "no-end-tag":
-        _("This element (%(name)s) has no end tag."),
+        "This element (%(name)s) has no end tag.",
     "unexpected-implied-end-tag-in-table":
-        _("Unexpected implied end tag (%(name)s) in the table phase."),
+        "Unexpected implied end tag (%(name)s) in the table phase.",
     "unexpected-implied-end-tag-in-table-body":
-        _("Unexpected implied end tag (%(name)s) in the table body phase."),
+        "Unexpected implied end tag (%(name)s) in the table body phase.",
     "unexpected-char-implies-table-voodoo":
-        _("Unexpected non-space characters in "
-          "table context caused voodoo mode."),
+        "Unexpected non-space characters in "
+        "table context caused voodoo mode.",
     "unexpected-hidden-input-in-table":
-        _("Unexpected input with type hidden in table context."),
+        "Unexpected input with type hidden in table context.",
     "unexpected-form-in-table":
-        _("Unexpected form in table context."),
+        "Unexpected form in table context.",
     "unexpected-start-tag-implies-table-voodoo":
-        _("Unexpected start tag (%(name)s) in "
-          "table context caused voodoo mode."),
+        "Unexpected start tag (%(name)s) in "
+        "table context caused voodoo mode.",
     "unexpected-end-tag-implies-table-voodoo":
-        _("Unexpected end tag (%(name)s) in "
-          "table context caused voodoo mode."),
+        "Unexpected end tag (%(name)s) in "
+        "table context caused voodoo mode.",
     "unexpected-cell-in-table-body":
-        _("Unexpected table cell start tag (%(name)s) "
-          "in the table body phase."),
+        "Unexpected table cell start tag (%(name)s) "
+        "in the table body phase.",
     "unexpected-cell-end-tag":
-        _("Got table cell end tag (%(name)s) "
-          "while required end tags are missing."),
+        "Got table cell end tag (%(name)s) "
+        "while required end tags are missing.",
     "unexpected-end-tag-in-table-body":
-        _("Unexpected end tag (%(name)s) in the table body phase. Ignored."),
+        "Unexpected end tag (%(name)s) in the table body phase. Ignored.",
     "unexpected-implied-end-tag-in-table-row":
-        _("Unexpected implied end tag (%(name)s) in the table row phase."),
+        "Unexpected implied end tag (%(name)s) in the table row phase.",
     "unexpected-end-tag-in-table-row":
-        _("Unexpected end tag (%(name)s) in the table row phase. Ignored."),
+        "Unexpected end tag (%(name)s) in the table row phase. Ignored.",
     "unexpected-select-in-select":
-        _("Unexpected select start tag in the select phase "
-          "treated as select end tag."),
+        "Unexpected select start tag in the select phase "
+        "treated as select end tag.",
     "unexpected-input-in-select":
-        _("Unexpected input start tag in the select phase."),
+        "Unexpected input start tag in the select phase.",
     "unexpected-start-tag-in-select":
-        _("Unexpected start tag token (%(name)s in the select phase. "
-          "Ignored."),
+        "Unexpected start tag token (%(name)s in the select phase. "
+        "Ignored.",
     "unexpected-end-tag-in-select":
-        _("Unexpected end tag (%(name)s) in the select phase. Ignored."),
+        "Unexpected end tag (%(name)s) in the select phase. Ignored.",
     "unexpected-table-element-start-tag-in-select-in-table":
-        _("Unexpected table element start tag (%(name)s) in the select in table phase."),
+        "Unexpected table element start tag (%(name)s) in the select in table phase.",
     "unexpected-table-element-end-tag-in-select-in-table":
-        _("Unexpected table element end tag (%(name)s) in the select in table phase."),
+        "Unexpected table element end tag (%(name)s) in the select in table phase.",
     "unexpected-char-after-body":
-        _("Unexpected non-space characters in the after body phase."),
+        "Unexpected non-space characters in the after body phase.",
     "unexpected-start-tag-after-body":
-        _("Unexpected start tag token (%(name)s)"
-          " in the after body phase."),
+        "Unexpected start tag token (%(name)s)"
+        " in the after body phase.",
     "unexpected-end-tag-after-body":
-        _("Unexpected end tag token (%(name)s)"
-          " in the after body phase."),
+        "Unexpected end tag token (%(name)s)"
+        " in the after body phase.",
     "unexpected-char-in-frameset":
-        _("Unexpected characters in the frameset phase. Characters ignored."),
+        "Unexpected characters in the frameset phase. Characters ignored.",
     "unexpected-start-tag-in-frameset":
-        _("Unexpected start tag token (%(name)s)"
-          " in the frameset phase. Ignored."),
+        "Unexpected start tag token (%(name)s)"
+        " in the frameset phase. Ignored.",
     "unexpected-frameset-in-frameset-innerhtml":
-        _("Unexpected end tag token (frameset) "
-          "in the frameset phase (innerHTML)."),
+        "Unexpected end tag token (frameset) "
+        "in the frameset phase (innerHTML).",
     "unexpected-end-tag-in-frameset":
-        _("Unexpected end tag token (%(name)s)"
-          " in the frameset phase. Ignored."),
+        "Unexpected end tag token (%(name)s)"
+        " in the frameset phase. Ignored.",
     "unexpected-char-after-frameset":
-        _("Unexpected non-space characters in the "
-          "after frameset phase. Ignored."),
+        "Unexpected non-space characters in the "
+        "after frameset phase. Ignored.",
     "unexpected-start-tag-after-frameset":
-        _("Unexpected start tag (%(name)s)"
-          " in the after frameset phase. Ignored."),
+        "Unexpected start tag (%(name)s)"
+        " in the after frameset phase. Ignored.",
     "unexpected-end-tag-after-frameset":
-        _("Unexpected end tag (%(name)s)"
-          " in the after frameset phase. Ignored."),
+        "Unexpected end tag (%(name)s)"
+        " in the after frameset phase. Ignored.",
     "unexpected-end-tag-after-body-innerhtml":
-        _("Unexpected end tag after body(innerHtml)"),
+        "Unexpected end tag after body(innerHtml)",
     "expected-eof-but-got-char":
-        _("Unexpected non-space characters. Expected end of file."),
+        "Unexpected non-space characters. Expected end of file.",
     "expected-eof-but-got-start-tag":
-        _("Unexpected start tag (%(name)s)"
-          ". Expected end of file."),
+        "Unexpected start tag (%(name)s)"
+        ". Expected end of file.",
     "expected-eof-but-got-end-tag":
-        _("Unexpected end tag (%(name)s)"
-          ". Expected end of file."),
+        "Unexpected end tag (%(name)s)"
+        ". Expected end of file.",
     "eof-in-table":
-        _("Unexpected end of file. Expected table content."),
+        "Unexpected end of file. Expected table content.",
     "eof-in-select":
-        _("Unexpected end of file. Expected select content."),
+        "Unexpected end of file. Expected select content.",
     "eof-in-frameset":
-        _("Unexpected end of file. Expected frameset content."),
+        "Unexpected end of file. Expected frameset content.",
     "eof-in-script-in-script":
-        _("Unexpected end of file. Expected script content."),
+        "Unexpected end of file. Expected script content.",
     "eof-in-foreign-lands":
-        _("Unexpected end of file. Expected foreign content"),
+        "Unexpected end of file. Expected foreign content",
     "non-void-element-with-trailing-solidus":
-        _("Trailing solidus not allowed on element %(name)s"),
+        "Trailing solidus not allowed on element %(name)s",
     "unexpected-html-element-in-foreign-content":
-        _("Element %(name)s not allowed in a non-html context"),
+        "Element %(name)s not allowed in a non-html context",
     "unexpected-end-tag-before-html":
-        _("Unexpected end tag (%(name)s) before html."),
+        "Unexpected end tag (%(name)s) before html.",
     "XXX-undefined-error":
-        _("Undefined error (this sucks and should be fixed)"),
+        "Undefined error (this sucks and should be fixed)",
 }
 
 namespaces = {
@@ -298,7 +296,7 @@ namespaces = {
     "xmlns": "http://www.w3.org/2000/xmlns/"
 }
 
-scopingElements = frozenset((
+scopingElements = frozenset([
     (namespaces["html"], "applet"),
     (namespaces["html"], "caption"),
     (namespaces["html"], "html"),
@@ -316,9 +314,9 @@ scopingElements = frozenset((
     (namespaces["svg"], "foreignObject"),
     (namespaces["svg"], "desc"),
     (namespaces["svg"], "title"),
-))
+])
 
-formattingElements = frozenset((
+formattingElements = frozenset([
     (namespaces["html"], "a"),
     (namespaces["html"], "b"),
     (namespaces["html"], "big"),
@@ -333,9 +331,9 @@ formattingElements = frozenset((
     (namespaces["html"], "strong"),
     (namespaces["html"], "tt"),
     (namespaces["html"], "u")
-))
+])
 
-specialElements = frozenset((
+specialElements = frozenset([
     (namespaces["html"], "address"),
     (namespaces["html"], "applet"),
     (namespaces["html"], "area"),
@@ -416,22 +414,22 @@ specialElements = frozenset((
     (namespaces["html"], "wbr"),
     (namespaces["html"], "xmp"),
     (namespaces["svg"], "foreignObject")
-))
+])
 
-htmlIntegrationPointElements = frozenset((
+htmlIntegrationPointElements = frozenset([
     (namespaces["mathml"], "annotaion-xml"),
     (namespaces["svg"], "foreignObject"),
     (namespaces["svg"], "desc"),
     (namespaces["svg"], "title")
-))
+])
 
-mathmlTextIntegrationPointElements = frozenset((
+mathmlTextIntegrationPointElements = frozenset([
     (namespaces["mathml"], "mi"),
     (namespaces["mathml"], "mo"),
     (namespaces["mathml"], "mn"),
     (namespaces["mathml"], "ms"),
     (namespaces["mathml"], "mtext")
-))
+])
 
 adjustForeignAttributes = {
     "xlink:actuate": ("xlink", "actuate", namespaces["xlink"]),
@@ -451,21 +449,21 @@ adjustForeignAttributes = {
 unadjustForeignAttributes = dict([((ns, local), qname) for qname, (prefix, local, ns) in
                                   adjustForeignAttributes.items()])
 
-spaceCharacters = frozenset((
+spaceCharacters = frozenset([
     "\t",
     "\n",
     "\u000C",
     " ",
     "\r"
-))
+])
 
-tableInsertModeElements = frozenset((
+tableInsertModeElements = frozenset([
     "table",
     "tbody",
     "tfoot",
     "thead",
     "tr"
-))
+])
 
 asciiLowercase = frozenset(string.ascii_lowercase)
 asciiUppercase = frozenset(string.ascii_uppercase)
@@ -486,7 +484,7 @@ headingElements = (
     "h6"
 )
 
-voidElements = frozenset((
+voidElements = frozenset([
     "base",
     "command",
     "event-source",
@@ -502,11 +500,11 @@ voidElements = frozenset((
     "input",
     "source",
     "track"
-))
+])
 
-cdataElements = frozenset(('title', 'textarea'))
+cdataElements = frozenset(['title', 'textarea'])
 
-rcdataElements = frozenset((
+rcdataElements = frozenset([
     'style',
     'script',
     'xmp',
@@ -514,27 +512,27 @@ rcdataElements = frozenset((
     'noembed',
     'noframes',
     'noscript'
-))
+])
 
 booleanAttributes = {
-    "": frozenset(("irrelevant",)),
-    "style": frozenset(("scoped",)),
-    "img": frozenset(("ismap",)),
-    "audio": frozenset(("autoplay", "controls")),
-    "video": frozenset(("autoplay", "controls")),
-    "script": frozenset(("defer", "async")),
-    "details": frozenset(("open",)),
-    "datagrid": frozenset(("multiple", "disabled")),
-    "command": frozenset(("hidden", "disabled", "checked", "default")),
-    "hr": frozenset(("noshade")),
-    "menu": frozenset(("autosubmit",)),
-    "fieldset": frozenset(("disabled", "readonly")),
-    "option": frozenset(("disabled", "readonly", "selected")),
-    "optgroup": frozenset(("disabled", "readonly")),
-    "button": frozenset(("disabled", "autofocus")),
-    "input": frozenset(("disabled", "readonly", "required", "autofocus", "checked", "ismap")),
-    "select": frozenset(("disabled", "readonly", "autofocus", "multiple")),
-    "output": frozenset(("disabled", "readonly")),
+    "": frozenset(["irrelevant"]),
+    "style": frozenset(["scoped"]),
+    "img": frozenset(["ismap"]),
+    "audio": frozenset(["autoplay", "controls"]),
+    "video": frozenset(["autoplay", "controls"]),
+    "script": frozenset(["defer", "async"]),
+    "details": frozenset(["open"]),
+    "datagrid": frozenset(["multiple", "disabled"]),
+    "command": frozenset(["hidden", "disabled", "checked", "default"]),
+    "hr": frozenset(["noshade"]),
+    "menu": frozenset(["autosubmit"]),
+    "fieldset": frozenset(["disabled", "readonly"]),
+    "option": frozenset(["disabled", "readonly", "selected"]),
+    "optgroup": frozenset(["disabled", "readonly"]),
+    "button": frozenset(["disabled", "autofocus"]),
+    "input": frozenset(["disabled", "readonly", "required", "autofocus", "checked", "ismap"]),
+    "select": frozenset(["disabled", "readonly", "autofocus", "multiple"]),
+    "output": frozenset(["disabled", "readonly"]),
 }
 
 # entitiesWindows1252 has to be _ordered_ and needs to have an index. It
@@ -574,7 +572,7 @@ entitiesWindows1252 = (
     376     # 0x9F  0x0178  LATIN CAPITAL LETTER Y WITH DIAERESIS
 )
 
-xmlEntities = frozenset(('lt;', 'gt;', 'amp;', 'apos;', 'quot;'))
+xmlEntities = frozenset(['lt;', 'gt;', 'amp;', 'apos;', 'quot;'])
 
 entities = {
     "AElig": "\xc6",
@@ -3088,8 +3086,8 @@ tokenTypes = {
     "ParseError": 7
 }
 
-tagTokenTypes = frozenset((tokenTypes["StartTag"], tokenTypes["EndTag"],
-                           tokenTypes["EmptyTag"]))
+tagTokenTypes = frozenset([tokenTypes["StartTag"], tokenTypes["EndTag"],
+                           tokenTypes["EmptyTag"]])
 
 
 prefixes = dict([(v, k) for k, v in namespaces.items()])

included_dependencies/html5lib/filters/lint.py

@@ -1,8 +1,5 @@
 from __future__ import absolute_import, division, unicode_literals
 
-from gettext import gettext
-_ = gettext
-
 from . import _base
 from ..constants import cdataElements, rcdataElements, voidElements
 
@@ -23,24 +20,24 @@ class Filter(_base.Filter):
             if type in ("StartTag", "EmptyTag"):
                 name = token["name"]
                 if contentModelFlag != "PCDATA":
-                    raise LintError(_("StartTag not in PCDATA content model flag: %(tag)s") % {"tag": name})
+                    raise LintError("StartTag not in PCDATA content model flag: %(tag)s" % {"tag": name})
                 if not isinstance(name, str):
-                    raise LintError(_("Tag name is not a string: %(tag)r") % {"tag": name})
+                    raise LintError("Tag name is not a string: %(tag)r" % {"tag": name})
                 if not name:
-                    raise LintError(_("Empty tag name"))
+                    raise LintError("Empty tag name")
                 if type == "StartTag" and name in voidElements:
-                    raise LintError(_("Void element reported as StartTag token: %(tag)s") % {"tag": name})
+                    raise LintError("Void element reported as StartTag token: %(tag)s" % {"tag": name})
                 elif type == "EmptyTag" and name not in voidElements:
-                    raise LintError(_("Non-void element reported as EmptyTag token: %(tag)s") % {"tag": token["name"]})
+                    raise LintError("Non-void element reported as EmptyTag token: %(tag)s" % {"tag": token["name"]})
                 if type == "StartTag":
                     open_elements.append(name)
                 for name, value in token["data"]:
                     if not isinstance(name, str):
-                        raise LintError(_("Attribute name is not a string: %(name)r") % {"name": name})
+                        raise LintError("Attribute name is not a string: %(name)r" % {"name": name})
                     if not name:
-                        raise LintError(_("Empty attribute name"))
+                        raise LintError("Empty attribute name")
                     if not isinstance(value, str):
-                        raise LintError(_("Attribute value is not a string: %(value)r") % {"value": value})
+                        raise LintError("Attribute value is not a string: %(value)r" % {"value": value})
                 if name in cdataElements:
                     contentModelFlag = "CDATA"
                 elif name in rcdataElements:
@@ -51,43 +48,43 @@ class Filter(_base.Filter):
             elif type == "EndTag":
                 name = token["name"]
                 if not isinstance(name, str):
-                    raise LintError(_("Tag name is not a string: %(tag)r") % {"tag": name})
+                    raise LintError("Tag name is not a string: %(tag)r" % {"tag": name})
                 if not name:
-                    raise LintError(_("Empty tag name"))
+                    raise LintError("Empty tag name")
                 if name in voidElements:
-                    raise LintError(_("Void element reported as EndTag token: %(tag)s") % {"tag": name})
+                    raise LintError("Void element reported as EndTag token: %(tag)s" % {"tag": name})
                 start_name = open_elements.pop()
                 if start_name != name:
-                    raise LintError(_("EndTag (%(end)s) does not match StartTag (%(start)s)") % {"end": name, "start": start_name})
+                    raise LintError("EndTag (%(end)s) does not match StartTag (%(start)s)" % {"end": name, "start": start_name})
                 contentModelFlag = "PCDATA"
 
             elif type == "Comment":
                 if contentModelFlag != "PCDATA":
-                    raise LintError(_("Comment not in PCDATA content model flag"))
+                    raise LintError("Comment not in PCDATA content model flag")
 
             elif type in ("Characters", "SpaceCharacters"):
                 data = token["data"]
                 if not isinstance(data, str):
-                    raise LintError(_("Attribute name is not a string: %(name)r") % {"name": data})
+                    raise LintError("Attribute name is not a string: %(name)r" % {"name": data})
                 if not data:
-                    raise LintError(_("%(type)s token with empty data") % {"type": type})
+                    raise LintError("%(type)s token with empty data" % {"type": type})
                 if type == "SpaceCharacters":
                     data = data.strip(spaceCharacters)
                     if data:
-                        raise LintError(_("Non-space character(s) found in SpaceCharacters token: %(token)r") % {"token": data})
+                        raise LintError("Non-space character(s) found in SpaceCharacters token: %(token)r" % {"token": data})
 
             elif type == "Doctype":
                 name = token["name"]
                 if contentModelFlag != "PCDATA":
-                    raise LintError(_("Doctype not in PCDATA content model flag: %(name)s") % {"name": name})
+                    raise LintError("Doctype not in PCDATA content model flag: %(name)s" % {"name": name})
                 if not isinstance(name, str):
-                    raise LintError(_("Tag name is not a string: %(tag)r") % {"tag": name})
+                    raise LintError("Tag name is not a string: %(tag)r" % {"tag": name})
                 # XXX: what to do with token["data"] ?
 
             elif type in ("ParseError", "SerializeError"):
                 pass
 
             else:
-                raise LintError(_("Unknown token type: %(type)s") % {"type": type})
+                raise LintError("Unknown token type: %(type)s" % {"type": type})
 
             yield token

included_dependencies/html5lib/html5parser.py

@@ -18,6 +18,7 @@ from .constants import cdataElements, rcdataElements
 from .constants import tokenTypes, ReparseException, namespaces
 from .constants import htmlIntegrationPointElements, mathmlTextIntegrationPointElements
 from .constants import adjustForeignAttributes as adjustForeignAttributesMap
+from .constants import E
 
 
 def parse(doc, treebuilder="etree", encoding=None,
@@ -129,6 +130,17 @@ class HTMLParser(object):
 
         self.framesetOK = True
 
+    @property
+    def documentEncoding(self):
+        """The name of the character encoding
+        that was used to decode the input stream,
+        or :obj:`None` if that is not determined yet.
+
+        """
+        if not hasattr(self, 'tokenizer'):
+            return None
+        return self.tokenizer.stream.charEncoding[0]
+
     def isHTMLIntegrationPoint(self, element):
         if (element.name == "annotation-xml" and
                 element.namespace == namespaces["mathml"]):
@@ -245,7 +257,7 @@ class HTMLParser(object):
         # XXX The idea is to make errorcode mandatory.
         self.errors.append((self.tokenizer.stream.position(), errorcode, datavars))
         if self.strict:
-            raise ParseError
+            raise ParseError(E[errorcode] % datavars)
 
     def normalizeToken(self, token):
         """ HTML5 specific normalizations to the token stream """
@@ -868,7 +880,7 @@ def getPhases(debug):
             self.startTagHandler = utils.MethodDispatcher([
                 ("html", self.startTagHtml),
                 (("base", "basefont", "bgsound", "command", "link", "meta",
-                  "noframes", "script", "style", "title"),
+                  "script", "style", "title"),
                  self.startTagProcessInHead),
                 ("body", self.startTagBody),
                 ("frameset", self.startTagFrameset),
@@ -1205,8 +1217,7 @@ def getPhases(debug):
             attributes["name"] = "isindex"
             self.processStartTag(impliedTagToken("input", "StartTag",
                                                  attributes=attributes,
-                                                 selfClosing=
-                                                 token["selfClosing"]))
+                                                 selfClosing=token["selfClosing"]))
             self.processEndTag(impliedTagToken("label"))
             self.processStartTag(impliedTagToken("hr", "StartTag"))
             self.processEndTag(impliedTagToken("form"))

included_dependencies/html5lib/inputstream.py

@@ -28,7 +28,18 @@ asciiLettersBytes = frozenset([item.encode("ascii") for item in asciiLetters])
 asciiUppercaseBytes = frozenset([item.encode("ascii") for item in asciiUppercase])
 spacesAngleBrackets = spaceCharactersBytes | frozenset([b">", b"<"])
 
-invalid_unicode_re = re.compile("[\u0001-\u0008\u000B\u000E-\u001F\u007F-\u009F\uD800-\uDFFF\uFDD0-\uFDEF\uFFFE\uFFFF\U0001FFFE\U0001FFFF\U0002FFFE\U0002FFFF\U0003FFFE\U0003FFFF\U0004FFFE\U0004FFFF\U0005FFFE\U0005FFFF\U0006FFFE\U0006FFFF\U0007FFFE\U0007FFFF\U0008FFFE\U0008FFFF\U0009FFFE\U0009FFFF\U000AFFFE\U000AFFFF\U000BFFFE\U000BFFFF\U000CFFFE\U000CFFFF\U000DFFFE\U000DFFFF\U000EFFFE\U000EFFFF\U000FFFFE\U000FFFFF\U0010FFFE\U0010FFFF]")
+
+invalid_unicode_no_surrogate = "[\u0001-\u0008\u000B\u000E-\u001F\u007F-\u009F\uFDD0-\uFDEF\uFFFE\uFFFF\U0001FFFE\U0001FFFF\U0002FFFE\U0002FFFF\U0003FFFE\U0003FFFF\U0004FFFE\U0004FFFF\U0005FFFE\U0005FFFF\U0006FFFE\U0006FFFF\U0007FFFE\U0007FFFF\U0008FFFE\U0008FFFF\U0009FFFE\U0009FFFF\U000AFFFE\U000AFFFF\U000BFFFE\U000BFFFF\U000CFFFE\U000CFFFF\U000DFFFE\U000DFFFF\U000EFFFE\U000EFFFF\U000FFFFE\U000FFFFF\U0010FFFE\U0010FFFF]"
+
+if utils.supports_lone_surrogates:
+    # Use one extra step of indirection and create surrogates with
+    # unichr. Not using this indirection would introduce an illegal
+    # unicode literal on platforms not supporting such lone
+    # surrogates.
+    invalid_unicode_re = re.compile(invalid_unicode_no_surrogate +
+                                    eval('"\\uD800-\\uDFFF"'))
+else:
+    invalid_unicode_re = re.compile(invalid_unicode_no_surrogate)
 
 non_bmp_invalid_codepoints = set([0x1FFFE, 0x1FFFF, 0x2FFFE, 0x2FFFF, 0x3FFFE,
                                   0x3FFFF, 0x4FFFE, 0x4FFFF, 0x5FFFE, 0x5FFFF,
@@ -164,13 +175,18 @@ class HTMLUnicodeInputStream(object):
 
         """
 
-        # Craziness
-        if len("\U0010FFFF") == 1:
+        if not utils.supports_lone_surrogates:
+            # Such platforms will have already checked for such
+            # surrogate errors, so no need to do this checking.
+            self.reportCharacterErrors = None
+            self.replaceCharactersRegexp = None
+        elif len("\U0010FFFF") == 1:
             self.reportCharacterErrors = self.characterErrorsUCS4
-            self.replaceCharactersRegexp = re.compile("[\uD800-\uDFFF]")
+            self.replaceCharactersRegexp = re.compile(eval('"[\\uD800-\\uDFFF]"'))
         else:
             self.reportCharacterErrors = self.characterErrorsUCS2
-            self.replaceCharactersRegexp = re.compile("([\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?<![\uD800-\uDBFF])[\uDC00-\uDFFF])")
+            self.replaceCharactersRegexp = re.compile(
+                eval('"([\\uD800-\\uDBFF](?![\\uDC00-\\uDFFF])|(?<![\\uD800-\\uDBFF])[\\uDC00-\\uDFFF])"'))
 
         # List of where new lines occur
         self.newLines = [0]
@@ -265,11 +281,12 @@ class HTMLUnicodeInputStream(object):
                 self._bufferedCharacter = data[-1]
                 data = data[:-1]
 
-        self.reportCharacterErrors(data)
+        if self.reportCharacterErrors:
+            self.reportCharacterErrors(data)
 
-        # Replace invalid characters
-        # Note U+0000 is dealt with in the tokenizer
-        data = self.replaceCharactersRegexp.sub("\ufffd", data)
+            # Replace invalid characters
+            # Note U+0000 is dealt with in the tokenizer
+            data = self.replaceCharactersRegexp.sub("\ufffd", data)
 
         data = data.replace("\r\n", "\n")
         data = data.replace("\r", "\n")

included_dependencies/html5lib/sanitizer.py

@@ -2,11 +2,26 @@ from __future__ import absolute_import, division, unicode_literals
 
 import re
 from xml.sax.saxutils import escape, unescape
+from six.moves import urllib_parse as urlparse
 
 from .tokenizer import HTMLTokenizer
 from .constants import tokenTypes
 
 
+content_type_rgx = re.compile(r'''
+                               ^
+                               # Match a content type <application>/<type>
+                               (?P<content_type>[-a-zA-Z0-9.]+/[-a-zA-Z0-9.]+)
+                               # Match any character set and encoding
+                               (?:(?:;charset=(?:[-a-zA-Z0-9]+)(?:;(?:base64))?)
+                                 |(?:;(?:base64))?(?:;charset=(?:[-a-zA-Z0-9]+))?)
+                               # Assume the rest is data
+                               ,.*
+                               $
+                               ''',
+                              re.VERBOSE)
+
+
 class HTMLSanitizerMixin(object):
     """ sanitization of XHTML+MathML+SVG and of inline style attributes."""
 
@@ -100,8 +115,8 @@ class HTMLSanitizerMixin(object):
                       'xml:base', 'xml:lang', 'xml:space', 'xmlns', 'xmlns:xlink', 'y',
                       'y1', 'y2', 'zoomAndPan']
 
-    attr_val_is_uri = ['href', 'src', 'cite', 'action', 'longdesc', 'poster',
-                       'xlink:href', 'xml:base']
+    attr_val_is_uri = ['href', 'src', 'cite', 'action', 'longdesc', 'poster', 'background', 'datasrc',
+                       'dynsrc', 'lowsrc', 'ping', 'poster', 'xlink:href', 'xml:base']
 
     svg_attr_val_allows_ref = ['clip-path', 'color-profile', 'cursor', 'fill',
                                'filter', 'marker', 'marker-start', 'marker-mid', 'marker-end',
@@ -138,7 +153,9 @@ class HTMLSanitizerMixin(object):
     acceptable_protocols = ['ed2k', 'ftp', 'http', 'https', 'irc',
                             'mailto', 'news', 'gopher', 'nntp', 'telnet', 'webcal',
                             'xmpp', 'callto', 'feed', 'urn', 'aim', 'rsync', 'tag',
-                            'ssh', 'sftp', 'rtsp', 'afs']
+                            'ssh', 'sftp', 'rtsp', 'afs', 'data']
+
+    acceptable_content_types = ['image/png', 'image/jpeg', 'image/gif', 'image/webp', 'image/bmp', 'text/plain']
 
     # subclasses may define their own versions of these constants
     allowed_elements = acceptable_elements + mathml_elements + svg_elements
@@ -147,6 +164,7 @@ class HTMLSanitizerMixin(object):
     allowed_css_keywords = acceptable_css_keywords
     allowed_svg_properties = acceptable_svg_properties
     allowed_protocols = acceptable_protocols
+    allowed_content_types = acceptable_content_types
 
     # Sanitize the +html+, escaping all elements not in ALLOWED_ELEMENTS, and
     # stripping out all # attributes not in ALLOWED_ATTRIBUTES. Style
@@ -189,10 +207,17 @@ class HTMLSanitizerMixin(object):
                                        unescape(attrs[attr])).lower()
                 # remove replacement characters from unescaped characters
                 val_unescaped = val_unescaped.replace("\ufffd", "")
-                if (re.match("^[a-z0-9][-+.a-z0-9]*:", val_unescaped) and
-                    (val_unescaped.split(':')[0] not in
-                     self.allowed_protocols)):
-                    del attrs[attr]
+                uri = urlparse.urlparse(val_unescaped)
+                if uri:
+                    if uri.scheme not in self.allowed_protocols:
+                        del attrs[attr]
+                    if uri.scheme == 'data':
+                        m = content_type_rgx.match(uri.path)
+                        if not m:
+                            del attrs[attr]
+                        elif m.group('content_type') not in self.allowed_content_types:
+                            del attrs[attr]
+
             for attr in self.svg_attr_val_allows_ref:
                 if attr in attrs:
                     attrs[attr] = re.sub(r'url\s*\(\s*[^#\s][^)]+?\)',
@@ -245,7 +270,7 @@ class HTMLSanitizerMixin(object):
             elif prop.split('-')[0].lower() in ['background', 'border', 'margin',
                                                 'padding']:
                 for keyword in value.split():
-                    if not keyword in self.acceptable_css_keywords and \
+                    if keyword not in self.acceptable_css_keywords and \
                             not re.match("^(#[0-9a-f]+|rgb\(\d+%?,\d*%?,?\d*%?\)?|\d{0,2}\.?\d{0,2}(cm|em|ex|in|mm|pc|pt|px|%|,|\))?)$", keyword):
                         break
                 else:

included_dependencies/html5lib/serializer/htmlserializer.py

@@ -1,9 +1,6 @@
 from __future__ import absolute_import, division, unicode_literals
 from six import text_type
 
-import gettext
-_ = gettext.gettext
-
 try:
     from functools import reduce
 except ImportError:
@@ -35,7 +32,7 @@ else:
                 v = utils.surrogatePairToCodepoint(v)
             else:
                 v = ord(v)
-            if not v in encode_entity_map or k.islower():
+            if v not in encode_entity_map or k.islower():
                 # prefer < over < and similarly for &, >, etc.
                 encode_entity_map[v] = k
 
@@ -208,7 +205,7 @@ class HTMLSerializer(object):
                 if token["systemId"]:
                     if token["systemId"].find('"') >= 0:
                         if token["systemId"].find("'") >= 0:
-                            self.serializeError(_("System identifer contains both single and double quote characters"))
+                            self.serializeError("System identifer contains both single and double quote characters")
                         quote_char = "'"
                     else:
                         quote_char = '"'
@@ -220,7 +217,7 @@ class HTMLSerializer(object):
             elif type in ("Characters", "SpaceCharacters"):
                 if type == "SpaceCharacters" or in_cdata:
                     if in_cdata and token["data"].find("</") >= 0:
-                        self.serializeError(_("Unexpected </ in CDATA"))
+                        self.serializeError("Unexpected </ in CDATA")
                     yield self.encode(token["data"])
                 else:
                     yield self.encode(escape(token["data"]))
@@ -231,7 +228,7 @@ class HTMLSerializer(object):
                 if name in rcdataElements and not self.escape_rcdata:
                     in_cdata = True
                 elif in_cdata:
-                    self.serializeError(_("Unexpected child element of a CDATA element"))
+                    self.serializeError("Unexpected child element of a CDATA element")
                 for (attr_namespace, attr_name), attr_value in token["data"].items():
                     # TODO: Add namespace support here
                     k = attr_name
@@ -279,20 +276,20 @@ class HTMLSerializer(object):
                 if name in rcdataElements:
                     in_cdata = False
                 elif in_cdata:
-                    self.serializeError(_("Unexpected child element of a CDATA element"))
+                    self.serializeError("Unexpected child element of a CDATA element")
                 yield self.encodeStrict("</%s>" % name)
 
             elif type == "Comment":
                 data = token["data"]
                 if data.find("--") >= 0:
-                    self.serializeError(_("Comment contains --"))
+                    self.serializeError("Comment contains --")
                 yield self.encodeStrict("<!--%s-->" % token["data"])
 
             elif type == "Entity":
                 name = token["name"]
                 key = name + ";"
-                if not key in entities:
-                    self.serializeError(_("Entity %s not recognized" % name))
+                if key not in entities:
+                    self.serializeError("Entity %s not recognized" % name)
                 if self.resolve_entities and key not in xmlEntities:
                     data = entities[key]
                 else:

included_dependencies/html5lib/treebuilders/dom.py

@@ -158,7 +158,7 @@ def getDomBuilder(DomImplementation):
             else:
                 # HACK: allow text nodes as children of the document node
                 if hasattr(self.dom, '_child_node_types'):
-                    if not Node.TEXT_NODE in self.dom._child_node_types:
+                    if Node.TEXT_NODE not in self.dom._child_node_types:
                         self.dom._child_node_types = list(self.dom._child_node_types)
                         self.dom._child_node_types.append(Node.TEXT_NODE)
                 self.dom.appendChild(self.dom.createTextNode(data))

included_dependencies/html5lib/treewalkers/__init__.py

@@ -10,8 +10,12 @@ returning an iterator generating tokens.
 
 from __future__ import absolute_import, division, unicode_literals
 
+__all__ = ["getTreeWalker", "pprint", "dom", "etree", "genshistream", "lxmletree",
+           "pulldom"]
+
 import sys
 
+from .. import constants
 from ..utils import default_etree
 
 treeWalkerCache = {}
@@ -55,3 +59,89 @@ def getTreeWalker(treeType, implementation=None, **kwargs):
             # XXX: NEVER cache here, caching is done in the etree submodule
             return etree.getETreeModule(implementation, **kwargs).TreeWalker
     return treeWalkerCache.get(treeType)
+
+
+def concatenateCharacterTokens(tokens):
+    pendingCharacters = []
+    for token in tokens:
+        type = token["type"]
+        if type in ("Characters", "SpaceCharacters"):
+            pendingCharacters.append(token["data"])
+        else:
+            if pendingCharacters:
+                yield {"type": "Characters", "data": "".join(pendingCharacters)}
+                pendingCharacters = []
+            yield token
+    if pendingCharacters:
+        yield {"type": "Characters", "data": "".join(pendingCharacters)}
+
+
+def pprint(walker):
+    """Pretty printer for tree walkers"""
+    output = []
+    indent = 0
+    for token in concatenateCharacterTokens(walker):
+        type = token["type"]
+        if type in ("StartTag", "EmptyTag"):
+            # tag name
+            if token["namespace"] and token["namespace"] != constants.namespaces["html"]:
+                if token["namespace"] in constants.prefixes:
+                    ns = constants.prefixes[token["namespace"]]
+                else:
+                    ns = token["namespace"]
+                name = "%s %s" % (ns, token["name"])
+            else:
+                name = token["name"]
+            output.append("%s<%s>" % (" " * indent, name))
+            indent += 2
+            # attributes (sorted for consistent ordering)
+            attrs = token["data"]
+            for (namespace, localname), value in sorted(attrs.items()):
+                if namespace:
+                    if namespace in constants.prefixes:
+                        ns = constants.prefixes[namespace]
+                    else:
+                        ns = namespace
+                    name = "%s %s" % (ns, localname)
+                else:
+                    name = localname
+                output.append("%s%s=\"%s\"" % (" " * indent, name, value))
+            # self-closing
+            if type == "EmptyTag":
+                indent -= 2
+
+        elif type == "EndTag":
+            indent -= 2
+
+        elif type == "Comment":
+            output.append("%s<!-- %s -->" % (" " * indent, token["data"]))
+
+        elif type == "Doctype":
+            if token["name"]:
+                if token["publicId"]:
+                    output.append("""%s<!DOCTYPE %s "%s" "%s">""" %
+                                  (" " * indent,
+                                   token["name"],
+                                   token["publicId"],
+                                   token["systemId"] if token["systemId"] else ""))
+                elif token["systemId"]:
+                    output.append("""%s<!DOCTYPE %s "" "%s">""" %
+                                  (" " * indent,
+                                   token["name"],
+                                   token["systemId"]))
+                else:
+                    output.append("%s<!DOCTYPE %s>" % (" " * indent,
+                                                       token["name"]))
+            else:
+                output.append("%s<!DOCTYPE >" % (" " * indent,))
+
+        elif type == "Characters":
+            output.append("%s\"%s\"" % (" " * indent, token["data"]))
+
+        elif type == "SpaceCharacters":
+            assert False, "concatenateCharacterTokens should have got rid of all Space tokens"
+
+        else:
+            raise ValueError("Unknown token type, %s" % type)
+
+    return "\n".join(output)

included_dependencies/html5lib/treewalkers/_base.py

@@ -1,8 +1,8 @@
 from __future__ import absolute_import, division, unicode_literals
 from six import text_type, string_types
 
-import gettext
-_ = gettext.gettext
+__all__ = ["DOCUMENT", "DOCTYPE", "TEXT", "ELEMENT", "COMMENT", "ENTITY", "UNKNOWN",
+           "TreeWalker", "NonRecursiveTreeWalker"]
 
 from xml.dom import Node
 
@@ -58,7 +58,7 @@ class TreeWalker(object):
                "namespace": to_text(namespace),
                "data": attrs}
         if hasChildren:
-            yield self.error(_("Void element has children"))
+            yield self.error("Void element has children")
 
     def startTag(self, namespace, name, attrs):
         assert namespace is None or isinstance(namespace, string_types), type(namespace)
@@ -122,7 +122,7 @@ class TreeWalker(object):
         return {"type": "Entity", "name": text_type(name)}
 
     def unknown(self, nodeType):
-        return self.error(_("Unknown node type: ") + nodeType)
+        return self.error("Unknown node type: " + nodeType)
 
 
 class NonRecursiveTreeWalker(TreeWalker):

included_dependencies/html5lib/treewalkers/dom.py

@@ -2,9 +2,6 @@ from __future__ import absolute_import, division, unicode_literals
 
 from xml.dom import Node
 
-import gettext
-_ = gettext.gettext
-
 from . import _base
 
 

included_dependencies/html5lib/treewalkers/etree.py

@@ -7,8 +7,6 @@ except ImportError:
         from ordereddict import OrderedDict
     except ImportError:
         OrderedDict = dict
-import gettext
-_ = gettext.gettext
 
 import re
 

included_dependencies/html5lib/treewalkers/lxmletree.py

@@ -4,9 +4,6 @@ from six import text_type
 from lxml import etree
 from ..treebuilders.etree import tag_regexp
 
-from gettext import gettext
-_ = gettext
-
 from . import _base
 
 from .. import ihatexml
@@ -130,7 +127,7 @@ class TreeWalker(_base.NonRecursiveTreeWalker):
     def getNodeDetails(self, node):
         if isinstance(node, tuple):  # Text node
             node, key = node
-            assert key in ("text", "tail"), _("Text nodes are text or tail, found %s") % key
+            assert key in ("text", "tail"), "Text nodes are text or tail, found %s" % key
             return _base.TEXT, ensure_str(getattr(node, key))
 
         elif isinstance(node, Root):
@@ -169,7 +166,7 @@ class TreeWalker(_base.NonRecursiveTreeWalker):
                     attrs, len(node) > 0 or node.text)
 
     def getFirstChild(self, node):
-        assert not isinstance(node, tuple), _("Text nodes have no children")
+        assert not isinstance(node, tuple), "Text nodes have no children"
 
         assert len(node) or node.text, "Node has no children"
         if node.text:
@@ -180,7 +177,7 @@ class TreeWalker(_base.NonRecursiveTreeWalker):
     def getNextSibling(self, node):
         if isinstance(node, tuple):  # Text node
             node, key = node
-            assert key in ("text", "tail"), _("Text nodes are text or tail, found %s") % key
+            assert key in ("text", "tail"), "Text nodes are text or tail, found %s" % key
             if key == "text":
                 # XXX: we cannot use a "bool(node) and node[0] or None" construct here
                 # because node[0] might evaluate to False if it has no child element
@@ -196,7 +193,7 @@ class TreeWalker(_base.NonRecursiveTreeWalker):
     def getParentNode(self, node):
         if isinstance(node, tuple):  # Text node
             node, key = node
-            assert key in ("text", "tail"), _("Text nodes are text or tail, found %s") % key
+            assert key in ("text", "tail"), "Text nodes are text or tail, found %s" % key
             if key == "text":
                 return node
             # else: fallback to "normal" processing

included_dependencies/html5lib/utils.py

@@ -2,6 +2,8 @@ from __future__ import absolute_import, division, unicode_literals
 
 from types import ModuleType
 
+from six import text_type
+
 try:
     import xml.etree.cElementTree as default_etree
 except ImportError:
@@ -9,7 +11,26 @@ except ImportError:
 
 
 __all__ = ["default_etree", "MethodDispatcher", "isSurrogatePair",
-           "surrogatePairToCodepoint", "moduleFactoryFactory"]
+           "surrogatePairToCodepoint", "moduleFactoryFactory",
+           "supports_lone_surrogates"]
+
+
+# Platforms not supporting lone surrogates (\uD800-\uDFFF) should be
+# caught by the below test. In general this would be any platform
+# using UTF-16 as its encoding of unicode strings, such as
+# Jython. This is because UTF-16 itself is based on the use of such
+# surrogates, and there is no mechanism to further escape such
+# escapes.
+try:
+    _x = eval('"\\uD800"')
+    if not isinstance(_x, text_type):
+        # We need this with u"" because of http://bugs.jython.org/issue2039
+        _x = eval('u"\\uD800"')
+        assert isinstance(_x, text_type)
+except:
+    supports_lone_surrogates = False
+else:
+    supports_lone_surrogates = True
 
 
 class MethodDispatcher(dict):

setup.py

@@ -10,13 +10,11 @@ https://github.com/pypa/sampleproject
 # Always prefer setuptools over distutils
 from setuptools import setup, find_packages
 # To use a consistent encoding
-from codecs import open
+import codecs
 from os import path
 
-here = path.abspath(path.dirname(__file__))
-
 # Get the long description from the relevant file
-with open(path.join(here, 'DESCRIPTION.rst'), encoding='utf-8') as f:
+with codecs.open('DESCRIPTION.rst', encoding='utf-8') as f:
     long_description = f.read()
 
 setup(
@@ -25,7 +23,7 @@ setup(
     # Versions should comply with PEP440.  For a discussion on single-sourcing
     # the version across setup.py and the project code, see
     # https://packaging.python.org/en/latest/single_source_version.html
-    version="2.2.5",
+    version="2.2.12",
 
     description='A tool for downloading fanfiction to eBook formats',
     long_description=long_description,

webservice/app.yaml

@@ -1,6 +1,6 @@
-# ffd-retief-hrd fanfictiondownloader fanficfare
+# ffd-retief-hrd fanficfare
 application: fanficfare
-version: 2-2-5
+version: 2-2-12
 runtime: python27
 api_version: 1
 threadsafe: true

webservice/ffstorage.py

@@ -41,6 +41,8 @@ class DownloadMeta(db.Model):
 	completed =  db.BooleanProperty(default=False)
 	date = db.DateTimeProperty(auto_now_add=True)
 	version = db.StringProperty()
+	ch_begin = db.StringProperty()
+	ch_end = db.StringProperty()
 	# data_chunks is implicit from DownloadData def.
 
 class DownloadData(db.Model):

webservice/index.html

@@ -46,9 +46,8 @@
           </p>
 	  <h3>Changes:</h3>
 	  <ul>
-            <li>Fix ficwad.com changing author URLs.</li>
-            <li>Fix for ponyfictionarchive.net stories with neither In-Progress or Complete flag.</li>
-            <li>Fix for grangerenchanted.com bad HTML summary causing crash.</li>
+            <li>New site: <a href="https://forum.questionablequesting.com/forums/creative-writing.19/">forum.questionablequesting.com</a> base_xenforoforum adapter</li>
+            <li>Add minor fix for portkey.org chapter parsing.</li>
 	  </ul>
 	  <p>
 	    Questions?  Check out our 
@@ -58,7 +57,7 @@
 	    If you have any problems with this application, please
 	    report them in
 	    the <a href="http://groups.google.com/group/fanfic-downloader">FanFicFare Google Group</a>.  The
-	    <a href="http://2-2-4.fanficfare.appspot.com">previous version
+	    <a href="http://2-2-11.fanficfare.appspot.com">previous version
 	    </a> is also available for you to use if necessary.
 	  </p>
 	  <div id='error'>

webservice/main.py

@@ -62,7 +62,7 @@ class UserConfigServer(webapp2.RequestHandler):
     
     def getUserConfig(self,user,url,fileformat):
 
-        configuration = Configuration(adapters.getConfigSectionFor(url),fileformat)
+        configuration = Configuration(adapters.getConfigSectionsFor(url),fileformat)
         
         logging.debug('reading defaults.ini config file')
         configuration.read('fanficfare/defaults.ini')
@@ -366,6 +366,16 @@ class FanfictionDownloader(UserConfigServer):
             self.redirect('/')
             return
 
+        # Allow chapter range with URL.
+        # test1.com?sid=5[4-6]
+        mc = re.match(r"^(?P<url>.*?)(?:\[(?P<begin>\d+)?(?P<comma>[,-])?(?P<end>\d+)?\])?$",url)
+        #print("url:(%s) begin:(%s) end:(%s)"%(mc.group('url'),mc.group('begin'),mc.group('end')))
+        url = mc.group('url')
+        ch_begin = mc.group('begin')
+        ch_end = mc.group('end')
+        if ch_begin and not mc.group('comma'):
+            ch_end = ch_begin
+        
         logging.info("Queuing Download: %s" % url)
         login = self.request.get('login')
         password = self.request.get('password')
@@ -385,6 +395,7 @@ class FanfictionDownloader(UserConfigServer):
                 return
 
             adapter = adapters.getAdapter(configuration,url)
+            adapter.setChaptersRange(ch_begin,ch_end)
             logging.info('Created an adaper: %s' % adapter)
 
             if login or password:
@@ -408,6 +419,8 @@ class FanfictionDownloader(UserConfigServer):
             download.title = story.getMetadata('title')
             download.author = story.getMetadata('author')
             download.url = story.getMetadata('storyUrl')
+            download.ch_begin = ch_begin
+            download.ch_end = ch_end
             download.put()
 
             taskqueue.add(url='/fdowntask',
@@ -490,6 +503,7 @@ class FanfictionDownloaderTask(UserConfigServer):
         try:
             configuration = self.getUserConfig(user,url,format)
             adapter = adapters.getAdapter(configuration,url)
+            adapter.setChaptersRange(download.ch_begin,download.ch_end)
 
             logging.info('Created an adapter: %s' % adapter)