Bump versions

Added adapter for quotev.com - thanks cryzed!

MANIFEST.in

@@ -0,0 +1 @@
+include DESCRIPTION.rst

calibre-plugin/__init__.py

@@ -42,7 +42,7 @@ class FanFicFareBase(InterfaceActionBase):
     description         = _('UI plugin to download FanFiction stories from various sites.')
     supported_platforms = ['windows', 'osx', 'linux']
     author              = 'Jim Miller'
-    version             = (2, 2, 10)
+    version             = (2, 2, 13)
     minimum_calibre_version = (1, 48, 0)
 
     #: This field defines the GUI plugin class that contains all the code

calibre-plugin/basicinihighlighter.py

@@ -1,5 +1,5 @@
-#!/usr/bin/env python
-# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+# -*- coding: utf-8 -*-
+
 from __future__ import (unicode_literals, division,
                         print_function)
 

calibre-plugin/common_utils.py

@@ -1,5 +1,5 @@
-#!/usr/bin/env python
-# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+# -*- coding: utf-8 -*-
+
 from __future__ import (unicode_literals, division, absolute_import,
                         print_function)
 

calibre-plugin/config.py

@@ -1,5 +1,5 @@
-#!/usr/bin/env python
-# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+# -*- coding: utf-8 -*-
+
 from __future__ import (unicode_literals, division, absolute_import,
                         print_function)
 
@@ -89,7 +89,7 @@ from calibre_plugins.fanficfare_plugin.dialogs \
             EditTextDialog, IniTextDialog, RejectUrlEntry)
     
 from calibre_plugins.fanficfare_plugin.fanficfare.adapters \
-    import getConfigSections
+    import getSiteSections
 
 from calibre_plugins.fanficfare_plugin.common_utils \
     import ( KeyboardConfigDialog, PrefsViewerDialog )
@@ -272,6 +272,7 @@ class ConfigWidget(QWidget):
             prefs['addtolists'] = self.readinglist_tab.addtolists.isChecked()
             prefs['addtoreadlists'] = self.readinglist_tab.addtoreadlists.isChecked()
             prefs['addtolistsonread'] = self.readinglist_tab.addtolistsonread.isChecked()
+            prefs['autounnew'] = self.readinglist_tab.autounnew.isChecked()
 
         # personal.ini
         ini = self.personalini_tab.personalini
@@ -356,6 +357,7 @@ class ConfigWidget(QWidget):
         prefs['imapmarkread'] = self.imap_tab.imapmarkread.isChecked()
         prefs['imapsessionpass'] = self.imap_tab.imapsessionpass.isChecked()
         prefs['auto_reject_from_email'] = self.imap_tab.auto_reject_from_email.isChecked()
+        prefs['download_from_email_immediately'] = self.imap_tab.download_from_email_immediately.isChecked()
         
         prefs.save_to_db()
 
@@ -781,6 +783,11 @@ class ReadingListTab(QWidget):
         self.addtolistsonread.setChecked(prefs['addtolistsonread'])
         self.l.addWidget(self.addtolistsonread)
             
+        self.autounnew = QCheckBox(_('Automatically run Remove "New" Chapter Marks when marking books "Read".'),self)
+        self.autounnew.setToolTip(_('Menu option to remove from "To Read" lists will also remove "(new)" chapter marks created by personal.ini <i>mark_new_chapters</i> setting.'))
+        self.autounnew.setChecked(prefs['autounnew'])
+        self.l.addWidget(self.autounnew)
+            
         self.l.insertStretch(-1)
         
 class CalibreCoverTab(QWidget):
@@ -928,7 +935,7 @@ class CalibreCoverTab(QWidget):
         
         self.gc_dropdowns = {}
 
-        sitelist = getConfigSections()
+        sitelist = getSiteSections()
         sitelist.sort()
         sitelist.insert(0,_("Default"))
         for site in sitelist:
@@ -1408,6 +1415,12 @@ class ImapTab(QWidget):
         self.l.addWidget(self.auto_reject_from_email,row,0,1,-1)
         row+=1
         
+        self.download_from_email_immediately = QCheckBox(_('Download from Email Immediately'),self)
+        self.download_from_email_immediately.setToolTip(_('If checked, FanFicFare will start downloading story URLs from emails immediately.<br>Otherwise the usual Download from URLs dialog will appear.'))
+        self.download_from_email_immediately.setChecked(prefs['download_from_email_immediately'])
+        self.l.addWidget(self.download_from_email_immediately,row,0,1,-1)
+        row+=1
+        
         label = QLabel(_("<b>It's safest if you create a separate email account that you use only "
                          "for your story update notices.  FanFicFare and calibre cannot guarantee that "
                          "malicious code cannot get your email password once you've entered it. "

calibre-plugin/dialogs.py

@@ -1,5 +1,5 @@
-#!/usr/bin/env python
-# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+# -*- coding: utf-8 -*-
+
 from __future__ import (unicode_literals, division,
                         print_function)
 
@@ -117,7 +117,7 @@ save_collisions={
     SAVE_CALIBREONLY:CALIBREONLY,
     SAVE_CALIBREONLYSAVECOL:CALIBREONLYSAVECOL,
     }
-    
+
 anthology_collision_order=[UPDATE,
                            UPDATEALWAYS,
                            OVERWRITEALWAYS]
@@ -127,7 +127,7 @@ gpstyle='QGroupBox {border:0; padding-top:10px; padding-bottom:0px; margin-botto
 class RejectUrlEntry:
 
     matchpat=re.compile(r"^(?P<url>[^,]+?)(,(?P<fullnote>(((?P<title>.+?) by (?P<auth>.+?)( - (?P<note>.+))?)|.*)))?$")
-    
+
     def __init__(self,url_or_line,note=None,title=None,auth=None,
                  addreasontext=None,fromline=False,book_id=None):
 
@@ -151,7 +151,7 @@ class RejectUrlEntry:
                 self.note=note
                 self.title=title
                 self.auth=auth
-        
+
         if not self.note:
             if addreasontext:
                 self.note = addreasontext
@@ -160,14 +160,14 @@ class RejectUrlEntry:
         else:
             if addreasontext:
                 self.note = self.note + ' - ' + addreasontext
-                
+
         self.url = getNormalStoryURL(self.url)
         self.valid = self.url != None
-                
+
     def to_line(self):
         # always 'url,'
         return self.url+","+self.fullnote()
-        
+
     def fullnote(self):
         retval = ""
         if self.title and self.auth:
@@ -175,10 +175,10 @@ class RejectUrlEntry:
             retval = retval + "%s by %s"%(self.title,self.auth)
             if self.note:
                 retval = retval + " - "
-                
+
         if self.note:
             retval = retval + self.note
-            
+
         return retval
 
 class NotGoingToDownload(Exception):
@@ -222,7 +222,7 @@ class DroppableQTextEdit(QTextEdit):
             self.append("\n".join(urllist))
             return None
         return QTextEdit.dropEvent(self,event)
-        
+
     def canInsertFromMimeData(self, source):
         if source.hasUrls():
             return True
@@ -234,7 +234,7 @@ class DroppableQTextEdit(QTextEdit):
             self.append(source.text())
         else:
             return QTextEdit.insertFromMimeData(self, source)
-                            
+
 class AddNewDialog(SizePersistedDialog):
 
     go_signal = pyqtSignal(object, object, object, object)
@@ -242,7 +242,7 @@ class AddNewDialog(SizePersistedDialog):
     def __init__(self, gui, prefs, icon):
         SizePersistedDialog.__init__(self, gui, 'fff:add new dialog')
         self.prefs = prefs
-        
+
         self.setMinimumWidth(300)
         self.l = QVBoxLayout()
         self.setLayout(self.l)
@@ -258,7 +258,8 @@ class AddNewDialog(SizePersistedDialog):
         self.l.addWidget(self.url)
 
         self.merge = self.newmerge = False
-        
+        self.extraoptions = {}
+
         # elements to hide when doing merge.
         self.mergehide = []
         # elements to show again when doing *update* merge
@@ -281,11 +282,11 @@ class AddNewDialog(SizePersistedDialog):
 
         self.gbf.setVisible(False)
         self.groupbox.toggled.connect(self.gbf.setVisible)
-        
+
         horz = QHBoxLayout()
         label = QLabel(_('Output &Format:'))
         self.mergehide.append(label)
-        
+
         self.fileform = QComboBox(self)
         self.fileform.addItem('epub')
         self.fileform.addItem('mobi')
@@ -293,7 +294,7 @@ class AddNewDialog(SizePersistedDialog):
         self.fileform.addItem('txt')
         self.fileform.setToolTip(_('Choose output format to create.  May set default from plugin configuration.'))
         self.fileform.activated.connect(self.set_collisions)
-        
+
         horz.addWidget(label)
         label.setBuddy(self.fileform)
         horz.addWidget(self.fileform)
@@ -331,7 +332,7 @@ class AddNewDialog(SizePersistedDialog):
         self.updateepubcover.setChecked(self.prefs['updateepubcover'])
         horz.addWidget(self.updateepubcover)
         self.mergehide.append(self.updateepubcover)
-        
+
         self.gbl.addLayout(horz)
 
         self.button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
@@ -339,7 +340,7 @@ class AddNewDialog(SizePersistedDialog):
         self.button_box.rejected.connect(self.reject)
         self.l.addWidget(self.button_box)
 
-    # invoke the 
+    # invoke the
     def ok_clicked(self):
         self.dialog_closing(None) # save persistent size.
         self.hide()
@@ -375,7 +376,7 @@ class AddNewDialog(SizePersistedDialog):
         self.extrapayload = extrapayload
 
         self.groupbox.setVisible(not(self.merge and self.newmerge))
-        
+
         if self.merge:
             self.toplabel.setText(_('Story URLs for anthology, one per line:'))
             self.url.setToolTip(_('URLs for stories to include in the anthology, one per line.\nWill take URLs from clipboard, but only valid URLs.'))
@@ -397,7 +398,7 @@ class AddNewDialog(SizePersistedDialog):
         # Need to re-able after hiding/showing
         self.setAcceptDrops(True)
         self.url.setFocus()
-            
+
         if self.prefs['adddialogstaysontop']:
             QDialog.setWindowFlags ( self, Qt.Dialog | Qt.WindowStaysOnTopHint )
         else:
@@ -414,18 +415,18 @@ class AddNewDialog(SizePersistedDialog):
         i = self.collision.findText(save_collisions[self.prefs['collision']])
         if i > -1:
             self.collision.setCurrentIndex(i)
-            
+
         self.updatemeta.setChecked(self.prefs['updatemeta'])
-            
+
         if not self.merge:
             self.updateepubcover.setChecked(self.prefs['updateepubcover'])
-                
+
         self.url.setText(url_list_text)
         if url_list_text:
             self.button_box.button(QDialogButtonBox.Ok).setFocus()
         # restore saved size.
         self.resize_dialog()
-        
+
         if show: # so anthology update can be modal still.
             self.show()
         #self.resize(self.sizeHint())
@@ -446,11 +447,11 @@ class AddNewDialog(SizePersistedDialog):
 
         for o in order:
             self.collision.addItem(o)
-            
+
         i = self.collision.findText(prev)
         if i > -1:
             self.collision.setCurrentIndex(i)
-        
+
     def get_fff_options(self):
         retval =  {
             'fileform': unicode(self.fileform.currentText()),
@@ -459,14 +460,14 @@ class AddNewDialog(SizePersistedDialog):
             'updateepubcover': self.updateepubcover.isChecked(),
             'smarten_punctuation':self.prefs['smarten_punctuation']
                 }
-        
+
         if self.merge:
             retval['fileform']=='epub'
             retval['updateepubcover']=True
             if self.newmerge:
                 retval['updatemeta']=True
                 retval['collision']=ADDNEW
-            
+
         return dict(retval.items() + self.extraoptions.items() )
 
     def get_urlstext(self):
@@ -476,21 +477,21 @@ class AddNewDialog(SizePersistedDialog):
 class FakeLineEdit():
     def __init__(self):
         pass
-    
+
     def text(self):
         pass
-    
+
 class CollectURLDialog(SizePersistedDialog):
     '''
     Collect single url for get urls.
     '''
-    def __init__(self, gui, title, url_text, anthology=False, indiv=True): 
+    def __init__(self, gui, title, url_text, anthology=False, indiv=True):
         SizePersistedDialog.__init__(self, gui, 'fff:get story urls')
         self.status=False
         self.anthology=False
 
         self.setMinimumWidth(300)
-        
+
         self.l = QVBoxLayout()
         self.setLayout(self.l)
 
@@ -560,16 +561,16 @@ class UserPassDialog(QDialog):
         else:
             self.setWindowTitle(_('User/Password'))
             self.l.addWidget(QLabel(_("%s requires you to login to download this story.")%site),0,0,1,2)
-        
+
             self.l.addWidget(QLabel(_("User:")),1,0)
             self.user = QLineEdit(self)
             self.l.addWidget(self.user,1,1)
-   
+
         self.l.addWidget(QLabel(_("Password:")),2,0)
         self.passwd = QLineEdit(self)
         self.passwd.setEchoMode(QLineEdit.Password)
         self.l.addWidget(self.passwd,2,1)
-   
+
         self.ok_button = QPushButton(_('OK'), self)
         self.ok_button.clicked.connect(self.ok)
         self.l.addWidget(self.ok_button,3,0)
@@ -615,6 +616,9 @@ class LoopProgressDialog(QProgressDialog):
         from calibre_plugins.fanficfare_plugin.prefs import prefs
         self.show_est_time = prefs['show_est_time']
 
+        self.setLabelText('%s %d / %d' % (self.status_prefix, self.i, len(self.book_list)))
+        self.setValue(self.i)
+
         ## self.do_loop does QTimer.singleShot on self.do_loop also.
         ## A weird way to do a loop, but that was the example I had.
         QTimer.singleShot(0, self.do_loop)
@@ -641,7 +645,7 @@ class LoopProgressDialog(QProgressDialog):
             ## collision spec passed into getadapter by partial from fff_plugin
             ## no retval only if it exists, but collision is SKIP
             self.foreach_function(book)
-            
+
         except NotGoingToDownload as d:
             book['good']=False
             book['comment']=unicode(d)
@@ -652,10 +656,10 @@ class LoopProgressDialog(QProgressDialog):
             book['comment']=unicode(e)
             logger.error("Exception: %s:%s"%(book,unicode(e)))
             traceback.print_exc()
-            
+
         self.updateStatus()
         self.i += 1
-            
+
         if self.i >= len(self.book_list) or self.wasCanceled():
             return self.do_when_finished()
         else:
@@ -746,7 +750,7 @@ class UpdateExistingDialog(SizePersistedDialog):
         self.prefs = prefs
         self.setWindowTitle(header)
         self.setWindowIcon(icon)
-        
+
         layout = QVBoxLayout(self)
         self.setLayout(layout)
         title_layout = ImageTitleLayout(self, 'images/icon.png',
@@ -789,7 +793,7 @@ class UpdateExistingDialog(SizePersistedDialog):
 
         gbf.setVisible(False)
         groupbox.toggled.connect(gbf.setVisible)
-        
+
         label = QLabel(_('Output &Format:'))
         gbl.addWidget(label)
         self.fileform = QComboBox(self)
@@ -802,7 +806,7 @@ class UpdateExistingDialog(SizePersistedDialog):
         self.fileform.activated.connect(self.set_collisions)
         label.setBuddy(self.fileform)
         gbl.addWidget(self.fileform)
-        
+
         label = QLabel(_('Update Mode:'))
         gbl.addWidget(label)
         self.collision = QComboBox(self)
@@ -819,21 +823,21 @@ class UpdateExistingDialog(SizePersistedDialog):
         self.updatemeta.setToolTip(_("Update metadata for existing stories in Calibre from web site?\n(Columns set to 'New Only' in the column tabs will only be set for new books.)"))
         self.updatemeta.setChecked(self.prefs['updatemeta'])
         gbl.addWidget(self.updatemeta)
-                
+
         self.updateepubcover = QCheckBox(_('Update EPUB Cover?'),self)
         self.updateepubcover.setToolTip(_('Update book cover image from site or defaults (if found) <i>inside</i> the EPUB when EPUB is updated.'))
         self.updateepubcover.setChecked(self.prefs['updateepubcover'])
         gbl.addWidget(self.updateepubcover)
 
 
-        
+
         button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
         button_box.accepted.connect(self.accept)
         button_box.rejected.connect(self.reject)
         options_layout.addWidget(button_box)
-        
+
         layout.addLayout(options_layout)
-        
+
         # Cause our dialog size to be restored from prefs or created on first usage
         self.resize_dialog()
         self.books_table.populate_table(books)
@@ -849,14 +853,14 @@ class UpdateExistingDialog(SizePersistedDialog):
             order.remove(UPDATEALWAYS)
         if self.prefs['savemetacol'] == '':
             order.remove(CALIBREONLYSAVECOL)
-            
+
         for o in order:
             self.collision.addItem(o)
-            
+
         i = self.collision.findText(prev)
         if i > -1:
             self.collision.setCurrentIndex(i)
-        
+
     def remove_from_list(self):
         self.books_table.remove_selected_rows()
 
@@ -923,16 +927,16 @@ class StoryListTableWidget(QTableWidget):
         status_cell = IconWidgetItem(None,icon,val)
         status_cell.setData(Qt.UserRole, val)
         self.setItem(row, 0, status_cell)
-        
+
         title_cell = ReadOnlyTableWidgetItem(book['title'])
         title_cell.setData(Qt.UserRole, row)
         self.setItem(row, 1, title_cell)
-        
+
         self.setItem(row, 2, AuthorTableWidgetItem(", ".join(book['author']), ", ".join(book['author_sort'])))
-        
+
         url_cell = ReadOnlyTableWidgetItem(book['url'])
         self.setItem(row, 3, url_cell)
-        
+
         comment_cell = ReadOnlyTableWidgetItem(book['comment'])
         self.setItem(row, 4, comment_cell)
 
@@ -1010,9 +1014,9 @@ class RejectListTableWidget(QTableWidget):
         self.setItem(row, 0, url_cell)
         self.setItem(row, 1, ReadOnlyTableWidgetItem(rej.title))
         self.setItem(row, 2, ReadOnlyTableWidgetItem(rej.auth))
-        
+
         note_cell = EditWithComplete(self,sort_func=lambda x:1)
-        
+
         items = [rej.note]+self.rejectreasons
         note_cell.update_items_cache(items)
         note_cell.show_initial_value(rej.note)
@@ -1020,7 +1024,7 @@ class RejectListTableWidget(QTableWidget):
         note_cell.setToolTip(_('Select or Edit Reject Note.'))
         self.setCellWidget(row, 3, note_cell)
         note_cell.setCursorPosition(0)
-        
+
     def remove_selected_rows(self):
         self.setFocus()
         rows = self.selectionModel().selectedRows()
@@ -1052,10 +1056,10 @@ class RejectListDialog(SizePersistedDialog):
                  show_all_reasons=True,
                  save_size_name='fff:reject list dialog'):
         SizePersistedDialog.__init__(self, gui, save_size_name)
-      
+
         self.setWindowTitle(header)
         self.setWindowIcon(get_icon(icon))
-      
+
         layout = QVBoxLayout(self)
         self.setLayout(layout)
         title_layout = ImageTitleLayout(self, icon, header,
@@ -1071,7 +1075,7 @@ class RejectListDialog(SizePersistedDialog):
         rejects_layout.addLayout(button_layout)
         spacerItem = QtGui.QSpacerItem(20, 40, QtGui.QSizePolicy.Minimum, QtGui.QSizePolicy.Expanding)
         button_layout.addItem(spacerItem)
-        
+
         self.remove_button = QtGui.QToolButton(self)
         self.remove_button.setToolTip(_('Remove selected URLs from the list'))
         self.remove_button.setIcon(get_icon('list_remove.png'))
@@ -1083,13 +1087,13 @@ class RejectListDialog(SizePersistedDialog):
 
         if show_all_reasons:
             self.reason_edit = EditWithComplete(self,sort_func=lambda x:1)
-            
+
             items = ['']+rejectreasons
             self.reason_edit.update_items_cache(items)
             self.reason_edit.show_initial_value('')
             self.reason_edit.set_separator(None)
             self.reason_edit.setToolTip(_("This will be added to whatever note you've set for each URL above."))
-            
+
             horz = QHBoxLayout()
             label = QLabel(_("Add this reason to all URLs added:"))
             label.setToolTip(_("This will be added to whatever note you've set for each URL above."))
@@ -1098,7 +1102,7 @@ class RejectListDialog(SizePersistedDialog):
             self.reason_edit.setCursorPosition(0)
             horz.insertStretch(-1)
             layout.addLayout(horz)
-                    
+
         options_layout = QHBoxLayout()
 
         if show_delete:
@@ -1106,14 +1110,14 @@ class RejectListDialog(SizePersistedDialog):
             self.deletebooks.setToolTip(_("Delete the selected books after adding them to the Rejected URLs list."))
             self.deletebooks.setChecked(True)
             options_layout.addWidget(self.deletebooks)
-        
+
         button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
         button_box.accepted.connect(self.accept)
         button_box.rejected.connect(self.reject)
         options_layout.addWidget(button_box)
-      
+
         layout.addLayout(options_layout)
-      
+
         # Cause our dialog size to be restored from prefs or created on first usage
         self.resize_dialog()
         self.rejects_table.populate_table(reject_list)
@@ -1146,7 +1150,7 @@ class RejectListDialog(SizePersistedDialog):
         except:
             # doesn't have self.reason_edit when editing existing list.
             return None
-    
+
     def get_deletebooks(self):
         return self.deletebooks.isChecked()
 
@@ -1168,7 +1172,7 @@ class EditTextDialog(SizePersistedDialog):
         if icon:
             self.setWindowIcon(icon)
         self.l.addWidget(self.label)
-        
+
         self.textedit = QTextEdit(self)
         self.textedit.setLineWrapMode(QTextEdit.NoWrap)
         self.textedit.setReadOnly(read_only)
@@ -1181,13 +1185,13 @@ class EditTextDialog(SizePersistedDialog):
 
         if rejectreasons or reasonslabel:
             self.reason_edit = EditWithComplete(self,sort_func=lambda x:1)
-            
+
             items = ['']+rejectreasons
             self.reason_edit.update_items_cache(items)
             self.reason_edit.show_initial_value('')
             self.reason_edit.set_separator(None)
             self.reason_edit.setToolTip(reasonslabel)
-            
+
             if reasonslabel:
                 horz = QHBoxLayout()
                 label = QLabel(reasonslabel)
@@ -1198,12 +1202,12 @@ class EditTextDialog(SizePersistedDialog):
             else:
                 self.l.addWidget(self.reason_edit)
             self.reason_edit.setCursorPosition(0)
-            
+
         button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
         button_box.accepted.connect(self.accept)
         button_box.rejected.connect(self.reject)
         self.l.addWidget(button_box)
-        
+
         # Cause our dialog size to be restored from prefs or created on first usage
         self.resize_dialog()
 
@@ -1212,7 +1216,7 @@ class EditTextDialog(SizePersistedDialog):
 
     def get_reason_text(self):
         return unicode(self.reason_edit.currentText()).strip()
-    
+
 class IniTextDialog(SizePersistedDialog):
 
     def __init__(self, parent, text,
@@ -1222,9 +1226,9 @@ class IniTextDialog(SizePersistedDialog):
                  save_size_name='fff:ini text dialog',
                  ):
         SizePersistedDialog.__init__(self, parent, save_size_name)
-        
+
         self.keys=dict()
-        
+
         self.l = QVBoxLayout()
         self.setLayout(self.l)
         self.label = QLabel(label)
@@ -1233,7 +1237,7 @@ class IniTextDialog(SizePersistedDialog):
         if icon:
             self.setWindowIcon(icon)
         self.l.addWidget(self.label)
-        
+
         self.textedit = QTextEdit(self)
 
         highlighter = IniHighlighter(self.textedit,
@@ -1242,7 +1246,7 @@ class IniTextDialog(SizePersistedDialog):
                                      entries=get_valid_entries(),
                                      entry_keywords=get_valid_entry_keywords(),
                                      )
-        
+
         self.textedit.setLineWrapMode(QTextEdit.NoWrap)
         try:
             self.textedit.setFont(QFont("Courier",
@@ -1260,41 +1264,41 @@ class IniTextDialog(SizePersistedDialog):
         if use_find:
 
             findtooltip=_('Search for string in edit box.')
-            
+
             horz = QHBoxLayout()
             label = QLabel(_('Find:'))
 
             label.setToolTip(findtooltip)
-                
+
             # Button to search the document for something
             self.findButton = QtGui.QPushButton(_('Find'),self)
             self.findButton.clicked.connect(self.find)
             self.findButton.setToolTip(findtooltip)
-        
+
             # The field into which to type the query
             self.findField = QLineEdit(self)
             self.findField.setToolTip(findtooltip)
             self.findField.returnPressed.connect(self.findButton.setFocus)
-            
+
             # Case Sensitivity option
             self.caseSens = QtGui.QCheckBox(_('Case sensitive'),self)
             self.caseSens.setToolTip(_("Search for case sensitive string; don't treat Harry, HARRY and harry all the same."))
-            
+
             horz.addWidget(label)
             horz.addWidget(self.findField)
             horz.addWidget(self.findButton)
             horz.addWidget(self.caseSens)
-            
+
             self.l.addLayout(horz)
 
             self.addCtrlKeyPress(QtCore.Qt.Key_F,self.findFocus)
             self.addCtrlKeyPress(QtCore.Qt.Key_G,self.find)
-        
+
         button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
         button_box.accepted.connect(self.accept)
         button_box.rejected.connect(self.reject)
         self.l.addWidget(button_box)
-        
+
         # Cause our dialog size to be restored from prefs or created on first usage
         self.resize_dialog()
 
@@ -1310,9 +1314,9 @@ class IniTextDialog(SizePersistedDialog):
                         _('Go back to fix errors?'),
                         errors)
             retry = d.exec_() == d.Accepted
-            
+
             # print("retry:%s"%retry)
-        
+
         if retry:
             lineno=d.get_lineno()
             if lineno:
@@ -1334,19 +1338,19 @@ class IniTextDialog(SizePersistedDialog):
             return func()
         else:
             return SizePersistedDialog.keyPressEvent(self, event)
-    
+
     def get_plain_text(self):
         return unicode(self.textedit.toPlainText())
 
     def findFocus(self):
         # print("findFocus called")
         self.findField.setFocus()
-        self.findField.selectAll()        
-        
+        self.findField.selectAll()
+
     def find(self):
 
         #print("find self.lastStart:%s"%self.lastStart)
-        
+
         # Grab the parent's text
         text = self.textedit.toPlainText()
 
@@ -1361,7 +1365,7 @@ class IniTextDialog(SizePersistedDialog):
         # last starting position
         self.lastStart = text.find(query,self.lastStart + 1)
         # If the find() method didn't return -1 (not found)
-        
+
         if self.lastStart >= 0:
             end = self.lastStart + len(query)
             self.moveCursor(self.lastStart,end)
@@ -1369,7 +1373,7 @@ class IniTextDialog(SizePersistedDialog):
             # Make the next search start from the begining again
             self.lastStart = 0
             self.textedit.moveCursor(self.textedit.textCursor().Start)
-                    
+
     def moveCursor(self,start,end):
 
         # We retrieve the QTextCursor object from the parent's QTextEdit
@@ -1386,7 +1390,7 @@ class IniTextDialog(SizePersistedDialog):
         self.textedit.setTextCursor(cursor)
 
     def select_line(self,lineno):
-        
+
         # We retrieve the QTextCursor object from the parent's QTextEdit
         cursor = self.textedit.textCursor()
 
@@ -1412,8 +1416,8 @@ class ViewLog(SizePersistedDialog):
 
     def get_lineno(self):
         return self.lineno
-        
-    def __init__(self, parent, title, errors, 
+
+    def __init__(self, parent, title, errors,
                  save_size_name='fff:view log dialog',):
         SizePersistedDialog.__init__(self, parent,save_size_name)
         self.l = l = QVBoxLayout()
@@ -1434,9 +1438,9 @@ class ViewLog(SizePersistedDialog):
             label.setToolTip(_('Click to go to line %s')%lineno)
             label.mouseReleaseEvent = partial(self.label_clicked, lineno=lineno)
             self.l.addWidget(label)
-        
+
         # html='<p>'+'</p><p>'.join([ '(lineno: %s) %s'%e for e in errors ])+'</p>'
-        
+
         # self.tb = QTextBrowser(self)
         # self.tb.setFont(QFont("Courier",
         #                       parent.font().pointSize()+1))
@@ -1454,13 +1458,13 @@ class ViewLog(SizePersistedDialog):
         saveanyway = QPushButton(_('Save Anyway'), self)
         saveanyway.clicked.connect(self.reject)
         horz.addWidget(saveanyway)
-        
+
         l.addLayout(horz)
         self.setModal(False)
         self.setWindowTitle(title)
         self.setWindowIcon(QIcon(I('debug.png')))
         #self.show()
-        
+
         # Cause our dialog size to be restored from prefs or created on first usage
         self.resize_dialog()
 
@@ -1481,7 +1485,7 @@ class EmailPassDialog(QDialog):
 
         self.setWindowTitle(_('Password'))
         self.l.addWidget(QLabel(_("Enter Email Password for %s:")%user),0,0,1,2)
-   
+
         # self.l.addWidget(QLabel(_("Password:")),1,0)
         self.passwd = QLineEdit(self)
         self.passwd.setEchoMode(QLineEdit.Password)

calibre-plugin/fff_plugin.py

@@ -1,5 +1,5 @@
-#!/usr/bin/env python
-# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+# -*- coding: utf-8 -*-
+
 from __future__ import (unicode_literals, division, absolute_import,
                         print_function)
 
@@ -66,7 +66,8 @@ from calibre_plugins.fanficfare_plugin.fanficfare import (
     adapters, exceptions)
 
 from calibre_plugins.fanficfare_plugin.fanficfare.epubutils import (
-    get_dcsource, get_dcsource_chaptercount, get_story_url_from_html)
+    get_dcsource, get_dcsource_chaptercount, get_story_url_from_html,
+    reset_orig_chapters_epub)
 
 from calibre_plugins.fanficfare_plugin.fanficfare.geturls import (
     get_urls_from_page, get_urls_from_html,get_urls_from_text,
@@ -335,6 +336,12 @@ class FanFicFarePlugin(InterfaceAction):
                                                                       image='minusminus.png',
                                                                       triggered=partial(self.update_lists,add=False))
 
+            self.menu.addSeparator()
+            self.get_list_action = self.create_menu_item_ex(self.menu, _('Remove "New" Chapter Marks from Selected books'),
+                                                            unique_name='Remove "(new)" chapter marks created by personal.ini <i>mark_new_chapters</i> setting.',
+                                                            image='edit-undo.png',
+                                                            triggered=self.unnew_books)
+
             self.menu.addSeparator()
             self.get_list_action = self.create_menu_item_ex(self.menu, _('Get Story URLs from Selected Books'),
                                                             unique_name='Get URLs from Selected Books',
@@ -415,6 +422,8 @@ class FanFicFarePlugin(InterfaceAction):
                 return
 
             self.update_reading_lists(self.gui.library_view.get_selected_ids(),add)
+            if not add and prefs['autounnew']:
+                self.unnew_books()
 
     def get_urls_from_imap_menu(self):
 
@@ -445,27 +454,39 @@ class FanFicFarePlugin(InterfaceAction):
                                       imap_pass,
                                       prefs['imapfolder'],
                                       prefs['imapmarkread'],)
+
+        ## reject will now be redundant with reject check inside
+        ## prep_downloads because of change-able story URLs.
+        ## Keeping here to because it's the far more common case.
         reject_list=set()
         if prefs['auto_reject_from_email']:
             # need to normalize for reject list.
             reject_list = set([x for x in url_list if rejecturllist.check(adapters.getNormalStoryURLSite(x)[0])])
         url_list = url_list - reject_list
 
-        self.gui.status_bar.show_message(_('Finished Fetching Story URLs from Email.'),3000)
+        self.gui.status_bar.show_message(_('No Valid Story URLs Found in Unread Emails.'),3000)
         self.restore_cursor()
-        
-        if url_list:
-            self.add_dialog("\n".join(url_list),merge=False)
-        else:
-            
-            msg = _('No Valid Story URLs Found in Unread Emails.')
-            if reject_list:
-                msg = msg + '<p>'+(_('(%d Story URLs Skipped, on Rejected URL List)')%len(reject_list))+'</p>'
-            info_dialog(self.gui, _('Get Story URLs from Email'),
-                        msg,
-                        show=True,
-                        show_copy_button=False)
 
+        if prefs['download_from_email_immediately']:
+            ## do imap fetch w/o GUI elements
+            if url_list:
+                self.prep_downloads(self.add_new_dialog.get_fff_options(),
+                                    "\n".join(url_list))
+            else:
+                self.gui.status_bar.show_message(_('Finished Fetching Story URLs from Email.'),3000)
+                
+        else:
+            if url_list:
+                self.add_dialog("\n".join(url_list),merge=False)
+            else:            
+                msg = _('No Valid Story URLs Found in Unread Emails.')
+                if reject_list:
+                    msg = msg + '<p>'+(_('(%d Story URLs Skipped, on Rejected URL List)')%len(reject_list))+'</p>'
+                info_dialog(self.gui, _('Get Story URLs from Email'),
+                            msg,
+                            show=True,
+                            show_copy_button=False)
+    
     def get_urls_from_page_menu(self,anthology=False):
 
         urltxt = ""
@@ -555,6 +576,70 @@ class FanFicFarePlugin(InterfaceAction):
                         show=True,
                         show_copy_button=False)
 
+    def unnew_books(self):
+        '''Get list of URLs from existing books.'''
+        if not self.is_library_view():
+            self.gui.status_bar.show_message(_('Can only UnNew books in library'),
+                                             3000)
+            return
+        
+        if not self.gui.current_view().selectionModel().selectedRows() :
+            self.gui.status_bar.show_message(_('No Selected Books to Get URLs From'),
+                                             3000)
+            return
+
+        book_list = map( partial(self.make_book_id_only),
+                         self.gui.library_view.get_selected_ids() )
+
+        tdir = PersistentTemporaryDirectory(prefix='fanficfare_')
+        LoopProgressDialog(self.gui,
+                           book_list,
+                           partial(self.get_unnew_books_loop, db=self.gui.current_db, tdir=tdir),
+                           partial(self.get_unnew_books_finish, tdir=tdir),
+                           init_label=_("UnNewing books..."),
+                           win_title=_("UnNew Books"),
+                           status_prefix=_("Books UnNewed"))
+
+    def get_unnew_books_loop(self,book,db=None,tdir=None):
+        
+        if book['calibre_id'] and db.has_format(book['calibre_id'],'EPUB',index_is_id=True):
+            tmp = PersistentTemporaryFile(prefix='%s-'%book['calibre_id'],
+                                          suffix='.epub',
+                                          dir=tdir)
+            db.copy_format_to(book['calibre_id'],'EPUB',tmp,index_is_id=True)
+            
+            unnewtmp = PersistentTemporaryFile(prefix='unnew-%s-'%book['calibre_id'],
+                                               suffix='.epub',
+                                               dir=tdir)
+            book['changed']=reset_orig_chapters_epub(tmp,unnewtmp)
+            if book['changed']:
+                db.add_format_with_hooks(book['calibre_id'],
+                                         'EPUB',
+                                         unnewtmp,
+                                         index_is_id=True)
+                if prefs['deleteotherforms']:
+                    fmts = db.formats(book['calibre_id'], index_is_id=True).split(',')
+                    for fmt in fmts:
+                        if fmt.lower() != formmapping['epub'].lower():
+                            logger.debug("deleteotherforms remove f:"+fmt)
+                            db.remove_format(book['calibre_id'], fmt, index_is_id=True)#, notify=False
+                elif prefs['autoconvert']:
+                    ## 'Convert Book'.auto_convert_auto_add doesn't convert if
+                    ## the format is already there.
+                    fmt = calibre_prefs['output_format']
+                    # delete if there, but not if the format we just made.
+                    if fmt.lower() != 'epub' and db.has_format(book['calibre_id'],fmt,index_is_id=True):
+                        logger.debug("autoconvert remove f:"+fmt)
+                        db.remove_format(book['calibre_id'], fmt, index_is_id=True)#, notify=False
+                
+    def get_unnew_books_finish(self, book_list, tdir=None):
+        remove_dir(tdir)
+        if prefs['autoconvert']:
+            changed_ids = [ x['calibre_id'] for x in book_list if x['changed'] ]
+            if changed_ids:
+                self.gui.status_bar.show_message(_('Starting auto conversion of %d books.')%(len(changed_ids)), 3000)
+                self.gui.iactions['Convert Books'].auto_convert_auto_add(changed_ids)
+
     def reject_list_urls(self):
         if self.is_library_view():
             book_list = map( partial(self.make_book_id_only),
@@ -850,6 +935,43 @@ class FanFicFarePlugin(InterfaceAction):
         # LoopProgressDialog calls start_download_job at the end which goes
         # into the BG, or shows list if no 'good' books.
 
+    def reject_url(self,merge,book):
+        url = book['url']
+        if not merge and rejecturllist.check(url): # skip reject list when merging.
+            rejnote = rejecturllist.get_full_note(url)
+            if prefs['reject_always'] or question_dialog(self.gui, _('Reject URL?'),'''
+                      <h3>%s</h3>
+                      <p>%s</p>
+                      <p>"<b>%s</b>"</p>
+                      <p>%s</p>
+                      <p>%s</p>'''%(
+                    _('Reject URL?'),
+                    _('<b>%s</b> is on your Reject URL list:')%url,
+                    rejnote,
+                    _("Click '<b>Yes</b>' to Reject."),
+                    _("Click '<b>No</b>' to download anyway.")),
+                               show_copy_button=False):
+                book['comment'] = _("Story on Reject URLs list (%s).")%rejnote
+                book['good']=False
+                book['icon']='rotate-right.png'
+                book['status'] = _('Rejected')
+                return True
+            else:
+                if question_dialog(self.gui, _('Remove Reject URL?'),'''
+                          <h3>%s</h3>
+                          <p>%s</p>
+                          <p>"<b>%s</b>"</p>
+                          <p>%s</p>
+                          <p>%s</p>'''%(
+                        _("Remove URL from Reject List?"),
+                        _('<b>%s</b> is on your Reject URL list:')%url,
+                        rejnote,
+                        _("Click '<b>Yes</b>' to remove it from the list,"),
+                        _("Click '<b>No</b>' to leave it on the list.")),
+                                   show_copy_button=False):
+                    rejecturllist.remove(url)
+        return False
+            
     def prep_download_loop(self,book,
                            options={'fileform':'epub',
                                     'collision':ADDNEW,
@@ -866,40 +988,11 @@ class FanFicFarePlugin(InterfaceAction):
         logger.debug("url:%s"%url)
         mi = None
 
-        if not merge: # skip reject list when merging.
-            if rejecturllist.check(url):
-                rejnote = rejecturllist.get_full_note(url)
-                if prefs['reject_always'] or question_dialog(self.gui, _('Reject URL?'),'''
-                          <h3>%s</h3>
-                          <p>%s</p>
-                          <p>"<b>%s</b>"</p>
-                          <p>%s</p>
-                          <p>%s</p>'''%(
-                        _('Reject URL?'),
-                        _('<b>%s</b> is on your Reject URL list:')%url,
-                        rejnote,
-                        _("Click '<b>Yes</b>' to Reject."),
-                        _("Click '<b>No</b>' to download anyway.")),
-                                   show_copy_button=False):
-                    book['comment'] = _("Story on Reject URLs list (%s).")%rejnote
-                    book['good']=False
-                    book['icon']='rotate-right.png'
-                    book['status'] = _('Rejected')
-                    return
-                else:
-                    if question_dialog(self.gui, _('Remove Reject URL?'),'''
-                              <h3>%s</h3>
-                              <p>%s</p>
-                              <p>"<b>%s</b>"</p>
-                              <p>%s</p>
-                              <p>%s</p>'''%(
-                            _("Remove URL from Reject List?"),
-                            _('<b>%s</b> is on your Reject URL list:')%url,
-                            rejnote,
-                            _("Click '<b>Yes</b>' to remove it from the list,"),
-                            _("Click '<b>No</b>' to leave it on the list.")),
-                                       show_copy_button=False):
-                        rejecturllist.remove(url)
+        ## Check reject list.  Redundant with below for when story URL
+        ## changes, but also kept here to avoid network hit in most
+        ## common case where given url is story url.
+        if self.reject_url(merge,book):
+            return
 
         # The current database shown in the GUI
         # db is an instance of the class LibraryDatabase2 from database.py
@@ -921,6 +1014,8 @@ class FanFicFarePlugin(InterfaceAction):
             return
 
         adapter = get_fff_adapter(url,fileform)
+        ## chapter range for title_chapter_range_pattern
+        adapter.setChaptersRange(book['begin'],book['end'])
         ## save and share cookiejar and pagecache between all
         ## downloads.
         if 'pagecache' not in options:
@@ -991,7 +1086,16 @@ class FanFicFarePlugin(InterfaceAction):
     
             # let other exceptions percolate up.
             story = adapter.getStoryMetadataOnly(get_cover=False)
+            book['title'] = story.getMetadata('title')
+            book['author'] = [story.getMetadata('author')]
+            book['url'] = story.getMetadata('storyUrl')
     
+            ## Check reject list.  Redundant with below for when story
+            ## URL changes, but also kept here to avoid network hit in
+            ## most common case where given url is story url.
+            if self.reject_url(merge,book):
+                return
+
             series = story.getMetadata('series')
             if not merge and series and prefs['checkforseriesurlid']:
                 # try to find *series anthology* by *seriesUrl* identifier url or uri first.
@@ -1011,6 +1115,7 @@ class FanFicFarePlugin(InterfaceAction):
                     book['comment'] = _("Story in Series Anthology(%s).")%series
                     book['title'] = story.getMetadata('title')
                     book['author'] = [story.getMetadata('author')]
+                    book['url'] = story.getMetadata('storyUrl')
                     book['good']=False
                     book['icon']='rotate-right.png'
                     book['status'] = _('Skipped')
@@ -1031,6 +1136,7 @@ class FanFicFarePlugin(InterfaceAction):
         book['title'] = story.getMetadata("title", removeallentities=True)
         book['author_sort'] = book['author'] = story.getList("author", removeallentities=True)
         book['publisher'] = story.getMetadata("site")
+        book['url'] = story.getMetadata("storyUrl")
         book['tags'] = story.getSubjectTags(removeallentities=True)
         if story.getMetadata("description"):
             book['comments'] = sanitize_comments_html(story.getMetadata("description"))
@@ -2051,6 +2157,7 @@ class FanFicFarePlugin(InterfaceAction):
         book['site'] = ''
         book['added'] = False
         book['pubdate'] = None
+        book['publisher'] = None
         return book
 
     def convert_urls_to_books(self, urls):

calibre-plugin/fff_util.py

@@ -1,5 +1,5 @@
-#!/usr/bin/env python
-# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+# -*- coding: utf-8 -*-
+
 from __future__ import (unicode_literals, division, absolute_import,
                         print_function)
 
@@ -23,12 +23,12 @@ def get_fff_personalini():
 def get_fff_config(url,fileform="epub",personalini=None):
     if not personalini:
         personalini = get_fff_personalini()
-    site='unknown'
+    sections=['unknown']
     try:
-        site = adapters.getConfigSectionFor(url)
+        sections = adapters.getConfigSectionsFor(url)
     except Exception as e:
-        logger.debug("Failed trying to get ini config for url(%s): %s, using section [%s] instead"%(url,e,site))
-    configuration = Configuration(site,fileform)
+        logger.debug("Failed trying to get ini config for url(%s): %s, using section %s instead"%(url,e,sections))
+    configuration = Configuration(sections,fileform)
     configuration.readfp(StringIO(get_resources("plugin-defaults.ini")))
     configuration.readfp(StringIO(personalini))
 

calibre-plugin/inihighlighter.py

@@ -1,5 +1,5 @@
-#!/usr/bin/env python
-# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+# -*- coding: utf-8 -*-
+
 from __future__ import (unicode_literals, division,
                         print_function)
 

calibre-plugin/jobs.py

@@ -1,5 +1,5 @@
-#!/usr/bin/env python
-# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+# -*- coding: utf-8 -*-
+
 from __future__ import (unicode_literals, division, absolute_import,
                         print_function)
 
@@ -192,7 +192,9 @@ def do_download_for_worker(book,options,notification=lambda x,y:x):
                  adapter.oldimgs,
                  adapter.oldcover,
                  adapter.calibrebookmark,
-                 adapter.logfile) = get_update_data(book['epub_for_update'])[0:7]
+                 adapter.logfile,
+                 adapter.oldchaptersmap,
+                 adapter.oldchaptersdata) = get_update_data(book['epub_for_update'])[0:9]
     
                 # dup handling from fff_plugin needed for anthology updates.
                 if options['collision'] == UPDATE:

calibre-plugin/plugin-defaults.ini

@@ -261,6 +261,11 @@ keep_summary_html:true
 ## kept.  Some sites(AO3) use <a title="Translation">Foreign language</a>
 #keep_title_attr: false
 
+## If a chapter range was given, use this pattern for the book title.
+## replace_metadata and include/exclude will be applied *after* this.
+## Set to empty value to disable.
+title_chapter_range_pattern:${title} (Ch ${first}-${last})
+
 ## Don't like the numbers at the start of chapter titles on some
 ## sites?  You can use strip_chapter_numbers to strip them off.  Just
 ## want to make them all look the same?  Strip them off, then add them
@@ -293,12 +298,31 @@ chapter_title_strip_pattern:^[0-9]+[\.: -]+
 ## etc
 #chapter_title_strip_pattern:^([0-9]+[\.: -]+)?(Chapter *[0-9]+[\.:, -]*)?
 
-## Uses a python template substitution.  The ${index} is the 'chapter'
-## number and ${title} is the chapter title, after applying
-## chapter_title_strip_pattern.  Those are the only variables available.
-## "The Beginning" => "1. The Beginning" 
+## If true, when updating an epub that already has old chapters, new
+## chapters will be marked in the TOC and chapter header by using
+## chapter_title_new_pattern and chapter_title_addnew_pattern to set the chapter.
+mark_new_chapters:false
+
+## chapter title patterns use python template substitution.  The
+## ${index} is the 'chapter' number and ${title} is the chapter title,
+## after applying chapter_title_strip_pattern.  Those are the only
+## variables available.
+
+## The basic pattern used when not using add_chapter_numbers or
+## mark_new_chapters
+chapter_title_def_pattern:${title}
+
+## Pattern used with add_chapter_numbers, but not mark_new_chapters
 chapter_title_add_pattern:${index}. ${title}
 
+## Pattern used with mark_new_chapters, but not add_chapter_numbers
+## (new) is just text and can be changed.
+chapter_title_new_pattern:(new) ${title}
+
+## Pattern used with add_chapter_numbers and mark_new_chapters
+## (new) is just text and can be changed.
+chapter_title_addnew_pattern:${index}. (new) ${title}
+
 ## Uses a python template substitution.  The ${title} is the default
 ## title of a new anthology, <series name> in the case of a series, or
 ## the first book title otherwise.  This is only applied to new
@@ -330,11 +354,77 @@ sort_ships:false
 ## User-agent
 user_agent:FFF/2.X
 
-## Virtually all eFiction Base adapters allow downloading the whole story in
-## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
-## metadata and chapters can be loaded in one step
+## Added for [base_xenforoforum], but can be used with other sites,
+## too.  Limit the 'description' to the first X *characters*
+## collected.  Character count includes HTML tags, so it can be
+## non-intuitive.
+#description_limit:1000
+
+[base_efiction]
+## At the time of writing, eFiction Base adapters allow downloading
+## the whole story in bulk using the 'Print' feature. If 'bulk_load'
+## is set to 'true', both metadata and chapters can be loaded in one
+## step
 bulk_load:true
 
+[base_xenforoforum]
+## Currently only forums.spacebattles.com and forums.sufficientvelocity.com
+
+cover_exclusion_regexp:/clear.png
+
+## I saw lots of chapters name simply '1.1' etc during testing.
+strip_chapter_numbers:false
+
+## Copy title to tagsfromtitle for parsing tags.
+add_to_extra_valid_entries:,tagsfromtitle,forumtags
+
+## '.NOREPL' tells the system to *not* apply title's
+## in/exclude/replace_metadata -- Only works on include_in_ lines.
+include_in_tagsfromtitle:title.NOREPL
+
+tagsfromtitle_label:Tags from Title
+forumtags_label:Tags from Forum
+
+## might want to do this, maybe not.  Will often include category, but
+## also often include non-category stuff.
+# include_in_category:tagsfromtitle 
+
+add_to_include_metadata_pre:
+# only keep tagsfromtitle with ( or [ in.
+ tagsfromtitle=~[\[\(]
+ 
+add_to_replace_metadata:
+# remove anything outside () or []
+ tagsfromtitle=>^.*?([\(\[]([^\]\)]+)[\)\]]).*?$=>\1
+# remove () []
+ tagsfromtitle=>[\(\)\[\]]=>
+# change (spaces)slash(or semicolon)(spaces) to comma
+ tagsfromtitle=> *[/;] *=>,
+ tagsfromtitle=> [xX] =>,
+ 
+# remove [] or () blocks and leading/trailing spaces/dashes/colons
+ title=>[-: ]*[\(\[]([^\]\)]+)[\)\]][-: ]*=>
+# remove 'Thread' and the next word, usually "Thread 2", "Thread
+# four", "Thread iv", etc
+ title,tagsfromtitle=>[-: ]*[Tt]hread [^ ]+[-: ]*=>
+
+add_to_extra_titlepage_entries:,tagsfromtitle,forumtags
+
+## '.SPLIT' tells the system to split by ','
+add_to_include_subject_tags:,tagsfromtitle.SPLIT
+## Or to also include forumtags add this to personal.ini:
+#add_to_include_subject_tags:,forumtags
+
+## base_xenforoforum reads Published and Updated datetimes from
+## Threadmarks if used, or from the posted & updated times of the
+## 'first' post if no threadmarks.
+datePublished_format:%%Y-%%m-%%d %%H:%%M:%%S
+dateUpdated_format:%%Y-%%m-%%d %%H:%%M:%%S
+
+## Only take the first X characters of the 'first' post to use as
+## the description.
+description_limit:500
+
 ## Each output format has a section that overrides [defaults]
 [html]
 
@@ -583,6 +673,8 @@ extratags: FanFiction,Testing,HTML
 ## doesn't like that.  If do_update_hook is uncommented and set true,
 ## the adapter will discard all existing chapters from the newest one
 ## on when updating to enforce accurate chapters.
+## Starting July 2015, FFF stores chapter URLs in the chapter files.
+## Stories downloaded after that shouldn't need this setting anymore.
 #do_update_hook:false
 
 ## AO3 adapter defines a few extra metadata entries.
@@ -1040,6 +1132,12 @@ extra_valid_entries:size
 # don't show twitter icon.
 cover_exclusion_regexp:/res/css/bir.png
 
+[forums.spacebattles.com]
+## see [base_xenforoforum]
+
+[forums.sufficientvelocity.com]
+## see [base_xenforoforum]
+
 [grangerenchanted.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -1170,6 +1268,12 @@ extracategories:NCIS
 extracategories:Buffy: The Vampire Slayer
 extracharacters:Willow
 
+[ninelives.dark-solace.org]
+## Site dedicated to these categories/characters/ships
+extracategories:The Walking Dead
+extracharacters:Carol,Daryl
+extraships:Carol/Daryl
+
 [nocturnal-light.net]
 ## Extra metadata that this adapter knows about.  See [dramione.org]
 ## for examples of how to use them.
@@ -1747,7 +1851,38 @@ extraships:InuYasha/Kagome
 ## Site dedicated to these categories/characters/ships
 extracategories:Lord of the Rings
 
+[www.masseffect2.in]
+## Site dedicated to this fandom.
+extracategories: Mass Effect
+
+## Ucoz.com engine, upon which MassEffect2.in is based, imposes an unspecified limit on request frequency.
+## Reports vary from `5 requests per second' to `2 requests per second for more than 10 per minute'.
+## With default settings, a several-hours IP ban may follow, so set it higher.
+slow_down_sleep_time: 2
+
+## Whether to exclude editor signature from the bottom of chapter text.
+exclude_editor_signature: false
+
+## Stories on the site almost never have cover image, and for the stories which do,
+## this may be adjusted in `personal.ini' before downloading.
+never_make_cover: true
+
+## Titles for ratings identified by 1- or 2-letter codes from `ERATING system'
+## (`система Р.Е.Й.Т.И.Н.Г.').  MassEffect2.in and some other sites adopted it,
+## but changed titles and update them occasionally.
+rating_titles: R=RESTRICTED (16+), E=EXEMPT (18+), I=ART HOUSE, T=To every, A=IN=Иной мир, Nn=Новый мир, G=О\, Господи!
+adult_ratings: E,R
+
 [www.mediaminer.org]
+dateUpdated_format:%%Y-%%m-%%d %%H:%%M:%%S
+## Note that mediaminer doesn't give datePublished on the story's
+## index page--it's collected from the earliest uploaded chapter.  So
+## it's not available when only fetching metadata.
+datePublished_format:%%Y-%%m-%%d %%H:%%M:%%S
+
+## some sites include images that we don't ever want becoming the
+## cover image.  This lets you exclude them.
+cover_exclusion_regexp:/img/rss.png
 
 [www.midnightwhispers.ca]
 ## Some sites do not require a login, but do require the user to
@@ -1815,6 +1950,13 @@ extracategories:Psych
 ## Site dedicated to these categories/characters/ships
 extracategories:Queer as Folk
 
+[quotev.com]
+extra_valid_entries:pages,readers,reads,favorites
+pages_label:Pages
+readers_label:Readers
+reads_label:Reads
+favorites_label:Favorites
+
 [www.restrictedsection.org]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In

calibre-plugin/prefs.py

@@ -1,5 +1,5 @@
-#!/usr/bin/env python
-# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+# -*- coding: utf-8 -*-
+
 from __future__ import (unicode_literals, division, absolute_import,
                         print_function)
 
@@ -85,6 +85,7 @@ default_prefs['read_lists'] = ''
 default_prefs['addtolists'] = False
 default_prefs['addtoreadlists'] = False
 default_prefs['addtolistsonread'] = False
+default_prefs['autounnew'] = False
 
 default_prefs['updatecalcover'] = None
 default_prefs['gencalcover'] = SAVE_YES
@@ -115,6 +116,7 @@ default_prefs['imapsessionpass'] = False
 default_prefs['imapfolder'] = 'INBOX'
 default_prefs['imapmarkread'] = True
 default_prefs['auto_reject_from_email'] = False
+default_prefs['download_from_email_immediately'] = False
 
 def set_library_config(library_config,db):
     db.prefs.set_namespaced(PREFS_NAMESPACE,

fanficfare/adapters/__init__.py

@@ -135,6 +135,12 @@ import adapter_fanfictionjunkiesde
 import adapter_devianthearts
 import adapter_tgstorytimecom
 import adapter_itcouldhappennet
+import adapter_forumsspacebattlescom
+import adapter_forumssufficientvelocitycom
+import adapter_questionablequestingcom
+import adapter_ninelivesdarksolaceorg
+import adapter_masseffect2in
+import adapter_quotevcom
 
 ## This bit of complexity allows adapters to be added by just adding
 ## importing.  It eliminates the long if/else clauses we used to need
@@ -194,14 +200,24 @@ def getAdapter(config,url,anyurl=False):
     # No adapter found.
     raise exceptions.UnknownSite( url, [cls.getSiteDomain() for cls in __class_list] )
 
-def getConfigSections():
+def getSiteSections():
+    # doesn't include base sections. Sections rather than site DNS because of squidge/peja
     return [cls.getConfigSection() for cls in __class_list]
 
+def getConfigSections():
+    # does include base sections.
+    sections = set()
+    for cls in __class_list:
+        sections.update(cls.getConfigSections())
+    return sections
+
 def get_bulk_load_sites():
     # for now, all eFiction Base adapters are assumed to allow bulk_load.
-    return [cls.getConfigSection().replace('www.','') for cls in
-            filter( lambda x : issubclass(x,base_efiction_adapter.BaseEfictionAdapter),
-                    __class_list)]
+    sections = set()
+    for cls in filter( lambda x : issubclass(x,base_efiction_adapter.BaseEfictionAdapter),
+                       __class_list):
+        sections.update( [ x.replace('www.','') for x in cls.getConfigSections() ] )
+    return sections
 
 def getSiteExamples():
     l=[]
@@ -209,10 +225,10 @@ def getSiteExamples():
         l.append((cls.getConfigSection(),cls.getSiteExampleURLs().split()))
     return l
 
-def getConfigSectionFor(url):
+def getConfigSectionsFor(url):
     (cls,fixedurl) = getClassFor(url)
     if cls:
-        return cls.getConfigSection()
+        return cls.getConfigSections()
 
     # No adapter found.
     raise exceptions.UnknownSite( url, [cls.getSiteDomain() for cls in __class_list] )

fanficfare/adapters/adapter_darksolaceorg.py

@@ -16,321 +16,26 @@
 #
 
 # Software: eFiction
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
+from base_efiction_adapter import BaseEfictionAdapter
 
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
+class DarkSolaceOrgAdapter(BaseEfictionAdapter):
 
-from base_adapter import BaseSiteAdapter,  makeDate
+    @staticmethod
+    def getSiteDomain():
+        return 'dark-solace.org'
 
+    @classmethod
+    def getPathToArchive(self):
+        return '/elysian'
+
+    @classmethod
+    def getSiteAbbrev(self):
+        return 'dksl'
+        
+    @classmethod
+    def getDateFormat(self):
+        return "%B %d, %Y"
+                           
 def getClass():
     return DarkSolaceOrgAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class DarkSolaceOrgAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/elysian/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','dksl')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%B %d, %Y"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'dark-solace.org'
-        
-    @classmethod
-    def getAcceptDomains(cls):
-        return ['www.dark-solace.org','dark-solace.org']
-
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://"+cls.getSiteDomain()+"/elysian/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://")+"(www\.)?"+re.escape(self.getSiteDomain()+"/elysian/viewstory.php?sid=")+r"\d+$"
-
-    ## Login seems to be reasonably standard across eFiction sites.
-    def needToLoginCheck(self, data):
-        if 'This story contains adult content not suitable for children' in data \
-                or "That password doesn't match the one in our database" in data \
-                or "Registered Users Only" in data:
-            return True
-        else:
-            return False
-        
-    def performLogin(self, url):
-        params = {}
-
-        if self.password:
-            params['penname'] = self.username
-            params['password'] = self.password
-        else:
-            params['penname'] = self.getConfig("username")
-            params['password'] = self.getConfig("password")
-        params['action'] = 'login'
-        params['submit'] = 'Submit'
     
-        loginUrl = 'http://www.' + self.getSiteDomain() + '/elysian/user.php'
-        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
-                                                              params['penname']))
-    
-        d = self._postUrl(loginUrl, params)
-    
-        if "Member Account" not in d : #User Account Page
-            logger.info("Failed to login to URL %s as %s, or have no authorization to access the story" % (loginUrl, params['penname']))
-            raise exceptions.FailedToLogin(url,params['penname'])
-            return False
-        else:
-            return True
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-        
-        if self.is_adult or self.getConfig("is_adult"):
-            # Weirdly, different sites use different warning numbers.
-            # If the title search below fails, there's a good chance
-            # you need a different number.  print data at that point
-            # and see what the 'click here to continue' url says.
-            addurl = "&ageconsent=ok&warning=5"
-        else:
-            addurl=""
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url+'&index=1'+addurl
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        if self.needToLoginCheck(data):
-            # need to log in for this one.
-            self.performLogin(url)
-            data = self._fetchUrl(url)
-            
-        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
-        if m != None:
-            if self.is_adult or self.getConfig("is_adult"):
-                # We tried the default and still got a warning, so
-                # let's pull the warning number from the 'continue'
-                # link and reload data.
-                addurl = m.group(1)
-                # correct stupid & error in url.
-                addurl = addurl.replace("&","&")
-                url = self.url+'&index=1'+addurl
-                logger.debug("URL 2nd try: "+url)
-
-                try:
-                    data = self._fetchUrl(url)
-                except urllib2.HTTPError, e:
-                    if e.code == 404:
-                        raise exceptions.StoryDoesNotExist(self.url)
-                    else:
-                        raise e    
-            else:
-                raise exceptions.AdultCheckRequired(self.url)
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-        
-        ## Title and author
-        div = soup.find('div', {'id' : 'pagetitle'})
-        
-        aut = div.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',aut['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/elysian/'+aut['href'])
-        self.story.setMetadata('author',aut.string)
-        aut.extract()
-
-        # first a tag in pagetitle is title
-        self.story.setMetadata('title',stripHTML(div.find('a')))
-        div.find('a').extract()
-        # only thing left in div(pagetitle) now should be 'by' and rating.
-        rating = stripHTML(div)
-        if '[' in rating:
-            self.story.setMetadata('rating', rating[rating.index('[')+1:-1])
-
-        for chapa in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+
-                                                       self.story.getMetadata('storyId')+'&chapter=\d+')):
-            self.chapterUrls.append((stripHTML(chapa),'http://'+self.host+'/elysian/'+chapa['href']))
-        
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
-        storylink = asoup.find('a', href=re.compile(r'viewstory.php\?sid='+
-                                                    self.story.getMetadata('storyId')+'($|[^\d])'))
-        # author's story list is paginated if there's a pagelinks div.
-        # Only need to look in it if the story wasn't on the first page.
-        pagelinks = asoup.find('div',{'id':'pagelinks'})
-        if pagelinks and storylink==None:
-            authpageslist = pagelinks.findAll('a',href=re.compile(r'action=storiesby'))
-            for page in authpageslist[1:]: # skip first, already checked above.
-                asoup = bs.BeautifulSoup(self._fetchUrl('http://'+self.host+'/elysian/'+page['href']))
-                storylink = asoup.find('a', href=re.compile(r'viewstory.php\?sid='+
-                                                            self.story.getMetadata('storyId')+'($|[^\d])'))
-                if storylink:
-                    break
-
-        if not storylink:
-            raise exceptions.FailedToDownload("Unable to find story metadata on author's page(s)")
-        
-        metalist = storylink.parent.parent
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-                
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
-        labels = metalist.findAll('span', {'class' : 'label'})
-        for labelspan in labels:
-            label = labelspan.text
-            value = labelspan.nextSibling
-
-            if 'Summary' in label:
-                ## Everything until the next span class='label'
-                svalue = ""
-                while value and not (defaultGetattr(value,'class') == 'label' or "Chapters: " in stripHTML(value)):
-                    svalue += str(value)
-                    value = value.nextSibling
-                self.setDescription(url,svalue)
-                #self.story.setMetadata('description',stripHTML(svalue))
-
-            if 'Word count' in label:
-                self.story.setMetadata('numWords', value)
-
-            if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-                for cat in cats:
-                    self.story.addToList('category',cat.string)
-
-            if 'Characters' in label:
-                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-                for char in chars:
-                    self.story.addToList('characters',char.string)
-
-            if 'Genre' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
-                for genre in genres:
-                    self.story.addToList('genre',genre.string)
-
-            if 'Warnings' in label:
-                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
-                for warning in warnings:
-                    self.story.addToList('warnings',warning.string)
-
-            if 'Completed' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-            
-            if 'Updated' in label:
-                # there's a stray [ at the end.
-                #value = value[0:-1]
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-
-        try:
-            # Find Series name from series URL.
-            a = metalist.find('a', href=re.compile(r"series.php\?seriesid=\d+"))
-            series_name = a.string
-            series_url = 'http://'+self.host+'/elysian/'+a['href']
-
-            # use BeautifulSoup HTML parser to make everything easier to find.
-            seriessoup = bs.BeautifulSoup(self._fetchUrl(series_url))
-            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
-            storylink = seriessoup.find('a', href=re.compile(r'viewstory.php\?sid='+
-                                                             self.story.getMetadata('storyId')+'($|[^\d])'))
-            if storylink and storylink.parent and storylink.parent['class'] != 'title': # in case of links inside story summaries.
-                storylink = None
-
-            offset = 0
-            # series story list is paginated if there's a pagelinks div.
-            # Only need to look in it if the story wasn't on the first page.
-            pagelinks = seriessoup.find('div',{'id':'pagelinks'})
-            if pagelinks and storylink==None:
-                authpageslist = pagelinks.findAll('a',href=re.compile(r'offset='))
-                for page in authpageslist[1:]: # skip first, already checked above.
-                    seriessoup = bs.BeautifulSoup(self._fetchUrl('http://'+self.host+'/elysian/'+page['href']))
-                    storylink = seriessoup.find('a', href=re.compile(r'viewstory.php\?sid='+
-                                                                     self.story.getMetadata('storyId')+'($|[^\d])'))
-                    if storylink and storylink.parent and storylink.parent['class'] != 'title': # in case of links inside story summaries.
-                        storylink = None
-                    if storylink:
-                        offset = int(page['href'].split('=')[-1]) # offset is last.
-                        break
-
-            # for reasons I don't understand, searching for story
-            # links by regex wasn't working reliably.  It was missing
-            # the javascript links sometimes.  This is cleaner anyway.
-            for i, div in enumerate(seriessoup.findAll('div', {'class':'title'})):
-                a = div.find('a') # first a is story link.
-                # skip 'report this' and 'TOC' links
-                if a == storylink:
-                    self.setSeries(series_name, 1+i+offset)
-                    self.story.setMetadata('seriesUrl',series_url)
-                    break
-            
-        except Exception, e:
-            logger.debug("Series parsing failed: %s"%e)
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = bs.BeautifulSoup(self._fetchUrl(url))
-        
-        div = soup.find('div', {'id' : 'story'})
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
-        return self.utf8FromSoup(url,div)

fanficfare/adapters/adapter_dramioneorg.py

@@ -38,8 +38,9 @@ class DramioneOrgAdapter(BaseSiteAdapter):
     def __init__(self, config, url):
         BaseSiteAdapter.__init__(self, config, url)
 
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
+        self.decode = ["utf8",
+                       "Windows-1252",]
+                               # 1252 is a superset of iso-8859-1.
                                # Most sites that claim to be
                                # iso-8859-1 (and some that claim to be
                                # utf8) are really windows-1252.

fanficfare/adapters/adapter_fanfictionnet.py

@@ -104,7 +104,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
             data = self._fetchUrl(url)
             #logger.debug("\n===================\n%s\n===================\n"%data)
             soup = self.make_soup(data)
-        except urllib2.HTTPError, e:
+        except urllib2.HTTPError as e:
             if e.code == 404:
                 raise exceptions.StoryDoesNotExist(url)
             else:
@@ -135,11 +135,15 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
                                                 chapcount+1)
                 logger.debug('=Trying newer chapter: %s' % tryurl)
                 newdata = self._fetchUrl(tryurl)
-                if "not found. Please check to see you are not using an outdated url." \
-                        not in newdata:
+                if "not found. Please check to see you are not using an outdated url." not in newdata \
+                        and "This request takes too long to process, it is timed out by the server." not in newdata:
                     logger.debug('=======Found newer chapter: %s' % tryurl)
                     soup = self.make_soup(newdata)
-            except:
+            except urllib2.HTTPError as e:
+                if e.code == 503:
+                    raise e
+            except e:
+                logger.warn("Caught an exception reading URL: %s sleeptime(%s) Exception %s."%(unicode(url),sleeptime,unicode(e)))
                 pass
             
         # Find authorid and URL from... author url.

fanficfare/adapters/adapter_fimfictionnet.py

@@ -178,8 +178,11 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
             if get_cover:
                 # try setting from href, if fails, try using the img src
                 if self.setCoverImage(self.url,coverurl)[0] == "failedtoload":
-                    coverurl = storyImage.find('img')['src']
-                    self.setCoverImage(self.url,coverurl)
+                    img = storyImage.find('img')
+                    # try src, then data-src, then leave None.
+                    coverurl = img.get('src',img.get('data-src',None))
+                    if coverurl:
+                        self.setCoverImage(self.url,coverurl)
 
             coverSource = storyImage.find('a', {'class':'source'})
             if coverSource:

fanficfare/adapters/adapter_forumsspacebattlescom.py

@@ -0,0 +1,35 @@
+#  -*- coding: utf-8 -*-
+
+# Copyright 2015 FanFicFare team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from base_xenforoforum_adapter import BaseXenForoForumAdapter
+
+def getClass():
+    return ForumsSpacebattlesComAdapter
+
+class ForumsSpacebattlesComAdapter(BaseXenForoForumAdapter):
+
+    def __init__(self, config, url):
+        BaseXenForoForumAdapter.__init__(self, config, url)
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','fsb')
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'forums.spacebattles.com'
+

fanficfare/adapters/adapter_forumssufficientvelocitycom.py

@@ -0,0 +1,34 @@
+#  -*- coding: utf-8 -*-
+
+# Copyright 2015 FanFicFare team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from base_xenforoforum_adapter import BaseXenForoForumAdapter
+
+def getClass():
+    return ForumsSufficientVelocityComAdapter
+
+class ForumsSufficientVelocityComAdapter(BaseXenForoForumAdapter):
+
+    def __init__(self, config, url):
+        BaseXenForoForumAdapter.__init__(self, config, url)
+
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','fsv')
+
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'forums.sufficientvelocity.com'

fanficfare/adapters/adapter_hpfandomnet.py

@@ -98,6 +98,26 @@ class HPFandomNetAdapterAdapter(BaseSiteAdapter): # XXX
 
         # Now go hunting for all the meta data and the chapter list.
         
+        # set first URL
+        a = soup.find('a', href=re.compile(r"viewstory.php\?sid=\d+"))
+        self._setURL('http://' + self.getSiteDomain() + '/eff/'+a['href'])
+        if self.parsedUrl.query.split('=',)[1] != self.story.getMetadata('storyId'):
+            self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+            url = self.url
+            logger.debug("reset URL: "+url)
+            try:
+                data = self._fetchUrl(url)
+            except urllib2.HTTPError, e:
+                if e.code == 404:
+                    raise exceptions.StoryDoesNotExist(self.url)
+                else:
+                    raise e
+
+            # use BeautifulSoup HTML parser to make everything easier to find.
+            soup = bs.BeautifulSoup(data)
+            
+#        self.story.setMetadata('storyId', re.compile(self.getSiteURLPattern()).match(a).group('storyId'))
+        
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
         self.story.setMetadata('authorId',a['href'].split('=')[1])

fanficfare/adapters/adapter_masseffect2in.py

@@ -0,0 +1,719 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2015 FanFicFare team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import bs4
+import datetime
+import logging
+import re
+import urllib2
+
+from ..htmlcleanup import removeEntities, stripHTML
+from .. import exceptions as exceptions
+from base_adapter import BaseSiteAdapter, makeDate
+
+
+_logger = logging.getLogger(__name__)
+
+
+def getClass():
+    """Returns adapter class defined in this module."""
+    return MassEffect2InAdapter
+
+
+class ParsingError(Exception):
+    """Indicates an error while parsing web page content."""
+    def __init__(self, message):
+        Exception.__init__(self)
+        self.message = message
+
+    def __str__(self):
+        return self.message
+
+
+class MassEffect2InAdapter(BaseSiteAdapter):
+    """
+    Provides support for MassEffect2.in site as story source.
+    Can be used as a template for sites build upon Ucoz.com engine (until no base class extracted).
+    Specializations:
+        1) Russian content (date format, genre names, etc.);
+        2) original `E.R.A.T.I.N.G.' rating scale, used by masseffect2.in
+           and some affiliated sites, denoted with images;
+        3) editor signatures an an option to remove them.
+    """
+
+    WORD_PATTERN = re.compile(u'\w+', re.UNICODE)
+    DOCUMENT_ID_PATTERN = re.compile(u'\d+-\d+-\d+-\d+')
+    SITE_LANGUAGE = u'Russian'
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["utf8"]
+
+        self.story.setMetadata('siteabbrev', 'me2in')
+        self.story.setMetadata('storyId', self._getDocumentId(self.url))
+
+        self._setURL(self._makeDocumentUrl(self.story.getMetadata('storyId')))
+
+        self._chapters = {}
+        self._parsingConfiguration = None
+
+    # Must be @staticmethod, not @classmethod!
+    @staticmethod
+    def getSiteDomain():
+        return 'www.masseffect2.in'
+
+    @classmethod
+    def getSiteExampleURLs(cls):
+        return u' '.join([cls._makeDocumentUrl('19-1-0-1234'),
+                          cls._makeDocumentUrl('24-1-0-4321')])
+
+    def getSiteURLPattern(self):
+        return re.escape(self._makeDocumentUrl('')) + self.DOCUMENT_ID_PATTERN.pattern
+
+    def use_pagecache(self):
+        """Allows use of downloaded page cache.  It is essential for this
+        adapter, because the site does not offers chapter URL list, and many
+        pages have to be fetched and parsed repeatedly."""
+        return True
+
+    def extractChapterUrlsAndMetadata(self):
+        """Extracts chapter URLs and story metadata.  Actually downloads all
+        chapters, which is not exactly right, but necessary due to technical
+        limitations of the site."""
+
+        def followChapters(starting, forward=True):
+            if forward:
+                url = starting.getNextChapterUrl()
+            else:
+                url = starting.getPreviousChapterUrl()
+            if url:
+                url = self._makeDocumentUrl(self._getDocumentId(url))
+                following = self._makeChapter(url)
+                # Do not follow links to related, but different stories (prequels or sequels).
+                if not following.isFromStory(starting.getHeading()):
+                    return
+                if forward:
+                    yield following
+                for chapter in followChapters(following, forward):
+                    yield chapter
+                if not forward:
+                    yield following
+
+        try:
+            startingChapter = self._makeChapter(self.url)
+        except urllib2.HTTPError, error:
+            if error.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            raise
+
+        # We only have one date for each chapter and assume the oldest one
+        # to be publication date and the most recent one to be update date.
+        datePublished = datetime.datetime.max
+        dateUpdated = datetime.datetime.min
+        wordCount = 0
+        # We aim at counting chapters, not chapter parts.
+        chapterCount = 0
+        storyInProgress = False
+
+        chapters = \
+            list(followChapters(startingChapter, forward=False)) + \
+            [startingChapter] + \
+            list(followChapters(startingChapter, forward=True))
+
+        headings = [chapter.getHeading() for chapter in chapters]
+        largestCommonPrefix = _getLargestCommonPrefix(*headings)
+        prefixLength = len(largestCommonPrefix)
+        storyTitleEnd, chapterTitleStart = prefixLength, prefixLength
+        match = re.search(u'[:\.\s]*(?P<chapter>глава\s+)?$', largestCommonPrefix, re.IGNORECASE | re.UNICODE)
+        if match:
+            storyTitleEnd -= len(match.group())
+            label = match.group('chapter')
+            if label:
+                chapterTitleStart -= len(label)
+        storyTitle = largestCommonPrefix[:storyTitleEnd]
+        self.story.setMetadata('title', storyTitle)
+
+        garbagePattern = re.compile(u'(?P<start>^)?[:\.\s]*(?(start)|$)', re.UNICODE)
+        indexPattern = re.compile(u'(?:глава\s)?(?:(?<!\d)(?P<index>\d{1,3})(?=\D|$))', re.IGNORECASE | re.UNICODE)
+
+        for chapter in chapters:
+            url = chapter.getUrl()
+            self._chapters[url] = chapter
+            _logger.debug(u"Processing chapter `%s'.", url)
+
+            try:
+                authorName = chapter.getAuthorName()
+                if authorName:
+                    self.story.extendList('author', [authorName])
+                    authorId = chapter.getAuthorId()
+                    if authorId:
+                        authorUrl = 'http://%s/index/%s' % (self.getSiteDomain(), authorId)
+                    else:
+                        authorId = u''
+                        authorUrl = u''
+                    self.story.extendList('authorId', [authorId])
+                    self.story.extendList('authorUrl', [authorUrl])
+
+                if not self.story.getMetadata('rating'):
+                    ratingTitle = chapter.getRatingTitle()
+                    if ratingTitle:
+                        self.story.setMetadata('rating', ratingTitle)
+
+                if not self.story.getMetadata('description'):
+                    summary = chapter.getSummary()
+                    if summary:
+                        self.story.setMetadata('description', summary)
+
+                datePublished = min(datePublished, chapter.getDate())
+                dateUpdated = max(dateUpdated, chapter.getDate())
+
+                self.story.extendList('genre', chapter.getGenres())
+                self.story.extendList('characters', chapter.getCharacters())
+                self.story.extendList('ships', chapter.getPairings())
+
+                wordCount += self._getWordCount(chapter.getTextElement())
+
+                # Chapter status usually represents the story status, so we want the last chapter status.
+                # Some chapters may have no status attribute.
+                chapterInProgress = chapter.isInProgress()
+                if chapterInProgress is not None:
+                    storyInProgress = chapterInProgress
+
+                # If any chapter is adult, consider the whole story adult.
+                if chapter.isAdult():
+                    self.story.setMetadata('is_adult', True)
+                    warning = chapter.getWarning()
+                    if warning:
+                        self.story.extendList('warnings', [warning])
+
+                chapterTitle = re.sub(garbagePattern, u'', chapter.getHeading()[chapterTitleStart:])
+
+                match = re.search(indexPattern, chapterTitle)
+                if match:
+                    index = int(match.group('index'))
+                    chapterCount = max(chapterCount, index)
+                else:
+                    chapterCount += 1
+
+                self.chapterUrls.append((chapterTitle, url))
+            except ParsingError, error:
+                raise exceptions.FailedToDownload(u"Failed to download chapter `%s': %s" % (url, error))
+
+        # Some metadata are handled separately due to format conversions.
+        self.story.setMetadata('status', 'In Progress' if storyInProgress else 'Completed')
+        self.story.setMetadata('datePublished', datePublished)
+        self.story.setMetadata('dateUpdated', dateUpdated)
+        self.story.setMetadata('numWords', str(wordCount))
+        self.story.setMetadata('numChapters', chapterCount)
+
+        # Site-specific metadata.
+        self.story.setMetadata('language', self.SITE_LANGUAGE)
+
+    def getChapterText(self, url):
+        """Grabs the text for an individual chapter."""
+        if url not in self._chapters:
+            raise exceptions.FailedToDownload(u"No chapter `%s' present!" % url)
+        chapter = self._chapters[url]
+        return self.utf8FromSoup(url, chapter.getTextElement())
+
+    def _makeChapter(self, url):
+        """Creates a chapter object given a URL."""
+        document = self.make_soup(self._fetchUrl(url))
+        chapter = Chapter(self._getParsingConfiguration(), url, document)
+        return chapter
+
+    def _getWordCount(self, element):
+        """Returns word count in plain text extracted from chapter body."""
+        text = stripHTML(element)
+        count = len(re.findall(self.WORD_PATTERN, text))
+        return count
+
+    def _getParsingConfiguration(self):
+        if not self._parsingConfiguration:
+            self._parsingConfiguration = {}
+
+            adultRatings = self.getConfigList('adult_ratings')
+            if not adultRatings:
+                raise exceptions.PersonalIniFailed(
+                    u"Missing `adult_ratings' setting", u"MassEffect2.in", u"?")
+            adultRatings = set(adultRatings)
+            self._parsingConfiguration['adultRatings'] = adultRatings
+
+            ratingTitleDescriptions = self.getConfigList('rating_titles')
+            if ratingTitleDescriptions:
+                ratingTitles = {}
+                for ratingDescription in ratingTitleDescriptions:
+                    parts = ratingDescription.split(u'=')
+                    if len(parts) < 2:
+                        _logger.warning(
+                            u"Invalid `rating_titles' setting, missing `=' in `%s'."
+                            % ratingDescription)
+                        continue
+                    labels = parts[:-1]
+                    title = parts[-1]
+                    for label in labels:
+                        ratingTitles[label] = title
+                        # Duplicate label aliasing in adult rating set.
+                        if label in adultRatings:
+                            adultRatings.add(*labels)
+                self._parsingConfiguration['adultRatings'] = list(adultRatings)
+                self._parsingConfiguration['ratingTitles'] = ratingTitles
+            else:
+                raise exceptions.PersonalIniFailed(
+                    u"Missing `rating_titles' setting", u"MassEffect2.in", u"?")
+
+            self._parsingConfiguration['excludeEditorSignature'] = \
+                self.getConfig('exclude_editor_signature', False)
+
+        return self._parsingConfiguration
+
+    def _getDocumentId(self, url):
+        """Extract document ID from MassEffect2.in URL."""
+        match = re.search(self.DOCUMENT_ID_PATTERN, url)
+        if not match:
+            raise ValueError(u"Failed to extract document ID from `'" % url)
+        documentId = url[match.start():match.end()]
+        return documentId
+
+    @classmethod
+    def _makeDocumentUrl(cls, documentId):
+        """Make a chapter URL given a document ID."""
+        return 'http://%s/publ/%s' % (cls.getSiteDomain(), documentId)
+
+
+class Chapter(object):
+    """Represents a lazily-parsed chapter of a story."""
+    def __init__(self, configuration, url, document):
+        self._configuration = configuration
+        self._url = url
+        self._document = document
+        # Lazy-loaded:
+        self._heading = None
+        self._date = None
+        self._author = None
+        self._attributes = None
+        self._textElement = None
+        self._infoBar = None
+
+    def getHeading(self):
+        return self._extractHeading()
+
+    def getSummary(self):
+        attributes = self.__getAttributes()
+        if 'summary' in attributes:
+            return attributes['summary']
+
+    def getAuthorId(self):
+        author = self._getAuthor()
+        if author:
+            return author['id']
+
+    def getAuthorName(self):
+        author = self._getAuthor()
+        if author:
+            return author['name']
+
+    def getDate(self):
+        return self.__getDate()
+
+    def getRatingTitle(self):
+        attributes = self.__getAttributes()
+        if 'rating' in attributes:
+            return attributes['rating']['title']
+
+    def isAdult(self):
+        attributes = self.__getAttributes()
+        if 'rating' in attributes and attributes['rating']['isAdult']:
+            return True
+        if 'warning' in attributes:
+            return True
+        return False
+
+    def getWarning(self):
+        attributes = self.__getAttributes()
+        if 'warning' in attributes:
+            return attributes['warning']
+
+    def getCharacters(self):
+        return self._getListAttribute('characters')
+
+    def getPairings(self):
+        return self._getListAttribute('pairings')
+
+    def getGenres(self):
+        return self._getListAttribute('genres')
+
+    def isInProgress(self):
+        attributes = self.__getAttributes()
+        if 'isInProgress' in attributes:
+            return attributes['isInProgress']
+
+    def getUrl(self):
+        return self._url
+
+    def getTextElement(self):
+        return self._getTextElement()
+
+    def getPreviousChapterUrl(self):
+        """Download chapters following `Previous chapter' links.
+        Returns a list of chapters' URLs."""
+        return self._getSiblingChapterUrl({'class': 'fl tal'})
+
+    def getNextChapterUrl(self):
+        """Download chapters following `Next chapter' links.
+        Returns a list of chapters' URLs."""
+        return self._getSiblingChapterUrl({'class': 'tar fr'})
+
+    def isFromStory(self, storyTitle, prefixThreshold=-1):
+        """Check if this chapter is from a story different from the given one.
+        Prefix threshold specifies how long common story title prefix shall be
+        for chapters from one story: negative value means implementation-defined
+        optimum, zero inhibits the check, and positive value adjusts threshold."""
+
+        def getFirstWord(string):
+            match = re.search(u'^\s*\w+', string, re.UNICODE)
+            return string[match.start():match.end()]
+
+        thisStoryTitle = self.getHeading()
+        if prefixThreshold != 0:
+            if prefixThreshold < 0:
+                prefixThreshold = min(
+                    len(getFirstWord(storyTitle)), len(getFirstWord(thisStoryTitle)))
+            else:
+                prefixThreshold = min(
+                    prefixThreshold, len(storyTitle), len(thisStoryTitle))
+            result = len(_getLargestCommonPrefix(storyTitle, thisStoryTitle)) >= prefixThreshold
+            return result
+        else:
+            return storyTitle != thisStoryTitle
+
+    def _getListAttribute(self, name):
+        """Return an attribute value as a list or an empty list if the attribute is absent."""
+        attributes = self.__getAttributes()
+        if name in attributes:
+            return attributes[name]
+        return []
+
+    def _extractHeading(self):
+        """Extract header text from the document."""
+        return stripHTML(
+            self._document.find('div', {'class': 'eTitle'}).string)
+
+    def __getHeading(self):
+        """Lazily parse and return heading."""
+        if not self._heading:
+            self._heading = self._extractHeading()
+        return self._heading
+
+    def _getAuthor(self):
+        """Lazily parse and return author's information."""
+        if not self._author:
+            self._author = self._parseAuthor()
+        return self._author
+
+    def _parseAuthor(self):
+        """Locate and parse chapter author's information to a dictionary with author's `id' and `name'."""
+        try:
+            authorLink = self._getInfoBarElement() \
+                .find('i', {'class': 'icon-user'}) \
+                .findNextSibling('a')
+        except AttributeError:
+            raise ParsingError(u'Failed to locate author link.')
+        match = re.search(u'(8-\d+)', authorLink['onclick'])
+        if not match:
+            raise ParsingError(u'Failed to extract author ID.')
+        authorId = match.group(0)
+        authorName = stripHTML(authorLink.text)
+        return {
+            'id': authorId,
+            'name': authorName
+        }
+
+    def __getDate(self):
+        """Lazily parse chapter date."""
+        if not self._date:
+            self._date = self._parseDate()
+        return self._date
+
+    def _parseDate(self):
+        """Locate and parse chapter date."""
+        try:
+            dateText = self._getInfoBarElement() \
+                .find('i', {'class': 'icon-eye'}) \
+                .findPreviousSibling(text=True) \
+                .strip(u'| \n')
+        except AttributeError:
+            raise ParsingError(u'Failed to locate date.')
+        date = makeDate(dateText, '%d.%m.%Y')
+        return date
+
+    def _getInfoBarElement(self):
+        """Locate informational bar element, containing chapter date and author, on the page."""
+        if not self._infoBar:
+            self._infoBar = self._document.find('td', {'class': 'eDetails2'})
+            if not self._infoBar:
+                raise ParsingError(u'No informational bar found.')
+        return self._infoBar
+
+    def __getAttributes(self):
+        """Lazily parse attributes."""
+        if not self._attributes:
+            self._attributes = self._parseAttributes()
+        return self._attributes
+
+    def _parseAttributes(self):
+        """Parse chapter attribute block and return it as a dictionary with standard entries."""
+
+        attributes = {}
+        attributesText = u''
+        try:
+            starter = self._document \
+                .find('div', {'class': 'comm-div'}) \
+                .findNextSibling('div', {'class': 'cb'})
+            bound = starter.findNextSibling('div', {'class': 'cb'})
+
+            def processElement(element):
+                """Return textual representation an *inline* element of chapter attribute block."""
+                result = u''
+                if isinstance(element, bs4.Tag):
+                    if element.name in ('b', 'strong', 'font', 'br'):
+                        result += u"\n"
+                    if element.name == 's':
+                        result += u"<s>%s</s>" % stripHTML(element)
+                    else:
+                        result += stripHTML(element)
+                else:
+                    result += removeEntities(element)
+                return result
+
+            elements = starter.nextSiblingGenerator()
+            for element in elements:
+                if isinstance(element, bs4.Tag):
+                    if element == bound:
+                        break
+                    else:
+                        if element.name in ('div', 'p'):
+                            attributesText += u"\n"
+                            for child in element.childGenerator():
+                                attributesText += processElement(child)
+                            continue
+                attributesText += processElement(element)
+
+            elements = starter.nextGenerator()
+            for element in elements:
+                if isinstance(element, bs4.Tag):
+                    if element == bound:
+                        break
+                    elif element.name == 'img':
+                        rating = self._parseRatingFromImage(element)
+                        if rating:
+                            attributes['rating'] = rating
+                            break
+        except AttributeError or TypeError:
+            raise ParsingError(u'Failed to locate and collect attributes.')
+
+        separators = u"\r\n :;."
+        freestandingText = u''
+        for line in attributesText.split(u'\n'):
+            if line.count(u':') != 1:
+                freestandingText += line
+                continue
+            key, value = line.split(u':', 1)
+            key = key.strip(separators).lower()
+            value = value.strip().strip(separators)
+            parsed = self._parseAttribute(key, value)
+            for parsedKey, parsedValue in parsed.iteritems():
+                attributes[parsedKey] = parsedValue
+
+        freestandingText = freestandingText.strip()
+        if 'summary' not in attributes and freestandingText:
+            attributes['summary'] = freestandingText
+
+        if 'rating' not in attributes:
+            _logger.warning(u"Failed to locate or recognize rating for `%s'!", self.getUrl())
+
+        return attributes
+
+    # Most, but not all, URLs of rating icons match this.
+    RATING_LABEL_PATTERN = re.compile(u'/(?P<rating>[ERATINnG]+)\.png$')
+
+    def _parseRatingFromImage(self, element):
+        """Given an image element, try to parse story rating from it."""
+        # Although deprecated, `has_key()' is required here.
+        if not element.has_attr('src'):
+            return
+        source = element['src']
+        if 'REITiNG' in source:
+            match = re.search(self.RATING_LABEL_PATTERN, source)
+            if not match:
+                return
+            label = match.group('rating')
+            if label in self._configuration['ratingTitles']:
+                return {
+                    'label': label,
+                    'title': self._configuration['ratingTitles'][label],
+                    'isAdult': label in self._configuration['adultRatings']
+                }
+            else:
+                _logger.warning(u"No title found for rating label `%s'!" % label)
+        # TODO: conduct a research on such abnormal URLs.
+        elif source == 'http://www.masseffect2.in/_fr/10/1360399.png':
+            label = 'Nn'
+            return {
+                'label': 'Nn',
+                'title': self._configuration['ratingTitles'][label],
+                'isAdult': label in self._configuration['adultRatings']
+            }
+
+    # Various `et cetera' and `et al' forms in Russian texts.
+    # Intended to be used with whole strings!
+    ETC_PATTERN = re.compile(
+        u'''[и&]\s(?:
+              (?:т\.?\s?[пд]?\.?)|
+              (?:др(?:угие|\.)?)|
+              (?:пр(?:очие|\.)?)|
+              # Note: identically looking letters `K' and `o'
+              # below are from Latin and Cyrillic alphabets.
+              (?:ко(?:мпания)?|[KК][oо°])
+            )$
+        ''',
+        re.IGNORECASE + re.UNICODE + re.VERBOSE)
+
+    # `Author's Notes' and its variants in Russian.
+    ANNOTATION_PATTERN = re.compile(u'аннотация|описание|(?:(?:за|при)мечание\s)?(?:от\s)?автора', re.UNICODE)
+
+    def _parseAttribute(self, key, value):
+        """
+        Parse a single a single record in chapter attributes for chapter metadata.
+        Return a dictionary of canonical attributes and values (i. e. multiple attributes may be discovered).
+        """
+
+        def refineCharacter(name):
+            """Refines character name from stop-words and distortions."""
+            strippedName = name.strip()
+            nameOnly = re.sub(self.ETC_PATTERN, u'', strippedName)
+            # TODO: extract canonical name (even ME-specific?).
+            canonicalName = nameOnly
+            return canonicalName
+
+        if re.match(u'жанры?', key, re.UNICODE):
+            genres = filter(bool, map(unicode.strip, re.split(u'[,;/]', value)))
+            return {'genres': genres}
+        elif key == u'статус':
+            isInProgress = value == u'в процессе'
+            return {'isInProgress': isInProgress}
+        elif key == u'персонажи':
+            participants = map(refineCharacter, re.split(u'[,;]', value))
+            characters = []
+            pairings = []
+            for participant in participants:
+                if u'/' in participant:
+                    pairings.append(participant)
+                else:
+                    characters.append(participant)
+            return {
+                'characters': characters,
+                'pairings': pairings
+            }
+        elif key == u'предупреждение':
+            return {'warning': value}
+        elif re.match(self.ANNOTATION_PATTERN, key):
+            if not value.endswith(u'.'):
+                value += u'.'
+            # Capitalize would make value[1:] lowercase, which we don't want.
+            value = value[:1].upper() + value[1:]
+            return {'summary': value}
+        else:
+            _logger.info(u"Unrecognized attribute `%s' ignored.", key)
+            return {}
+
+    def _getTextElement(self):
+        """Locate chapter body text element on the page."""
+        if not self._textElement:
+            self._textElement = self.__collectTextElements()
+        return self._textElement
+
+    def __collectTextElements(self):
+        """Return all elements containing parts of chapter text (which may be
+        <p>aragraphs, <div>isions or plain text nodes) under a single root."""
+        starter = self._document.find('div', {'id': u'article'})
+        if starter is None:
+            # FIXME: This will occur if the method is called more than once.
+            # The reason is elements appended to `root' are removed from the document.
+            # BS 4.4 implements cloning via `copy.copy()', but supporting it for BS 4.3
+            # would be error-prone (due to relying on BS internals) and is not needed.
+            if self._textElement:
+                _logger.debug(u"You may not call this function more than once!")
+            raise ParsingError(u'Failed to locate text.')
+        collection = [starter]
+        for element in starter.nextSiblingGenerator():
+            if element is None:
+                break
+            if isinstance(element, bs4.Tag) and element.name == 'tr':
+                break
+            collection.append(element)
+        root = bs4.Tag(name='td')
+        for element in collection:
+            root.append(element)
+
+        if self._configuration['excludeEditorSignature']:
+            root = self._excludeEditorSignature(root)
+
+        return root
+
+    def _getSiblingChapterUrl(self, selector):
+        """Locate a link to a sibling chapter, either previous or next one, and return its URL."""
+        block = self._document \
+            .find('td', {'class': 'eDetails1'}) \
+            .find('div', selector)
+        if not block:
+            return
+        link = block.find('a')
+        if not link:
+            return
+        return link['href']
+
+    # Editor signature always starts with something like this.
+    SIGNED_PATTERN = re.compile(u'отредактирова(?:но|ла?)[:.\s]', re.IGNORECASE + re.UNICODE)
+
+    def _excludeEditorSignature(self, root):
+        """Exclude editor signature from within `root' element."""
+        for textNode in root.findAll(text=True):
+            if re.match(self.SIGNED_PATTERN, textNode.string):
+                editorLink = textNode.findNext('a')
+                if editorLink:
+                    editorLink.extract()
+                # Seldom editor link has inner formatting, which is sibling DOM-wise.
+                editorName = textNode.findNext('i')
+                if editorName:
+                    editorName.extract()
+                textNode.extract()
+                # We could try removing container element, but there is a risk
+                # of removing text ending with it.  Better play safe here.
+                break
+        return root
+
+
+def _getLargestCommonPrefix(*args):
+    """Returns largest common prefix of all unicode arguments, ignoring case.
+    :rtype : unicode
+    """
+    from itertools import takewhile, izip
+    toLower = lambda xs: map(lambda x: x.lower(), xs)
+    allSame = lambda xs: len(set(toLower(xs))) == 1
+    return u''.join([i[0] for i in takewhile(allSame, izip(*args))])

fanficfare/adapters/adapter_mediaminerorg.py

@@ -40,39 +40,53 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
         
         # get storyId from url--url validation guarantees query correct
         m = re.match(self.getSiteURLPattern(),url)
+        urltitle='urltitle'
         if m:
-            if m.group('id'):
-                self.story.setMetadata('storyId',m.group('id'))
+            if m.group('id1'):
+                self.story.setMetadata('storyId',m.group('id1'))
+                urltitle=m.group('urltitle1')
             elif m.group('id2'):
                 self.story.setMetadata('storyId',m.group('id2'))
+                urltitle=m.group('urltitle2')
             elif m.group('id3'):
-                self.story.setMetadata('storyId',m.group('id2'))
+                self.story.setMetadata('storyId',m.group('id3'))
+            else:
+                raise InvalidStoryURL(url,
+                                      self.getSiteDomain(),
+                                      self.getSiteExampleURLs())
             
             # normalized story URL.
-            self._setURL('http://' + self.getSiteDomain() + '/fanfic/view_st.php/'+self.story.getMetadata('storyId'))
+            self._setURL('http://' + self.getSiteDomain() + '/fanfic/s/'+urltitle+'/'+self.story.getMetadata('storyId'))
         else:
             raise exceptions.InvalidStoryURL(url,
                                              self.getSiteDomain(),
                                              self.getSiteExampleURLs())
             
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%B %d, %Y %H:%M"
+            
     @staticmethod
     def getSiteDomain():
         return 'www.mediaminer.org'
 
     @classmethod
     def getSiteExampleURLs(cls):
-        return "http://"+cls.getSiteDomain()+"/fanfic/view_st.php/123456 http://"+cls.getSiteDomain()+"/fanfic/view_ch.php/1234123/123444#fic_c"
+        return "http://"+cls.getSiteDomain()+"/fanfic/s/story-title/123456 http://"+cls.getSiteDomain()+"/fanfic/c/story-title/chapter-title/123456/987612"
 
     def getSiteURLPattern(self):
-        ##  http://www.mediaminer.org/fanfic/view_st.php/76882
-        ##  http://www.mediaminer.org/fanfic/view_ch.php/167618/594087#fic_c
-        ##  http://www.mediaminer.org/fanfic/view_ch.php?submit=View+Chapter&id=105816&cid=357151
-        ##  http://www.mediaminer.org/fanfic/view_ch.php?cid=612153&submit=View+Chapter&id=171668
-        return re.escape("http://"+self.getSiteDomain())+\
-            r"/fanfic/view_(st|ch)\.php"+\
-            r"(/(?P<id>\d+)(/\d+(#fic_c)?)?/?|"+\
-            r"\?((submit=View(\+| )Chapter|id=(?P<id2>\d+)|cid=\d+)&?)+)"
-
+        ## old urls
+        ## http://www.mediaminer.org/fanfic/view_st.php/76882
+        ## new urls
+        ## http://www.mediaminer.org/fanfic/s/ghosts-from-the-past/72
+        ## http://www.mediaminer.org/fanfic/c/ghosts-from-the-past/chapter-2/72/174
+        ## http://www.mediaminer.org/fanfic/s/robtech-final-missions/61553
+        ## http://www.mediaminer.org/fanfic/c/robtech-final-missions/robotech-final-missions-oneshot/61553/189830
+        return re.escape("http://"+self.getSiteDomain())+r"/fanfic/"+\
+            r"((s/(?P<urltitle1>[^/]+)/(?P<id1>\d+))|"+\
+            r"((c/(?P<urltitle2>[^/]+)/[^/]+/(?P<id2>\d+))/\d+)|"+\
+            r"(view_st\.php/(?P<id3>\d+)))"
+            
     # Override stripURLParameters so the id parameter won't get stripped
     @classmethod
     def stripURLParameters(cls, url):
@@ -84,9 +98,10 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
         logger.debug("URL: "+url)
 
         try:
-            data = self._fetchUrl(url+'/') # trailing / gets 'chapter list' page even for one-shots.
+            data = self._fetchUrl(url) # w/o trailing / gets 'chapter list' page even for one-shots.
         except urllib2.HTTPError, e:
             if e.code == 404:
+                logger.error("404 on %s"%url)
                 raise exceptions.StoryDoesNotExist(self.url)
             else:
                 raise e
@@ -96,11 +111,13 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
 
         # [ A - All Readers ], strip '[' ']'
         ## Above title because we remove the smtxt font to get title.
-        smtxt = soup.find("font",{"class":"smtxt"})
+        smtxt = soup.find("h3",{"id":"post-rating"})
         if not smtxt:
+            logger.error("can't find rating")
             raise exceptions.StoryDoesNotExist(self.url)
-        rating = smtxt.string[1:-1]
-        self.story.setMetadata('rating',rating)
+        else:
+            rating = smtxt.string[1:-1]
+            self.story.setMetadata('rating',rating)
 
         # Find authorid and URL from... author url.
         a = soup.find('a', href=re.compile(r"/fanfic/src.php/u/\d+"))
@@ -116,37 +133,24 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
         ## <td class="ffh">The Kraut, The Bartender, and The Drunkard: Chapter 1</b> <font class="smtxt">[ P - Pre-Teen ]</font></td>
         ## <td class="ffh">Betrayal and Justice: A Cold Heart</b> <font size="-1">( Chapter 1 )</font> <font class="smtxt">[ A - All Readers ]</font></td>
         ## <td class="ffh">Question and Answer: Question and Answer</b> <font size="-1">( One-Shot )</font> <font class="smtxt">[ A - All Readers ]</font></td>
-        title = soup.find('td',{'class':'ffh'})
-        for font in title.findAll('font'):
-            font.extract() # removes 'font' tags from inside the td.        
-        if title.has_attr('colspan'):
-            titlet = stripHTML(title)
-        else:
-            ## No colspan, it's part chapter title--even if it's a one-shot.
-            titlet = ':'.join(stripHTML(title).split(':')[:-1]) # strip trailing 'Chapter X' or chapter title
-        self.story.setMetadata('title',titlet)
+        # title = soup.find('td',{'class':'ffh'})
+        # for font in title.findAll('font'):
+        #     font.extract() # removes 'font' tags from inside the td.        
+        # if title.has_attr('colspan'):
+        #     titlet = stripHTML(title)
+        # else:
+        #     ## No colspan, it's part chapter title--even if it's a one-shot.
+        #     titlet = ':'.join(stripHTML(title).split(':')[:-1]) # strip trailing 'Chapter X' or chapter title
+        self.story.setMetadata('title',stripHTML(soup.find('h1',{'id':'post-title'})))
 
         # save date from first for later.
         firstdate=None
         
-        # Find the chapters
-        select = soup.find('select',{'name':'cid'})
-        if not select:
-            self.chapterUrls.append(( self.story.getMetadata('title'),self.url))
-        else:
-            for option in select.findAll("option"):
-                chapter = stripHTML(option.string)
-                ## chapter can be: Chapter 7 [Jan 23, 2011]
-                ##             or: Vigilant Moonlight ( Chapter 1 ) [Jan 30, 2004]
-                ##        or even: Prologue ( Prologue ) [Jul 31, 2010]
-                m = re.match(r'^(.*?) (\( .*? \) )?\[(.*?)\]$',chapter)
-                chapter = m.group(1)
-                # save date from first for later.
-                if not firstdate:
-                    firstdate = m.group(3)
-                # http://www.mediaminer.org/fanfic/view_ch.php?cid=376587&submit=View+Chapter&id=105816
-                # self.chapterUrls.append((chapter,'http://'+self.host+'/fanfic/view_ch.php/'+self.story.getMetadata('storyId')+'/'+option['value']))
-                self.chapterUrls.append((chapter,'http://'+self.host+'/fanfic/view_ch.php?submit=View Chapter&id='+self.story.getMetadata('storyId')+'&cid='+option['value']))
+        # Find the chapters - one-shot now have chapter list, too.
+        chap_p = soup.find('p',{'style':'margin-left:10px;'})
+        for (atag,aurl,name) in [ (x,x['href'],stripHTML(x)) for x in chap_p.find_all('a') ]:
+            self.chapterUrls.append((name,'http://'+self.host+aurl))
+            
         self.story.setMetadata('numChapters',len(self.chapterUrls))
 
         # category
@@ -155,27 +159,20 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
             self.story.addToList('category',a.string)
         
         # genre
-        # <a href="/fanfic/src.php/a/567">Ranma 1/2</a>
+        # <a href="/fanfic/src.php/g/567">Ranma 1/2</a>
         for a in soup.findAll('a',href=re.compile(r"^/fanfic/src.php/g/")):
             self.story.addToList('genre',a.string)
 
-        # if firstdate, then the block below will only have last updated.
-        if firstdate:
-            self.story.setMetadata('datePublished', makeDate(firstdate, "%b %d, %Y"))
-        # Everything else is in <tr bgcolor="#EEEED4">
-
-        metastr = stripHTML(soup.find("tr",{"bgcolor":"#EEEED4"})).replace('\n',' ').replace('\r',' ').replace('\t',' ')
-        # Latest Revision: August 03, 2010
-        m = re.match(r".*?(?:Latest Revision|Uploaded On): ([a-zA-Z]+ \d\d, \d\d\d\d)",metastr)
+        metastr = stripHTML(soup.find("div",{"class":"post-meta"}))
+        
+        # Latest Revision: February 07, 2015 15:21 PST
+        m = re.match(r".*?(?:Latest Revision|Uploaded On): ([a-zA-Z]+ \d\d, \d\d\d\d \d\d:\d\d)",metastr)
         if m:
-            self.story.setMetadata('dateUpdated', makeDate(m.group(1), "%B %d, %Y"))
-            if not firstdate:
-                self.story.setMetadata('datePublished',
-                                       self.story.getMetadataRaw('dateUpdated'))
-                
-        else:
-            self.story.setMetadata('dateUpdated',
-                                   self.story.getMetadataRaw('datePublished'))
+            self.story.setMetadata('dateUpdated', makeDate(m.group(1), self.dateformat))
+            # site doesn't give date published on index page.
+            # set to updated, change in chapters below.
+            # self.story.setMetadata('datePublished',
+            #                        self.story.getMetadataRaw('dateUpdated'))
 
         # Words: 123456
         m = re.match(r".*?\| Words: (\d+) \|",metastr)
@@ -201,43 +198,54 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
 
         logger.debug('Getting chapter text from: %s' % url)
 
-        data=self._fetchUrl(url)
+        data = self._fetchUrl(url)
         soup = self.make_soup(data)
 
-        header = soup.find('div',{'class':'post-meta clearfix '})
+        headerstr = stripHTML(soup.find('div',{'class':'post-meta clearfix '}))
         # print("data:%s"%data)
+        #header.extract()
 
-        chapter=self.make_soup('<div class="story"></div>').find('div')
-
-        if None == header:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-
-        ## find divs with align=left, those are paragraphs in newer stories.
-        divlist = header.findAllNext('div',{'align':'left'})
-        if divlist:
-            for div in divlist:
-                div.name='p' # convert to <p> mediaminer uses div with
-                             # a margin for paragraphs.
-                chapter.append(div)
-                del div['style']
-                del div['align']
-            return self.utf8FromSoup(url,chapter)
-        
-        else:
-            logger.debug('Using kludgey text find for older mediaminer story.')
-            ## Some older mediaminer stories are unparsable with BeautifulSoup.
-            ## Really nasty formatting.  Sooo... Cheat!  Parse it ourselves a bit first.
-            ## Story stuff falls between:
-            data = "<div id='HERE'>" + data[data.find('<div class="adWrap">'):data.find('<div class="addthis_sharing_toolbox">')] +"</div>"
-            soup = self.make_soup(data)
-            for tag in soup.findAll('td',{'class':'ffh'}) + \
-                    soup.findAll('div',{'class':'acl'}) + \
-                    soup.findAll('div',{'class':'adWrap'}) + \
-                    soup.findAll('div',{'class':'footer smtxt'}) + \
-                    soup.findAll('table',{'class':'tbbrdr'}):
-                tag.extract() # remove tag from soup.
+        m = re.match(r".*?Uploaded On: ([a-zA-Z]+ \d\d, \d\d\d\d \d\d:\d\d)",headerstr)
+        if m:
+            date = makeDate(m.group(1), self.dateformat)
+            if not self.story.getMetadataRaw('datePublished') or date < self.story.getMetadataRaw('datePublished'):
+                self.story.setMetadata('datePublished', date)
                 
-            return self.utf8FromSoup(url,soup)
+        chapter = soup.find('div',{'id':'fanfic-text'})
+        
+        return self.utf8FromSoup(url,chapter)
+    
+        # chapter=self.make_soup('<div class="story"></div>').find('div')
+
+        # if None == header:
+        #     raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+
+        # ## find divs with align=left, those are paragraphs in newer stories.
+        # divlist = header.findAllNext('div',{'align':'left'})
+        # if divlist:
+        #     for div in divlist:
+        #         div.name='p' # convert to <p> mediaminer uses div with
+        #                      # a margin for paragraphs.
+        #         chapter.append(div)
+        #         del div['style']
+        #         del div['align']
+        #     return self.utf8FromSoup(url,chapter)
+        
+        # else:
+        #     logger.debug('Using kludgey text find for older mediaminer story.')
+        #     ## Some older mediaminer stories are unparsable with BeautifulSoup.
+        #     ## Really nasty formatting.  Sooo... Cheat!  Parse it ourselves a bit first.
+        #     ## Story stuff falls between:
+        #     data = "<div id='HERE'>" + data[data.find('<div class="adWrap">'):data.find('<div class="addthis_sharing_toolbox">')] +"</div>"
+        #     soup = self.make_soup(data)
+        #     for tag in soup.findAll('td',{'class':'ffh'}) + \
+        #             soup.findAll('div',{'class':'acl'}) + \
+        #             soup.findAll('div',{'class':'adWrap'}) + \
+        #             soup.findAll('div',{'class':'footer smtxt'}) + \
+        #             soup.findAll('table',{'class':'tbbrdr'}):
+        #         tag.extract() # remove tag from soup.
+                
+        #     return self.utf8FromSoup(url,soup)
         
 
 def getClass():

fanficfare/adapters/adapter_ninelivesdarksolaceorg.py

@@ -0,0 +1,38 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2015 Fanficdownloader team, 2015 FanFicFare team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Software: eFiction
+import re
+from base_efiction_adapter import BaseEfictionAdapter
+
+class NineLivesDarkSolaceAdapter(BaseEfictionAdapter):
+
+    @staticmethod
+    def getSiteDomain():
+        return 'ninelives.dark-solace.org'
+
+    @classmethod
+    def getSiteAbbrev(self):
+        return '9lvs'
+        
+    @classmethod
+    def getDateFormat(self):
+        return "%B %d, %Y" 
+                           
+def getClass():
+    return NineLivesDarkSolaceAdapter
+    

fanficfare/adapters/adapter_portkeyorg.py

@@ -258,6 +258,7 @@ class PortkeyOrgAdapter(BaseSiteAdapter): # XXX
         data = self._fetchUrl(url)
 
         data = data.replace("HTML>","div>")
+        data = data.replace("html>","div>")
         
         soup = bs.BeautifulSoup(data)
 

fanficfare/adapters/adapter_questionablequestingcom.py

@@ -0,0 +1,48 @@
+#  -*- coding: utf-8 -*-
+
+# Copyright 2015 FanFicFare team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import re
+
+from base_xenforoforum_adapter import BaseXenForoForumAdapter
+
+def getClass():
+    return QuestionablequestingComAdapter
+
+class QuestionablequestingComAdapter(BaseXenForoForumAdapter):
+
+    def __init__(self, config, url):
+        BaseXenForoForumAdapter.__init__(self, config, url)
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','qq')
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'questionablequesting.com'
+
+    @classmethod
+    def getAcceptDomains(cls):
+        return ["forum."+cls.getSiteDomain(),cls.getSiteDomain()]
+
+    @classmethod
+    def getConfigSections(cls):
+        "Only needs to be overriden if has additional ini sections."
+        return ['base_xenforoforum','forum.'+cls.getSiteDomain(),cls.getSiteDomain()]
+    
+    def getSiteURLPattern(self):
+        return r'https?://(forum.)?'+re.escape(self.getSiteDomain())+r'/(?P<tp>threads|posts)/(.+\.)?(?P<id>\d+)/?'
+        

fanficfare/adapters/adapter_quotevcom.py

@@ -0,0 +1,108 @@
+import re
+import urlparse
+import urllib2
+import datetime
+
+from .. import exceptions
+from base_adapter import BaseSiteAdapter
+
+SITE_DOMAIN = 'quotev.com'
+STORY_URL_TEMPLATE = 'http://www.quotev.com/story/%s'
+
+
+def getClass():
+    return QuotevComAdapter
+
+
+def get_url_path_segments(url):
+    return tuple(filter(None, url.split('/')[3:]))
+
+
+class QuotevComAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        story_id = get_url_path_segments(url)[1]
+        self._setURL(STORY_URL_TEMPLATE % story_id)
+        self.story.setMetadata('storyId', story_id)
+        self.story.setMetadata('siteabbrev', SITE_DOMAIN)
+
+    @staticmethod
+    def getSiteDomain():
+        return SITE_DOMAIN
+
+    @classmethod
+    def getSiteExampleURLs(cls):
+        return STORY_URL_TEMPLATE % '1234'
+
+    def getSiteURLPattern(self):
+        pattern = re.escape(STORY_URL_TEMPLATE.rsplit('%', 1)[0]) + r'(.+?)($|&|/)'
+        pattern = pattern.replace(r'http\:', r'https?\:')
+        pattern = pattern.replace(r'https?\:\/\/www\.', r'https?\:\/\/(www\.)?')
+        return pattern
+
+    def use_pagecache(self):
+        return True
+
+    def extractChapterUrlsAndMetadata(self):
+        try:
+            data = self._fetchUrl(self.url)
+        except urllib2.HTTPError:
+            raise exceptions.FailedToDownload(self.url)
+
+        soup = self.make_soup(data)
+
+        element = soup.find('div', {'class': 'result_head'})
+        if not element:
+            raise exceptions.StoryDoesNotExist(self.url)
+
+        self.story.setMetadata('title', element.find('span', recursive=False).get_text())
+
+        element = soup.find('div', {'class': 'desc_creator'})('a')[1]
+        self.story.setMetadata('author', element.get_text())
+        self.story.setMetadata('authorId', get_url_path_segments(element['href'])[0])
+        self.story.setMetadata('authorUrl', urlparse.urljoin(self.url, element['href']))
+        self.setDescription(self.url, soup.find('div', id='qdesct'))
+        self.setCoverImage(self.url, urlparse.urljoin(self.url, soup.find('img', {'class': 'logo'})['src']))
+
+        for a in soup.find('div', {'class': 'tag'})('a'):
+            if a['href'] == '#':
+                continue
+
+            self.story.addToList('category', a.get_text())
+
+        elements = soup('span', {'class': 'q_time'})
+        self.story.setMetadata('datePublished', datetime.datetime.fromtimestamp(float(elements[0]['ts'])))
+        self.story.setMetadata('dateUpdated', datetime.datetime.fromtimestamp(float(elements[1]['ts'])))
+
+        for a in soup.find('div', id='rselect')('a'):
+            self.chapterUrls.append((a.get_text(), urlparse.urljoin(self.url, a['href'])))
+
+        self.story.setMetadata('numChapters', len(self.chapterUrls))
+
+        element = soup.find('div', {'class': 't'})('div', recursive=False)[1].div
+        data = filter(None, (x.strip() for x in element('span')[1].next_sibling.split(u'\xb7')))
+
+        if 'completed' in data:
+            self.story.setMetadata('status', 'Completed')
+            data.remove('completed')
+        else:
+            self.story.setMetadata('status', 'In-Progress')
+
+        for datum in data:
+            value, key = datum.split()
+            self.story.setMetadata(key, value.replace(',', '').replace('.', ''))
+
+        self.story.setMetadata('favorites', soup.find('div', id='favqn').get_text())
+        self.story.setMetadata('comments', soup.find('a', id='comment_btn').span.get_text())
+
+    def getChapterText(self, url):
+        data = self._fetchUrl(url)
+        soup = self.make_soup(data)
+
+        element = soup.find('div', id='restxt')
+        for a in element('a'):
+            a.unwrap()
+
+        return self.utf8FromSoup(url, element)

fanficfare/adapters/adapter_spikeluvercom.py

@@ -106,7 +106,9 @@ class SpikeluverComAdapter(BaseSiteAdapter):
 
         listbox_tag = soup.find('div', {'class': 'listbox'})
         for span_tag in listbox_tag('span'):
-            key = span_tag.string.strip(' :')
+            key = span_tag.string
+            if key:
+                key = key.strip(' :')
             try:
                 value = stripHTML(span_tag.nextSibling)
             # This can happen with some fancy markup in the summary. Just
@@ -135,8 +137,10 @@ class SpikeluverComAdapter(BaseSiteAdapter):
                         contents.append(sibling)
 
                 # Remove the preceding break line tag and other crud
-                contents.pop()
-                contents.pop()
+                if contents:
+                    contents.pop()
+                if contents:
+                    contents.pop()
                 self.story.setMetadata('description', ''.join(contents))
 
             elif key == 'Rated':

fanficfare/adapters/adapter_test1.py

@@ -81,6 +81,12 @@ class TestSiteAdapter(BaseSiteAdapter):
             
             return
 
+        if idnum >= 700 and idnum <= 710:
+            self._setURL('http://test1.com?sid=%s'%(idnum+100))
+            self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+            idstr = self.story.getMetadata('storyId')
+            idnum = int(idstr)
+
         if idstr == '665' and not (self.is_adult or self.getConfig("is_adult")):
             logger.warn("self.is_adult:%s"%self.is_adult)
             raise exceptions.AdultCheckRequired(self.url)

fanficfare/adapters/base_adapter.py

@@ -104,6 +104,8 @@ class BaseSiteAdapter(Configurable):
         self.chapterFirst = None
         self.chapterLast = None
         self.oldchapters = None
+        self.oldchaptersmap = None
+        self.oldchaptersdata = None
         self.oldimgs = None
         self.oldcover = None # (data of existing cover html, data of existing cover image)
         self.calibrebookmark = None
@@ -140,28 +142,6 @@ class BaseSiteAdapter(Configurable):
         '''
         self.get_cookiejar().load(filename, ignore_discard=True, ignore_expires=True)
         
-    # def save_cookiejar(self,filename):
-    #     '''
-    #     Assumed to be a FileCookieJar if self.cookiejar set.
-    #     Takes file *name*.
-    #     '''
-    #     self.get_cookiejar().save(filename, ignore_discard=True, ignore_expires=True)
-
-    # def save_pagecache(self,filename):
-    #     '''
-    #     Writes pickle of pagecache to file *name*
-    #     '''
-    #     with open(filename, 'wb') as f:
-    #         pickle.dump(self.get_pagecache(),
-    #                     f,protocol=pickle.HIGHEST_PROTOCOL)
-        
-    # def load_pagecache(self,filename):
-    #     '''
-    #     Reads pickle of pagecache from file *name*
-    #     '''
-    #     with open(filename, 'rb') as f:
-    #         self.set_pagecache(pickle.load(f))
-
     def get_pagecache(self):
         return self.pagecache
     
@@ -185,9 +165,9 @@ class BaseSiteAdapter(Configurable):
         else:
             return None
 
-    def _set_to_pagecache(self,cachekey,data):
+    def _set_to_pagecache(self,cachekey,data,redirectedurl):
         if self.use_pagecache():
-            self.get_pagecache()[cachekey] = data
+            self.get_pagecache()[cachekey] = (data,redirectedurl)
 
     def use_pagecache(self):
         '''
@@ -257,7 +237,8 @@ class BaseSiteAdapter(Configurable):
         cachekey=self._get_cachekey(url, parameters, headers)
         if usecache and self._has_cachekey(cachekey):
             logger.debug("#####################################\npagecache HIT: %s"%cachekey)
-            return self._get_from_pagecache(cachekey)
+            data,redirecturl = self._get_from_pagecache(cachekey)
+            return data
         
         logger.debug("#####################################\npagecache MISS: %s"%cachekey)
         self.do_sleep(extrasleep)
@@ -272,13 +253,23 @@ class BaseSiteAdapter(Configurable):
                          data=urllib.urlencode(parameters),
                          headers=headers)
         data = self._decode(self.opener.open(req,None,float(self.getConfig('connect_timeout',30.0))).read())
-        self._set_to_pagecache(cachekey,data)
+        self._set_to_pagecache(cachekey,data,url)
         return data
 
     def _fetchUrlRaw(self, url,
                      parameters=None,
                      extrasleep=None,
                      usecache=True):
+        
+        return self._fetchUrlRawOpened(url,
+                                       parameters,
+                                       extrasleep,
+                                       usecache)[0]
+        
+    def _fetchUrlRawOpened(self, url,
+                           parameters=None,
+                           extrasleep=None,
+                           usecache=True):
         '''
         When should cache be cleared or not used? logins...
         
@@ -289,16 +280,25 @@ class BaseSiteAdapter(Configurable):
         cachekey=self._get_cachekey(url, parameters)
         if usecache and self._has_cachekey(cachekey):
             logger.debug("#####################################\npagecache HIT: %s"%cachekey)
-            return self._get_from_pagecache(cachekey)
+            data,redirecturl = self._get_from_pagecache(cachekey)
+            class FakeOpened:
+                def __init__(self,data,url):
+                    self.data=data
+                    self.url=url
+                def geturl(self): return self.url
+                def read(self): return self.data
+            return (data,FakeOpened(data,redirecturl))
         
         logger.debug("#####################################\npagecache MISS: %s"%cachekey)
         self.do_sleep(extrasleep)
         if parameters != None:
-            data = self.opener.open(url.replace(' ','%20'),urllib.urlencode(parameters),float(self.getConfig('connect_timeout',30.0))).read()
+            opened = self.opener.open(url.replace(' ','%20'),urllib.urlencode(parameters),float(self.getConfig('connect_timeout',30.0)))
         else:
-            data = self.opener.open(url.replace(' ','%20'),None,float(self.getConfig('connect_timeout',30.0))).read()
-        self._set_to_pagecache(cachekey,data)
-        return data
+            opened = self.opener.open(url.replace(' ','%20'),None,float(self.getConfig('connect_timeout',30.0)))
+        data = opened.read()
+        self._set_to_pagecache(cachekey,data,opened.url)
+        
+        return (data,opened)
 
     def set_sleep(self,val):
         logger.debug("\n===========\n set sleep time %s\n==========="%val)
@@ -312,20 +312,30 @@ class BaseSiteAdapter(Configurable):
         elif self.getConfig('slow_down_sleep_time'):
             time.sleep(float(self.getConfig('slow_down_sleep_time')))
         
-    # parameters is a dict()
     def _fetchUrl(self, url,
                   parameters=None,
                   usecache=True,
                   extrasleep=None):
+        return self._fetchUrlOpened(url,
+                                    parameters,
+                                    usecache,
+                                    extrasleep)[0]
+
+    # parameters is a dict()
+    def _fetchUrlOpened(self, url,
+                        parameters=None,
+                        usecache=True,
+                        extrasleep=None):
 
         excpt=None
         for sleeptime in [0, 0.5, 4, 9]:
             time.sleep(sleeptime)	
             try:
-                return self._decode(self._fetchUrlRaw(url,
+                (data,opened)=self._fetchUrlRawOpened(url,
                                                       parameters=parameters,
                                                       usecache=usecache,
-                                                      extrasleep=extrasleep))
+                                                      extrasleep=extrasleep)
+                return (self._decode(data),opened)
             except u2.HTTPError, he:
                 excpt=he
                 if he.code == 404:
@@ -345,6 +355,7 @@ class BaseSiteAdapter(Configurable):
             self.chapterFirst=int(first)-1
         if last:
             self.chapterLast=int(last)-1
+        self.story.set_chapters_range(first,last)
     
     # Does the download the first time it's called.
     def getStory(self):
@@ -352,21 +363,40 @@ class BaseSiteAdapter(Configurable):
             self.getStoryMetadataOnly(get_cover=True)
 
             for index, (title,url) in enumerate(self.chapterUrls):
+                newchap = False
                 if (self.chapterFirst!=None and index < self.chapterFirst) or \
                         (self.chapterLast!=None and index > self.chapterLast):
                     self.story.addChapter(url,
                                           removeEntities(title),
                                           None)
                 else:
-                    if self.oldchapters and index < len(self.oldchapters):
+                    data = None
+                    if self.oldchaptersmap:
+                        if url in self.oldchaptersmap:
+                            data = self.utf8FromSoup(None,
+                                                     self.oldchaptersmap[url],
+                                                     partial(cachedfetch,self._fetchUrlRaw,self.oldimgs))
+                    elif self.oldchapters and index < len(self.oldchapters):
                         data = self.utf8FromSoup(None,
                                                  self.oldchapters[index],
                                                  partial(cachedfetch,self._fetchUrlRaw,self.oldimgs))
-                    else:
+
+                    # if already marked new -- ie, origtitle and title don't match
+                    # logger.debug("self.oldchaptersdata[url]:%s"%(self.oldchaptersdata[url]))
+                    newchap = (self.oldchaptersdata is not None and
+                               url in self.oldchaptersdata and (
+                            self.oldchaptersdata[url]['chapterorigtitle'] !=
+                            self.oldchaptersdata[url]['chaptertitle']) )
+                    
+                    if not data:
                         data = self.getChapterText(url)
+                        # if had to fetch and has existing chapters
+                        newchap = bool(self.oldchapters or self.oldchaptersmap)
+                        
                     self.story.addChapter(url,
                                           removeEntities(title),
-                                          removeEntities(data))
+                                          removeEntities(data),
+                                          newchap)
             self.storyDone = True
             
             # include image, but no cover from story, add default_cover_image cover.
@@ -399,7 +429,10 @@ class BaseSiteAdapter(Configurable):
             self.doExtractChapterUrlsAndMetadata(get_cover=get_cover)
             
             if not self.story.getMetadataRaw('dateUpdated'):
-                self.story.setMetadata('dateUpdated',self.story.getMetadataRaw('datePublished'))
+                if self.story.getMetadataRaw('datePublished'):
+                    self.story.setMetadata('dateUpdated',self.story.getMetadataRaw('datePublished'))
+                else:
+                    self.story.setMetadata('dateUpdated',self.story.getMetadataRaw('dateCreated'))                
 
             self.metadataDone = True
         return self.story
@@ -409,7 +442,10 @@ class BaseSiteAdapter(Configurable):
             self.story.load_html_metadata(metahtml)
             self.metadataDone = True
             if not self.story.getMetadataRaw('dateUpdated'):
-                self.story.setMetadata('dateUpdated',self.story.getMetadataRaw('datePublished'))
+                if self.story.getMetadataRaw('datePublished'):
+                    self.story.setMetadata('dateUpdated',self.story.getMetadataRaw('datePublished'))
+                else:
+                    self.story.setMetadata('dateUpdated',self.story.getMetadataRaw('dateCreated'))                
     
     def hookForUpdates(self,chaptercount):
         "Usually not needed."
@@ -427,6 +463,11 @@ class BaseSiteAdapter(Configurable):
         "Only needs to be overriden if != site domain."
         return cls.getSiteDomain()
     
+    @classmethod
+    def getConfigSections(cls):
+        "Only needs to be overriden if has additional ini sections."
+        return [cls.getConfigSection()]
+    
     @classmethod
     def stripURLParameters(cls,url):
         "Only needs to be overriden if URL contains more than one parameter"
@@ -476,6 +517,13 @@ class BaseSiteAdapter(Configurable):
 
     def setDescription(self,url,svalue):
         #print("\n\nsvalue:\n%s\n"%svalue)
+        strval = u"%s"%svalue # works for either soup or string
+        if self.hasConfig('description_limit'):
+            limit = int(self.getConfig('description_limit'))
+            if limit and len(strval) > limit:
+                svalue = strval[:limit]
+
+        #print(u"[[[[[\n\n%s\n\n]]]]]]]]"%svalue) # works for either soup or string
         if self.getConfig('keep_summary_html'):
             if isinstance(svalue,basestring):
                 # bs4/html5lib add html, header and body tags, which
@@ -552,7 +600,8 @@ class BaseSiteAdapter(Configurable):
                     if t.name not in ('p') and t.string != None and len(t.string.strip()) == 0 :
                         t.extract()
         except AttributeError, ae:
-            logger.error("Error parsing HTML, probably poor input HTML. %s"%ae)
+            if "%s"%ae != "'NoneType' object has no attribute 'next_element'":
+                logger.error("Error parsing HTML, probably poor input HTML. %s"%ae)
         
         retval = unicode(soup)
 

fanficfare/adapters/base_efiction_adapter.py

@@ -70,6 +70,11 @@ class BaseEfictionAdapter(BaseSiteAdapter):
         self.triedAcceptWarnings = False
         self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
 
+    @classmethod
+    def getConfigSections(cls):
+        "Only needs to be overriden if has additional ini sections."
+        return ['base_efiction',cls.getConfigSection()]
+    
     @classmethod
     def getAcceptDomains(cls):
         return [cls.getSiteDomain(),'www.' + cls.getSiteDomain()]
@@ -143,7 +148,7 @@ class BaseEfictionAdapter(BaseSiteAdapter):
         """
         Get the URL to a user page on this site.
         """
-        return "%s?sid=%s" % (self.getUrlForPhp(self.getViewUserPhpName()), userId)
+        return "%s?uid=%s" % (self.getUrlForPhp(self.getViewUserPhpName()), userId)
 
     @classmethod
     def getLoginUrl(self):

fanficfare/adapters/base_xenforoforum_adapter.py

@@ -0,0 +1,244 @@
+#  -*- coding: utf-8 -*-
+
+# Copyright 2015 FanFicFare team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+logger = logging.getLogger(__name__)
+
+class BaseXenForoForumAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["utf8",
+                       "Windows-1252"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+							   
+							   
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])        
+        
+        # get storyId from url--url validation guarantees query correct
+        m = re.match(self.getSiteURLPattern(),url)
+        if m:
+            self.story.setMetadata('storyId',m.group('id'))
+
+            # normalized story URL.
+            self._setURL(self.getURLPrefix() + '/'+m.group('tp')+'/'+self.story.getMetadata('storyId')+'/')
+        else:
+            raise exceptions.InvalidStoryURL(url,
+                                             self.getSiteDomain(),
+                                             self.getSiteExampleURLs())
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','fsb')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%b %d, %Y at %I:%M %p"
+            
+    @classmethod
+    def getConfigSections(cls):
+        "Only needs to be overriden if has additional ini sections."
+        return ['base_xenforoforum',cls.getConfigSection()]
+    
+    @classmethod
+    def getURLPrefix(cls):
+        # The site domain.  Does have www here, if it uses it.
+        return 'https://' + cls.getSiteDomain() 
+
+    @classmethod
+    def getSiteExampleURLs(cls):
+        return cls.getURLPrefix()+"/threads/some-story-name.123456/"
+
+    def getSiteURLPattern(self):
+        return r"https?://"+re.escape(self.getSiteDomain())+r"/(?P<tp>threads|posts)/(.+\.)?(?P<id>\d+)/?"
+        
+    def use_pagecache(self):
+        '''
+        adapters that will work with the page cache need to implement
+        this and change it to True.
+        '''
+        return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        useurl = self.url
+        logger.info("url: "+useurl)
+
+        try:
+            (data,opened) = self._fetchUrlOpened(useurl)
+            useurl = opened.geturl()
+            logger.info("use useurl: "+useurl)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = self.make_soup(data)
+
+        a = soup.find('h3',{'class':'userText'}).find('a')
+        self.story.addToList('authorId',a['href'].split('/')[1])
+        self.story.addToList('authorUrl',self.getURLPrefix()+'/'+a['href'])
+        self.story.addToList('author',a.text)
+
+        h1 = soup.find('div',{'class':'titleBar'}).h1
+        self.story.setMetadata('title',stripHTML(h1))
+        
+        if '#' in useurl:
+            anchorid = useurl.split('#')[1]
+            soup = soup.find('li',id=anchorid)
+        else:
+            # try threadmarks if no '#' in , require at least 2.
+            threadmarksa = soup.find('a',{'class':'threadmarksTrigger'})
+            if threadmarksa:
+                soupmarks = self.make_soup(self._fetchUrl(self.getURLPrefix()+'/'+threadmarksa['href']))
+                markas = soupmarks.find('ol',{'class':'overlayScroll'}).find_all('a')
+                if len(markas) > 1:
+                    for (atag,url,name) in [ (x,x['href'],stripHTML(x)) for x in markas ]:
+                        date = self.make_date(atag.find_next_sibling('div',{'class':'extra'}))
+                        if not self.story.getMetadataRaw('datePublished') or date < self.story.getMetadataRaw('datePublished'):
+                            self.story.setMetadata('datePublished', date)
+                        if not self.story.getMetadataRaw('dateUpdated') or date > self.story.getMetadataRaw('dateUpdated'):
+                            self.story.setMetadata('dateUpdated', date)
+                            
+                        self.chapterUrls.append((name,self.getURLPrefix()+'/'+url))
+
+                    ## only use tags if threadmarks for chapters.
+                    ##  a bit arbitrary, but likely.
+                    for tag in soup.findAll('a',{'class':'tag'}):
+                        self.story.addToList('forumtags',stripHTML(tag))
+
+            soup = soup.find('li',{'class':'message'}) # limit first post for date stuff below. ('#' posts above)
+                
+        # Now go hunting for the 'chapter list'.
+        bq = soup.find('blockquote') # assume first posting contains TOC urls.
+        
+        bq.name='div'
+
+        for iframe in bq.find_all('iframe'):
+            iframe.extract() # calibre book reader & editor don't like iframes to youtube.
+
+        for qdiv in bq.find_all('div',{'class':'quoteExpand'}):
+            qdiv.extract() # Remove <div class="quoteExpand">click to expand</div>
+            
+        self.setDescription(useurl,bq)
+
+        # otherwise, use first post links--include first post since
+        # that's often also the first chapter.
+        if not self.chapterUrls:
+            self.chapterUrls.append(("First Post",useurl))
+            for (url,name) in [ (x['href'],stripHTML(x)) for x in bq.find_all('a') ]:
+                logger.debug("found chapurl:%s"%url)
+                if not url.startswith('http'):
+                    url = self.getURLPrefix()+'/'+url
+    
+                if ( url.startswith(self.getURLPrefix()) or
+                     url.startswith('http://'+self.getSiteDomain()) or
+                     url.startswith('https://'+self.getSiteDomain()) ) and ('/posts/' in url or '/threads/' in url):
+                    # brute force way to deal with SB's http->https change when hardcoded http urls.
+                    url = url.replace('http://'+self.getSiteDomain(),self.getURLPrefix())
+                    logger.debug("used chapurl:%s"%(url))
+                    self.chapterUrls.append((name,url))
+                    if url == useurl and 'First Post' == self.chapterUrls[0][0]:
+                        # remove "First Post" if included in list.
+                        logger.debug("delete dup 'First Post' chapter: %s %s"%self.chapterUrls[0])
+                        del self.chapterUrls[0]
+                        
+            # Didn't use threadmarks, so take created/updated dates
+            # from the 'first' posting created and updated.
+            date = self.make_date(soup.find('a',{'class':'datePermalink'}))
+            if date:
+                self.story.setMetadata('datePublished', date)
+                self.story.setMetadata('dateUpdated', date) # updated overwritten below if found.
+        
+            date = self.make_date(soup.find('div',{'class':'editDate'}))
+            if date:
+                self.story.setMetadata('dateUpdated', date) 
+            
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+    def make_date(self,parenttag): # forums use a BS thing where dates
+                                  # can appear different if recent.
+        datestr=None
+        try:
+            datetag = parenttag.find('span',{'class':'DateTime'})
+            if datetag:
+                datestr = datetag['title']
+            else:
+                datetag = parenttag.find('abbr',{'class':'DateTime'})
+                if datetag:
+                    datestr="%s at %s"%(datetag['data-datestring'],datetag['data-timestring'])
+            # Apr 24, 2015 at 4:39 AM
+            # May 1, 2015 at 5:47 AM
+            datestr = re.sub(r' (\d[^\d])',r' 0\1',datestr) # add leading 0 for single digit day & hours.
+            return makeDate(datestr, self.dateformat)
+        except:
+            logger.debug('No date found in %s'%parenttag)
+            return None
+        
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+        logger.debug('Getting chapter text from: %s' % url)
+
+        ## there's some history of stories with links to the wrong
+        ## page.  This changes page#post URLs to perma-link URLs.
+        ## Which will be redirected back to page#posts, but the
+        ## *correct* ones.
+        # http://forums.sufficientvelocity.com/threads/harry-potter-and-the-not-fatal-at-all-cultural-exchange-program.330/page-4#post-39915
+        # https://forums.sufficientvelocity.com/posts/39915/
+        if '#post-' in url:
+            url = self.getURLPrefix()+'/posts/'+url.split('#post-')[1]+'/'
+        
+        origurl = url
+        (data,opened) = self._fetchUrlOpened(url)
+        url = opened.geturl()
+        if '#' in origurl and '#' not in url:
+            url = url + origurl[origurl.index('#'):]
+        logger.debug("chapter URL redirected to: %s"%url)
+
+        soup = self.make_soup(data)
+
+        if '#' in url:
+            anchorid = url.split('#')[1]
+            soup = soup.find('li',id=anchorid)
+                
+        bq = soup.find('blockquote')
+
+        bq.name='div'
+
+        for iframe in bq.find_all('iframe'):
+            iframe.extract() # calibre book reader & editor don't like iframes to youtube.
+
+        for qdiv in bq.find_all('div',{'class':'quoteExpand'}):
+            qdiv.extract() # Remove <div class="quoteExpand">click to expand</div>
+
+        return self.utf8FromSoup(url,bq)

fanficfare/cli.py

@@ -16,7 +16,7 @@
 #
 
 from optparse import OptionParser
-from os.path import expanduser, isfile, join, dirname
+from os.path import expanduser, join, dirname
 from os import access, R_OK
 from subprocess import call
 import ConfigParser
@@ -41,12 +41,14 @@ try:
     # running under calibre
     from calibre_plugins.fanfictiondownloader_plugin.fanficfare import adapters, writers, exceptions
     from calibre_plugins.fanfictiondownloader_plugin.fanficfare.configurable import Configuration
-    from calibre_plugins.fanfictiondownloader_plugin.fanficfare.epubutils import get_dcsource_chaptercount, get_update_data
+    from calibre_plugins.fanfictiondownloader_plugin.fanficfare.epubutils import (
+        get_dcsource_chaptercount, get_update_data, reset_orig_chapters_epub)
     from calibre_plugins.fanfictiondownloader_plugin.fanficfare.geturls import get_urls_from_page
 except ImportError:
     from fanficfare import adapters, writers, exceptions
     from fanficfare.configurable import Configuration
-    from fanficfare.epubutils import get_dcsource_chaptercount, get_update_data
+    from fanficfare.epubutils import (
+        get_dcsource_chaptercount, get_update_data, reset_orig_chapters_epub)
     from fanficfare.geturls import get_urls_from_page
 
 
@@ -87,6 +89,9 @@ def main(argv=None, parser=None, passed_defaultsini=None, passed_personalini=Non
     parser.add_option('-u', '--update-epub',
                       action='store_true', dest='update',
                       help='Update an existing epub with new chapters, give epub filename instead of storyurl.', )
+    parser.add_option('--unnew',
+                      action='store_true', dest='unnew',
+                      help='Remove (new) chapter marks left by mark_new_chapters setting.', )
     parser.add_option('--update-cover',
                       action='store_true', dest='updatecover',
                       help='Update cover in an existing epub, otherwise existing cover (if any) is used on update.  Only valid with --update-epub.', )
@@ -129,6 +134,9 @@ def main(argv=None, parser=None, passed_defaultsini=None, passed_personalini=Non
     if options.update and options.format != 'epub':
         parser.error('-u/--update-epub only works with epub')
 
+    if options.unnew and options.format != 'epub':
+        parser.error('--unnew only works with epub')
+
     # for passing in a file list
     if options.infile:
         urls=[]
@@ -168,6 +176,12 @@ def do_download(arg,
     # Attempt to update an existing epub.
     chaptercount = None
     output_filename = None
+
+    if options.unnew:
+        # remove mark_new_chapters marks
+        reset_orig_chapters_epub(arg,arg)
+        return
+    
     if options.update:
         try:
             url, chaptercount = get_dcsource_chaptercount(arg)
@@ -184,7 +198,7 @@ def do_download(arg,
         url = arg
         
     try:
-        configuration = Configuration(adapters.getConfigSectionFor(url), options.format)
+        configuration = Configuration(adapters.getConfigSectionsFor(url), options.format)
     except exceptions.UnknownSite, e:
         if options.list or options.normalize:
             # list for page doesn't have to be a supported site.
@@ -200,25 +214,18 @@ def do_download(arg,
     if passed_defaultsini:
         configuration.readfp(passed_defaultsini)
 
-    if isfile(join(dirname(__file__), 'defaults.ini')):
-        conflist.append(join(dirname(__file__), 'defaults.ini'))
-        
-    if isfile(join(homepath, 'defaults.ini')):
-        conflist.append(join(homepath, 'defaults.ini'))
-    if isfile(join(homepath2, 'defaults.ini')):
-        conflist.append(join(homepath2, 'defaults.ini'))
-    if isfile('defaults.ini'):
-        conflist.append('defaults.ini')
+    # don't need to check existance for our selves.
+    conflist.append(join(dirname(__file__), 'defaults.ini'))
+    conflist.append(join(homepath, 'defaults.ini'))
+    conflist.append(join(homepath2, 'defaults.ini'))
+    conflist.append('defaults.ini')
 
     if passed_personalini:
         configuration.readfp(passed_personalini)
 
-    if isfile(join(homepath, 'personal.ini')):
-        conflist.append(join(homepath, 'personal.ini'))
-    if isfile(join(homepath2, 'personal.ini')):
-        conflist.append(join(homepath2, 'personal.ini'))
-    if isfile('personal.ini'):
-        conflist.append('personal.ini')
+    conflist.append(join(homepath, 'personal.ini'))
+    conflist.append(join(homepath2, 'personal.ini'))
+    conflist.append('personal.ini')
 
     if options.configfile:
         conflist.extend(options.configfile)
@@ -324,7 +331,9 @@ def do_download(arg,
                  adapter.oldimgs,
                  adapter.oldcover,
                  adapter.calibrebookmark,
-                 adapter.logfile) = (get_update_data(output_filename))[0:7]
+                 adapter.logfile,
+                 adapter.oldchaptersmap,
+                 adapter.oldchaptersdata) = (get_update_data(output_filename))[0:9]
 
                 print 'Do update - epub(%d) vs url(%d)' % (chaptercount, urlchaptercount)
 

fanficfare/configurable.py

@@ -17,6 +17,7 @@
 
 import ConfigParser, re
 import exceptions
+import codecs
 from ConfigParser import DEFAULTSECT, MissingSectionHeaderError, ParsingError
 
 # All of the writers(epub,html,txt) and adapters(ffnet,twlt,etc)
@@ -77,10 +78,12 @@ formatsections = ['html','txt','epub','mobi']
 othersections = ['defaults','overrides']
 
 def get_valid_sections():
-    sites = adapters.getConfigSections()
+    sites = adapters.getConfigSections() 
     sitesections = list(othersections)
     for section in sites:
         sitesections.append(section)
+        # also allows [www.base_efiction] and [www.base_forum]. Not
+        # likely to matter.
         if section.startswith('www.'):
             # add w/o www if has www
             sitesections.append(section[4:])
@@ -130,6 +133,7 @@ def get_valid_set_options():
                'replace_hr':(None,None,boollist),
                'sort_ships':(None,None,boollist),
                'strip_chapter_numbers':(None,None,boollist),
+               'mark_new_chapters':(None,None,boollist),
                'titlepage_use_table':(None,None,boollist),
                
                'use_ssl_unverified_context':(None,None,boollist),
@@ -202,6 +206,7 @@ def get_valid_keywords():
     return list(['(in|ex)clude_metadata_(pre|post)',
                  'add_chapter_numbers',
                  'add_genre_when_multi_category',
+                 'adult_ratings',
                  'allow_unsafe_filename',
                  'always_overwrite',
                  'anthology_tags',
@@ -210,8 +215,13 @@ def get_valid_keywords():
                  'bulk_load',
                  'chapter_end',
                  'chapter_start',
-                 'chapter_title_add_pattern',
                  'chapter_title_strip_pattern',
+                 'chapter_title_def_pattern',
+                 'chapter_title_add_pattern',
+                 'chapter_title_new_pattern',
+                 'chapter_title_addnew_pattern',
+                 'title_chapter_range_pattern',
+                 'mark_new_chapters',
                  'check_next_chapter',
                  'skip_author_cover',
                  'collect_series',
@@ -224,8 +234,10 @@ def get_valid_keywords():
                  'datePublished_format',
                  'dateUpdated_format',
                  'default_cover_image',
+                 'description_limit',
                  'do_update_hook',
                  'exclude_notes',
+                 'exclude_editor_signature',
                  'extra_logpage_entries',
                  'extra_subject_tags',
                  'extra_titlepage_entries',
@@ -277,6 +289,7 @@ def get_valid_keywords():
                  'output_filename_safepattern',
                  'password',
                  'post_process_cmd',
+                 'rating_titles',
                  'remove_transparency',
                  'replace_br_with_p',
                  'replace_hr',
@@ -331,7 +344,8 @@ def make_generate_cover_settings(param):
 
 class Configuration(ConfigParser.SafeConfigParser):
 
-    def __init__(self, site, fileform):
+    def __init__(self, sections, fileform):
+        site = sections[-1] # first section is site DN.
         ConfigParser.SafeConfigParser.__init__(self)
 
         self.linenos=dict() # key by section or section,key -> lineno
@@ -339,6 +353,11 @@ class Configuration(ConfigParser.SafeConfigParser):
         ## [injected] section has even less priority than [defaults]
         self.sectionslist = ['defaults','injected']
 
+        ## add other sections (not including site DN) after defaults,
+        ## but before site-specific.
+        for section in sections[:-1]:
+            self.addConfigSection(section)
+        
         if site.startswith("www."):
             sitewith = site
             sitewithout = site.replace("www.","")
@@ -348,8 +367,13 @@ class Configuration(ConfigParser.SafeConfigParser):
         
         self.addConfigSection(sitewith)
         self.addConfigSection(sitewithout)
+        
         if fileform:
             self.addConfigSection(fileform)
+            ## add other sections:fileform (not including site DN)
+            ## after fileform, but before site-specific:fileform.
+            for section in sections[:-1]:
+                self.addConfigSection(section+":"+fileform)
             self.addConfigSection(sitewith+":"+fileform)
             self.addConfigSection(sitewithout+":"+fileform)
         self.addConfigSection("overrides")
@@ -437,6 +461,30 @@ class Configuration(ConfigParser.SafeConfigParser):
         else:
             return self.linenos.get(section,None)
     
+    ## Copied from Python 2.7 library so as to make read utf8.
+    def read(self, filenames):
+        """Read and parse a filename or a list of filenames.
+        Files that cannot be opened are silently ignored; this is
+        designed so that you can specify a list of potential
+        configuration file locations (e.g. current directory, user's
+        home directory, systemwide directory), and all existing
+        configuration files in the list will be read.  A single
+        filename may also be given.
+        Return list of successfully read files.
+        """
+        if isinstance(filenames, basestring):
+            filenames = [filenames]
+        read_ok = []
+        for filename in filenames:
+            try:
+                fp = codecs.open(filename,encoding='utf-8')
+            except IOError:
+                continue
+            self._read(fp, filename)
+            fp.close()
+            read_ok.append(filename)
+        return read_ok
+    
     ## Copied from Python 2.7 library so as to make it save linenos too.
     #
     # Regular expressions for parsing section headers and options.

fanficfare/defaults.ini

@@ -148,8 +148,10 @@ zip_filename: ${title}-${siteabbrev}_${storyId}${formatext}.zip
 ## zip_filename.
 allow_unsafe_filename: false
 
-## The regex pattern of 'unsafe' filename chars for above.
-#output_filename_safepattern:[^a-zA-Z0-9_\. \[\]\(\)&'-]+
+## The regex pattern of 'unsafe' filename chars for above.  First
+## character . OR any one or more characters that are NOT a letter,
+## number, or one of _. []()&'-
+output_filename_safepattern:(^\.|/\.|[^a-zA-Z0-9_\. \[\]\(\)&'-]+)
 
 ## entries to make epub subjects and calibre tags
 ## lastupdate creates two tags: "Last Update Year/Month: %Y/%m" and "Last Update: %Y/%m/%d"
@@ -268,6 +270,11 @@ keep_summary_html:true
 ## kept.  Some sites(AO3) use <a title="Translation">Foreign language</a>
 #keep_title_attr: false
 
+## If a chapter range was given, use this pattern for the book title.
+## replace_metadata and include/exclude will be applied *after* this.
+## Set to empty value to disable.
+title_chapter_range_pattern:${title} (Ch ${first}-${last})
+
 ## Don't like the numbers at the start of chapter titles on some
 ## sites?  You can use strip_chapter_numbers to strip them off.  Just
 ## want to make them all look the same?  Strip them off, then add them
@@ -300,12 +307,31 @@ chapter_title_strip_pattern:^[0-9]+[\.: -]+
 ## etc
 #chapter_title_strip_pattern:^([0-9]+[\.: -]+)?(Chapter *[0-9]+[\.:, -]*)?
 
-## Uses a python template substitution.  The ${index} is the 'chapter'
-## number and ${title} is the chapter title, after applying
-## chapter_title_strip_pattern.  Those are the only variables available.
-## "The Beginning" => "1. The Beginning" 
+## If true, when updating an epub that already has old chapters, new
+## chapters will be marked in the TOC and chapter header by using
+## chapter_title_new_pattern and chapter_title_addnew_pattern to set the chapter.
+mark_new_chapters:false
+
+## chapter title patterns use python template substitution.  The
+## ${index} is the 'chapter' number and ${title} is the chapter title,
+## after applying chapter_title_strip_pattern.  Those are the only
+## variables available.
+
+## The basic pattern used when not using add_chapter_numbers or
+## mark_new_chapters
+chapter_title_def_pattern:${title}
+
+## Pattern used with add_chapter_numbers, but not mark_new_chapters
 chapter_title_add_pattern:${index}. ${title}
 
+## Pattern used with mark_new_chapters, but not add_chapter_numbers
+## (new) is just text and can be changed.
+chapter_title_new_pattern:(new) ${title}
+
+## Pattern used with add_chapter_numbers and mark_new_chapters
+## (new) is just text and can be changed.
+chapter_title_addnew_pattern:${index}. (new) ${title}
+
 ## Reorder ships so b/a and c/b/a become a/b and a/b/c. Only separates
 ## on '/', so use replace_metadata to change separator first if
 ## needed. Something like: ships=>[ ]*(/|&amp;|&)[ ]*=>/ You can use
@@ -327,11 +353,77 @@ sort_ships:false
 ## User-agent
 user_agent:FFF/2.X
 
-## Virtually all eFiction Base adapters allow downloading the whole story in
-## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
-## metadata and chapters can be loaded in one step
+## Added for [base_xenforoforum], but can be used with other sites,
+## too.  Limit the 'description' to the first X *characters*
+## collected.  Character count includes HTML tags, so it can be
+## non-intuitive.
+#description_limit:1000
+
+[base_efiction]
+## At the time of writing, eFiction Base adapters allow downloading
+## the whole story in bulk using the 'Print' feature. If 'bulk_load'
+## is set to 'true', both metadata and chapters can be loaded in one
+## step
 bulk_load:true
 
+[base_xenforoforum]
+## Currently only forums.spacebattles.com and forums.sufficientvelocity.com
+
+cover_exclusion_regexp:/clear.png
+
+## I saw lots of chapters name simply '1.1' etc during testing.
+strip_chapter_numbers:false
+
+## Copy title to tagsfromtitle for parsing tags.
+add_to_extra_valid_entries:,tagsfromtitle,forumtags
+
+## '.NOREPL' tells the system to *not* apply title's
+## in/exclude/replace_metadata -- Only works on include_in_ lines.
+include_in_tagsfromtitle:title.NOREPL
+
+tagsfromtitle_label:Tags from Title
+forumtags_label:Tags from Forum
+
+## might want to do this, maybe not.  Will often include category, but
+## also often include non-category stuff.
+# include_in_category:tagsfromtitle 
+
+add_to_include_metadata_pre:
+# only keep tagsfromtitle with ( or [ in.
+ tagsfromtitle=~[\[\(]
+ 
+add_to_replace_metadata:
+# remove anything outside () or []
+ tagsfromtitle=>^.*?([\(\[]([^\]\)]+)[\)\]]).*?$=>\1
+# remove () []
+ tagsfromtitle=>[\(\)\[\]]=>
+# change (spaces)slash(or semicolon)(spaces) to comma
+ tagsfromtitle=> *[/;] *=>,
+ tagsfromtitle=> [xX] =>,
+ 
+# remove [] or () blocks and leading/trailing spaces/dashes/colons
+ title=>[-: ]*[\(\[]([^\]\)]+)[\)\]][-: ]*=>
+# remove 'Thread' and the next word, usually "Thread 2", "Thread
+# four", "Thread iv", etc
+ title,tagsfromtitle=>[-: ]*[Tt]hread [^ ]+[-: ]*=>
+
+add_to_extra_titlepage_entries:,tagsfromtitle,forumtags
+
+## '.SPLIT' tells the system to split by ','
+add_to_include_subject_tags:,tagsfromtitle.SPLIT
+## Or to also include forumtags add this to personal.ini:
+#add_to_include_subject_tags:,forumtags
+
+## base_xenforoforum reads Published and Updated datetimes from
+## Threadmarks if used, or from the posted & updated times of the
+## 'first' post if no threadmarks.
+datePublished_format:%%Y-%%m-%%d %%H:%%M:%%S
+dateUpdated_format:%%Y-%%m-%%d %%H:%%M:%%S
+
+## Only take the first X characters of the 'first' post to use as
+## the description.
+description_limit:500
+
 ## Each output format has a section that overrides [defaults]
 [html]
 
@@ -587,6 +679,8 @@ extratags: FanFiction,Testing,HTML
 ## doesn't like that.  If do_update_hook is uncommented and set true,
 ## the adapter will discard all existing chapters from the newest one
 ## on when updating to enforce accurate chapters.
+## Starting July 2015, FFF stores chapter URLs in the chapter files.
+## Stories downloaded after that shouldn't need this setting anymore.
 #do_update_hook:false
 
 ## AO3 adapter defines a few extra metadata entries.
@@ -1026,6 +1120,12 @@ extra_valid_entries:size
 # don't show twitter icon.
 cover_exclusion_regexp:/res/css/bir.png
 
+[forums.spacebattles.com]
+## see [base_xenforoforum]
+
+[forums.sufficientvelocity.com]
+## see [base_xenforoforum]
+
 [grangerenchanted.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -1156,6 +1256,12 @@ extracategories:NCIS
 extracategories:Buffy: The Vampire Slayer
 extracharacters:Willow
 
+[ninelives.dark-solace.org]
+## Site dedicated to these categories/characters/ships
+extracategories:The Walking Dead
+extracharacters:Carol,Daryl
+extraships:Carol/Daryl
+
 [nocturnal-light.net]
 ## Extra metadata that this adapter knows about.  See [dramione.org]
 ## for examples of how to use them.
@@ -1727,7 +1833,38 @@ extraships:InuYasha/Kagome
 ## Site dedicated to these categories/characters/ships
 extracategories:Lord of the Rings
 
+[www.masseffect2.in]
+## Site dedicated to this fandom.
+extracategories: Mass Effect
+
+## Ucoz.com engine, upon which MassEffect2.in is based, imposes an unspecified limit on request frequency.
+## Reports vary from `5 requests per second' to `2 requests per second for more than 10 per minute'.
+## With default settings, a several-hours IP ban may follow, so set it higher.
+slow_down_sleep_time: 2
+
+## Whether to exclude editor signature from the bottom of chapter text.
+exclude_editor_signature: false
+
+## Stories on the site almost never have cover image, and for the stories which do,
+## this may be adjusted in `personal.ini' before downloading.
+never_make_cover: true
+
+## Titles for ratings identified by 1- or 2-letter codes from `ERATING system'
+## (`система Р.Е.Й.Т.И.Н.Г.').  MassEffect2.in and some other sites adopted it,
+## but changed titles and update them occasionally.
+rating_titles: R=RESTRICTED (16+), E=EXEMPT (18+), I=ART HOUSE, T=To every, A=IN=Иной мир, Nn=Новый мир, G=О\, Господи!
+adult_ratings: E,R
+
 [www.mediaminer.org]
+dateUpdated_format:%%Y-%%m-%%d %%H:%%M:%%S
+## Note that mediaminer doesn't give datePublished on the story's
+## index page--it's collected from the earliest uploaded chapter.  So
+## it's not available when only fetching metadata.
+datePublished_format:%%Y-%%m-%%d %%H:%%M:%%S
+
+## some sites include images that we don't ever want becoming the
+## cover image.  This lets you exclude them.
+cover_exclusion_regexp:/img/rss.png
 
 [www.midnightwhispers.ca]
 ## Some sites do not require a login, but do require the user to
@@ -1795,6 +1932,13 @@ extracategories:Psych
 ## Site dedicated to these categories/characters/ships
 extracategories:Queer as Folk
 
+[quotev.com]
+extra_valid_entries:pages,readers,reads,favorites
+pages_label:Pages
+readers_label:Readers
+reads_label:Reads
+favorites_label:Favorites
+
 [www.restrictedsection.org]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In

fanficfare/epubutils.py

@@ -1,18 +1,17 @@
-#!/usr/bin/env python
-# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
-from __future__ import (unicode_literals, division, absolute_import,
-                        print_function)
+# -*- coding: utf-8 -*-
 
 __license__   = 'GPL v3'
-__copyright__ = '2014, Jim Miller'
+__copyright__ = '2015, Jim Miller'
 __docformat__ = 'restructuredtext en'
 
 import logging
 logger = logging.getLogger(__name__)
 
 import re, os, traceback
-from zipfile import ZipFile
+from collections import defaultdict
+from zipfile import ZipFile, ZIP_STORED, ZIP_DEFLATED
 from xml.dom.minidom import parseString
+from StringIO import StringIO
 
 import bs4 as bs
 
@@ -42,7 +41,7 @@ def get_update_data(inputio,
 
     ## Save the path to the .opf file--hrefs inside it are relative to it.
     relpath = get_path_part(rootfilename)
-            
+
     oldcover = None
     calibrebookmark = None
     logfile = None
@@ -90,7 +89,9 @@ def get_update_data(inputio,
 
     filecount = 0
     soups = [] # list of xhmtl blocks
+    urlsoups = {} # map of xhtml blocks by url
     images = {} # dict() longdesc->data
+    datamaps = defaultdict(dict) # map of data maps by url
     if getfilecount:
         # spin through the manifest--only place there are item tags.
         for item in contentdom.getElementsByTagName("item"):
@@ -124,31 +125,57 @@ def get_update_data(inputio,
                                 logger.warn("Image %s not found!\n(originally:%s)"%(newsrc,longdesc))
                                 logger.warn("Exception: %s"%(unicode(e)))
                                 traceback.print_exc()
-                        soup = soup.find('body')
+                        bodysoup = soup.find('body')
                         # ffdl epubs have chapter title h3
-                        h3 = soup.find('h3')
+                        h3 = bodysoup.find('h3')
                         if h3:
                             h3.extract()
                         # TtH epubs have chapter title h2
-                        h2 = soup.find('h2')
+                        h2 = bodysoup.find('h2')
                         if h2:
                             h2.extract()
 
-                        for skip in soup.findAll(attrs={'class':'skip_on_ffdl_update'}):
+                        for skip in bodysoup.findAll(attrs={'class':'skip_on_ffdl_update'}):
                             skip.extract()
-                            
-                        soups.append(soup)
-                        
+
+                        ## <meta name="chapterurl" content="${url}"></meta>
+                        #print("look for meta chapurl")
+                        currenturl = None
+                        chapurl = soup.find('meta',{'name':'chapterurl'})
+                        if chapurl:
+                            if chapurl['content'] not in urlsoups: # keep first found if more than one.
+                            #print("Found chapurl['content']:%s"%chapurl['content'])
+                                currenturl = chapurl['content']
+                                urlsoups[chapurl['content']] = bodysoup
+                        else:
+                            # for older pre-meta.  Only temp.
+                            chapa = bodysoup.find('a',{'class':'chapterurl'})
+                            if chapa and chapa['href'] not in urlsoups: # keep first found if more than one.
+                                urlsoups[chapa['href']] = bodysoup
+                                currenturl = chapa['href']
+                                chapa.extract()
+
+                        chapterorigtitle = soup.find('meta',{'name':'chapterorigtitle'})
+                        if chapterorigtitle:
+                            datamaps[currenturl]['chapterorigtitle'] = chapterorigtitle['content']
+
+                        chaptertitle = soup.find('meta',{'name':'chaptertitle'})
+                        if chaptertitle:
+                            datamaps[currenturl]['chaptertitle'] = chaptertitle['content']
+
+                        soups.append(bodysoup)
+
                     filecount+=1
 
     try:
         calibrebookmark = epub.read("META-INF/calibre_bookmarks.txt")
     except:
         pass
-                    
+
     #for k in images.keys():
         #print("\tlongdesc:%s\n\tData len:%s\n"%(k,len(images[k])))
-    return (source,filecount,soups,images,oldcover,calibrebookmark,logfile)
+    # print("datamaps:%s"%datamaps)
+    return (source,filecount,soups,images,oldcover,calibrebookmark,logfile,urlsoups,datamaps)
 
 def get_path_part(n):
     relpath = os.path.dirname(n)
@@ -172,7 +199,7 @@ def get_story_url_from_html(inputio,_is_good_url=None):
 
     ## Save the path to the .opf file--hrefs inside it are relative to it.
     relpath = get_path_part(rootfilename)
-            
+
     # spin through the manifest--only place there are item tags.
     for item in contentdom.getElementsByTagName("item"):
         # First, count the 'chapter' files.  FFF uses file0000.xhtml,
@@ -192,3 +219,129 @@ def get_story_url_from_html(inputio,_is_good_url=None):
                 if _is_good_url == None or _is_good_url(ahref):
                     return ahref
     return None
+
+def reset_orig_chapters_epub(inputio,outfile):
+    inputepub = ZipFile(inputio, 'r') # works equally well with a path or a blob
+
+    ## build zip in memory in case updating in place(CLI).
+    zipio = StringIO()
+
+    ## Write mimetype file, must be first and uncompressed.
+    ## Older versions of python(2.4/5) don't allow you to specify
+    ## compression by individual file.
+    ## Overwrite if existing output file.
+    outputepub = ZipFile(zipio, 'w', compression=ZIP_STORED)
+    outputepub.debug = 3
+    outputepub.writestr("mimetype", "application/epub+zip")
+    outputepub.close()
+
+    ## Re-open file for content.
+    outputepub = ZipFile(zipio, "a", compression=ZIP_DEFLATED)
+    outputepub.debug = 3
+
+    changed = False
+
+    unmerge_tocncxdoms = {}
+    ## spin through file contents, saving any unmerge toc.ncx files.
+    for zf in inputepub.namelist():
+        ## logger.debug("zf:%s"%zf)
+        if zf.endswith('/toc.ncx'):
+            ## logger.debug("toc.ncx zf:%s"%zf)
+            unmerge_tocncxdoms[zf] = parseString(inputepub.read(zf))
+
+    tocncxdom = parseString(inputepub.read('toc.ncx'))
+    ## spin through file contents.
+    for zf in inputepub.namelist():
+        if zf not in ['mimetype','toc.ncx'] and not zf.endswith('/toc.ncx'):
+            entrychanged = False
+            data = inputepub.read(zf)
+            # if isinstance(data,unicode):
+            #     logger.debug("\n\n\ndata is unicode\n\n\n")
+            if re.match(r'.*/file\d+\.xhtml',zf):
+                #logger.debug("zf:%s"%zf)
+                data = data.decode('utf-8')
+                soup = bs.BeautifulSoup(data,"html5lib")
+
+                chapterorigtitle = None
+                tag = soup.find('meta',{'name':'chapterorigtitle'})
+                if tag:
+                    chapterorigtitle = tag['content']
+
+                # toctitle is separate for add_chapter_numbers:toconly users.
+                chaptertoctitle = None
+                tag = soup.find('meta',{'name':'chaptertoctitle'})
+                if tag:
+                    chaptertoctitle = tag['content']
+                elif chapterorigtitle:
+                    chaptertoctitle = chapterorigtitle
+
+                chaptertitle = None
+                tag = soup.find('meta',{'name':'chaptertitle'})
+                if tag:
+                    chaptertitle = tag['content']
+
+                if chaptertitle and chapterorigtitle and chapterorigtitle != chaptertitle:
+                    origdata = data
+                    # print("\n%s\n%s\n"%(chapterorigtitle,chaptertitle))
+                    data = data.replace(u'<meta name="chaptertitle" content="'+chaptertitle+u'"></meta>',
+                                        u'<meta name="chaptertitle" content="'+chapterorigtitle+u'"></meta>')
+                    data = data.replace(u'<title>'+chaptertitle+u'</title>',u'<title>'+chapterorigtitle+u'</title>')
+                    data = data.replace(u'<h3>'+chaptertitle+u'</h3>',u'<h3>'+chapterorigtitle+u'</h3>')
+
+                    entrychanged = ( origdata != data )
+                    changed = changed or entrychanged
+
+                    if entrychanged:
+                        _replace_tocncx(tocncxdom,zf,chaptertoctitle)
+                        ## Also look for and update individual
+                        ## book toc.ncx files for anthology in case
+                        ## it's unmerged.
+                        zf_toc = zf[:zf.rfind('/OEBPS/')]+'/toc.ncx'
+                        mergedprefix_len = len(zf[:zf.rfind('/OEBPS/')])+1
+
+                        if zf_toc in unmerge_tocncxdoms:
+                            _replace_tocncx(unmerge_tocncxdoms[zf_toc],zf[mergedprefix_len:],chaptertoctitle)
+
+                outputepub.writestr(zf,data.encode('utf-8'))
+            else:
+                # possibly binary data, thus no .encode().
+                outputepub.writestr(zf,data)
+
+    for tocnm, tocdom in unmerge_tocncxdoms.items():
+        outputepub.writestr(tocnm,tocdom.toxml(encoding='utf-8'))
+
+    outputepub.writestr('toc.ncx',tocncxdom.toxml(encoding='utf-8'))
+    outputepub.close()
+    # declares all the files created by Windows.  otherwise, when
+    # it runs in appengine, windows unzips the files as 000 perms.
+    for zf in outputepub.filelist:
+        zf.create_system = 0
+
+    # only *actually* write if changed.
+    if changed:
+        if isinstance(outfile,basestring):
+            with open(outfile,"wb") as outputio:
+                outputio.write(zipio.getvalue())
+        else:
+            outfile.write(zipio.getvalue())
+
+    inputepub.close()
+    zipio.close()
+
+    return changed
+
+
+def _replace_tocncx(tocncxdom,zf,chaptertoctitle):
+    ## go after the TOC entry, too.
+    # <navPoint id="file0005" playOrder="6">
+    #   <navLabel>
+    #     <text>5. (new) Chapter 4</text>
+    #   </navLabel>
+    #   <content src="OEBPS/file0005.xhtml"/>
+    # </navPoint>
+    for contenttag in tocncxdom.getElementsByTagName("content"):
+        if contenttag.getAttribute('src') == zf:
+            texttag = contenttag.parentNode.getElementsByTagName('navLabel')[0].getElementsByTagName('text')[0]
+            texttag.childNodes[0].replaceWholeText(chaptertoctitle)
+            #logger.debug("text label:%s"%texttag.toxml())
+            continue

fanficfare/story.py

@@ -16,6 +16,7 @@
 #
 
 import os, re
+from collections import namedtuple
 import urlparse
 import string
 import json
@@ -32,6 +33,8 @@ import exceptions
 from htmlcleanup import conditionalRemoveEntities, removeAllEntities
 from configurable import Configurable, re_compile
 
+Chapter = namedtuple('Chapter', 'url title html origtitle toctitle new')
+
 SPACE_REPLACE=u'\s'
 SPLIT_META=u'\,'
 
@@ -412,7 +415,9 @@ class Story(Configurable):
         except:
             self.metadata = {'version':'4.4'}
         self.in_ex_cludes = {}
-        self.chapters = [] # chapters will be tuples of (title,html)
+        self.chapters = [] # chapters will be namedtuple of Chapter(url,title,html,etc)
+        self.chapter_first = None
+        self.chapter_last = None
         self.imgurls = []
         self.imgtuples = []
 
@@ -441,8 +446,12 @@ class Story(Configurable):
                 iel = []
                 self.in_ex_cludes[ie] = set_in_ex_clude(ies)
 
+    def set_chapters_range(self,first=None,last=None):
+        self.chapter_first=first
+        self.chapter_last=last
+                
     def join_list(self, key, vallist):
-        return self.getConfig("join_string_"+key,u", ").replace(SPACE_REPLACE,' ').join(map(unicode, vallist))
+        return self.getConfig("join_string_"+key,u", ").replace(SPACE_REPLACE,' ').join(map(unicode, [ x for x in vallist if x is not None ]))
 
     def setMetadata(self, key, value, condremoveentities=True):
 
@@ -465,7 +474,7 @@ class Story(Configurable):
 
         if key == 'dateUpdated' and value:
             # Last Update tags for Bill.
-            self.addToList('lastupdate',value.strftime("Last Update Year/Month: %Y/%m"))
+            self.addToList('lastupdate',value.strftime("Last Update Year/Month: %Y/%m"),clear=True)
             self.addToList('lastupdate',value.strftime("Last Update: %Y/%m/%d"))
 
 
@@ -641,6 +650,14 @@ class Story(Configurable):
                 if key in ("datePublished","dateUpdated"):
                     value = value.strftime(self.getConfig(key+"_format","%Y-%m-%d"))
 
+                if key == "title" and (self.chapter_first or self.chapter_last) and self.getConfig("title_chapter_range_pattern"):
+                    first = self.chapter_first or "1"
+                    last = self.chapter_last or self.getMetadata("numChapters")
+                    templ = string.Template(self.getConfig("title_chapter_range_pattern"))
+                    value = templ.substitute({'title':value,
+                                              'first':commaGroups(first),
+                                              'last':commaGroups(last)})
+
             if doreplacements:
                 value=self.doReplacements(value,key)
             if removeallentities and value != None:
@@ -757,8 +774,12 @@ class Story(Configurable):
         # includelist prevents infinite recursion of include_in_'s
         if self.hasConfig("include_in_"+listname) and listname not in includelist:
             for k in self.getConfigList("include_in_"+listname):
+                ldorepl = doreplacements
+                if k.endswith('.NOREPL'):
+                    k = k[:-len('.NOREPL')]
+                    ldorepl = False
                 retlist.extend(self.getList(k,removeallentities=False,
-                                            doreplacements=doreplacements,includelist=includelist+[listname]))
+                                            doreplacements=ldorepl,includelist=includelist+[listname]))
         else:
 
             if not self.isList(listname):
@@ -813,7 +834,16 @@ class Story(Configurable):
 
         # metadata all go into dc:subject tags, but only if they are configured.
         for (name,value) in self.getAllMetadata(removeallentities=removeallentities,keeplists=True).iteritems():
-            if name in tags_list:
+            if name+'.SPLIT' in tags_list:
+                flist=[]
+                if isinstance(value,list):
+                    for tag in value:
+                        flist.extend(tag.split(','))
+                else:
+                    flist.extend(value)
+                for tag in flist:
+                    subjectset.add(tag)
+            elif name in tags_list:
                 if isinstance(value,list):
                     for tag in value:
                         subjectset.add(tag)
@@ -827,24 +857,63 @@ class Story(Configurable):
 
         return list(subjectset | set(self.getConfigList("extratags")))
 
-    def addChapter(self, url, title, html):
+    def addChapter(self, url, title, html, newchap=False):
+        # logger.debug("addChapter(%s,%s)"%(url,newchap))
         if self.getConfig('strip_chapter_numbers') and \
                 self.getConfig('chapter_title_strip_pattern'):
             title = re.sub(self.getConfig('chapter_title_strip_pattern'),"",title)
-        self.chapters.append( (url,title,html) )
+        self.chapters.append( Chapter(url,title,html,title,title,newchap) )
 
     def getChapters(self,fortoc=False):
-        "Chapters will be tuples of (title,html)"
+        "Chapters will be Chapter namedtuples"
         retval = []
-        ## only add numbers if more than one chapter.
-        if len(self.chapters) > 1 and \
-                (self.getConfig('add_chapter_numbers') == "true" \
-                     or (self.getConfig('add_chapter_numbers') == "toconly" and fortoc)) \
-                     and self.getConfig('chapter_title_add_pattern'):
-            for index, (url,title,html) in enumerate(self.chapters):
-                retval.append( (url,
-                                string.Template(self.getConfig('chapter_title_add_pattern')).substitute({'index':index+1,'title':title}),
-                                html) )
+
+        ## only add numbers if more than one chapter.  Ditto (new) marks.
+        if len(self.chapters) > 1:
+            addnums = ( self.getConfig('add_chapter_numbers') == "true" 
+                        or (self.getConfig('add_chapter_numbers') == "toconly" and fortoc) )
+            
+            marknew = self.getConfig('mark_new_chapters')=='true'
+
+            defpattern = self.getConfig('chapter_title_def_pattern','${title}') # default val in case of missing defaults.ini
+            if addnums and marknew:
+                pattern = self.getConfig('chapter_title_add_pattern')
+                newpattern = self.getConfig('chapter_title_addnew_pattern')
+            elif addnums:
+                pattern = self.getConfig('chapter_title_add_pattern')
+                newpattern = pattern
+            elif marknew:
+                pattern = defpattern
+                newpattern = self.getConfig('chapter_title_new_pattern')
+            else:
+                pattern = defpattern
+                newpattern = pattern
+
+            if self.getConfig('add_chapter_numbers') in ["true","toconly"]:
+                tocpattern = self.getConfig('chapter_title_add_pattern')
+            else:
+                tocpattern = defpattern
+
+            # logger.debug("Patterns: (%s)(%s)"%(pattern,newpattern))
+            templ = string.Template(pattern)
+            newtempl = string.Template(newpattern)
+            toctempl = string.Template(tocpattern)            
+            
+            for index, chap in enumerate(self.chapters):
+                if chap.new:
+                    usetempl = newtempl
+                else:
+                    usetempl = templ
+                # logger.debug("chap.url, chap.new: (%s)(%s)"%(chap.url,chap.new))
+                retval.append( Chapter(chap.url,
+                                       # 'new'
+                                       usetempl.substitute({'index':index+1,'title':chap.title}),
+                                       chap.html,
+                                       # 'orig'
+                                       templ.substitute({'index':index+1,'title':chap.title}),
+                                       # 'toc'
+                                       toctempl.substitute({'index':index+1,'title':chap.title}),
+                                       chap.new) )
         else:
             retval = self.chapters
 
@@ -858,9 +927,12 @@ class Story(Configurable):
 
         if not allowunsafefilename:
             values={}
-            pattern = re_compile(self.getConfig("output_filename_safepattern",r"[^a-zA-Z0-9_\. \[\]\(\)&'-]+"),"output_filename_safepattern")
+            pattern = re_compile(self.getConfig("output_filename_safepattern",r"(^\.|/\.|[^a-zA-Z0-9_\. \[\]\(\)&'-]+)"),"output_filename_safepattern")
             for k in origvalues.keys():
-                values[k]=re.sub(pattern,'_', removeAllEntities(self.getMetadata(k)))
+                if k == 'formatext': # don't do file extension--we set it anyway.
+                    values[k]=self.getMetadata(k)
+                else:
+                    values[k]=re.sub(pattern,'_', removeAllEntities(self.getMetadata(k)))
 
         return string.Template(template).substitute(values).encode('utf8')
 
@@ -911,6 +983,7 @@ class Story(Configurable):
                 #print("\n===========\nparsedUrl.path:%s\ntoppath:%s\nimgurl:%s\n\n"%(parsedUrl.path,toppath,imgurl))
 
         # apply coverexclusion to explicit covers, too.  Primarily for ffnet imageu.
+        #print("[[[[[\n\n %s %s \n\n]]]]]]]"%(imgurl,coverexclusion))
         if cover and coverexclusion and re.search(coverexclusion,imgurl):
             return (None,None)
 

fanficfare/writers/base_writer.py

@@ -148,12 +148,12 @@ class BaseStoryWriter(Configurable):
             
             self._write(out,START.substitute(self.story.getAllMetadata()))
 
-            for index, (url,title,html) in enumerate(self.story.getChapters(fortoc=True)):
-                if html:
-                    self._write(out,ENTRY.substitute({'chapter':title,
+            for index, chap in enumerate(self.story.getChapters(fortoc=True)):
+                if chap.html:
+                    self._write(out,ENTRY.substitute({'chapter':chap.title,
                                                       'number':index+1,
                                                       'index':"%04d"%(index+1),
-                                                      'url':url}))
+                                                      'url':chap.url}))
 
             self._write(out,END.substitute(self.story.getAllMetadata()))
 

fanficfare/writers/writer_epub.py

@@ -28,7 +28,7 @@ import re
 from xml.dom.minidom import parse, parseString, getDOMImplementation
 
 from base_writer import *
-from ..htmlcleanup import stripHTML
+from ..htmlcleanup import stripHTML,removeEntities
 
 logger = logging.getLogger(__name__)
 
@@ -133,6 +133,10 @@ ${value}<br />
 <head>
 <title>${chapter}</title>
 <link href="stylesheet.css" type="text/css" charset="UTF-8" rel="stylesheet"/>
+<meta name="chapterurl" content="${url}"></meta>
+<meta name="chapterorigtitle" content="${origchapter}"></meta>
+<meta name="chaptertoctitle" content="${tocchapter}"></meta>
+<meta name="chaptertitle" content="${chapter}"></meta>
 </head>
 <body>
 <h3>${chapter}</h3>
@@ -502,13 +506,13 @@ div { margin: 0pt; padding: 0pt; }
             items.append(("log_page","OEBPS/log_page.xhtml","application/xhtml+xml","Update Log"))
             itemrefs.append("log_page")
             
-        for index, (url,title,html) in enumerate(self.story.getChapters(fortoc=True)):
-            if html:
+        for index, chap in enumerate(self.story.getChapters(fortoc=True)):
+            if chap.html:
                 i=index+1
                 items.append(("file%04d"%i,
                               "OEBPS/file%04d.xhtml"%i,
                               "application/xhtml+xml",
-                              title))
+                              chap.title))
                 itemrefs.append("file%04d"%i)
 
         manifest = contentdom.createElement("manifest")
@@ -650,19 +654,21 @@ div { margin: 0pt; padding: 0pt; }
         else:
             CHAPTER_END = self.EPUB_CHAPTER_END
         
-        for index, (url,title,html) in enumerate(self.story.getChapters()):
-            if html:
-                logger.debug('Writing chapter text for: %s' % title)
-                vals={'url':url, 'chapter':title, 'index':"%04d"%(index+1), 'number':index+1}
-                fullhtml = CHAPTER_START.substitute(vals) + html + CHAPTER_END.substitute(vals)
+        for index, chap in enumerate(self.story.getChapters()): # (url,title,html)
+            if chap.html:
+                logger.debug('Writing chapter text for: %s' % chap.title)
+                vals={'url':removeEntities(chap.url),
+                      'chapter':chap.title,
+                      'origchapter':chap.origtitle,
+                      'tocchapter':chap.toctitle,
+                      'index':"%04d"%(index+1),
+                      'number':index+1}
+                fullhtml = CHAPTER_START.substitute(vals) + \
+                    chap.html + CHAPTER_END.substitute(vals)
                 # ffnet(& maybe others) gives the whole chapter text
                 # as one line.  This causes problems for nook(at
                 # least) when the chapter size starts getting big
                 # (200k+)
-                #fullhtml = fullhtml.replace('</p>','</p>\n').replace('<br />','<br />\n')
-                # The replaces above added tons of extra newlines
-                # during *each* epub update.  The regexp version adds
-                # only one and removes any extra.
                 fullhtml = re.sub(r'(</p>|<br ?/>)\n*',r'\1\n',fullhtml)
 
                 outputepub.writestr("OEBPS/file%04d.xhtml"%(index+1),fullhtml.encode('utf-8'))

fanficfare/writers/writer_html.py

@@ -128,12 +128,12 @@ ${output_css}
         else:
             CHAPTER_END = self.HTML_CHAPTER_END
         
-        for index, (url,title,html) in enumerate(self.story.getChapters()):
-            if html:
-                logging.debug('Writing chapter text for: %s' % title)
-                vals={'url':url, 'chapter':title, 'index':"%04d"%(index+1), 'number':index+1}
+        for index, chap in enumerate(self.story.getChapters()):
+            if chap.html:
+                logging.debug('Writing chapter text for: %s' % chap.title)
+                vals={'url':chap.url, 'chapter':chap.title, 'index':"%04d"%(index+1), 'number':index+1}
                 self._write(out,CHAPTER_START.substitute(vals))
-                self._write(out,html)
+                self._write(out,chap.html)
                 self._write(out,CHAPTER_END.substitute(vals))
 
         self._write(out,FILE_END.substitute(self.story.getAllMetadata()))

fanficfare/writers/writer_mobi.py

@@ -161,11 +161,11 @@ ${value}<br />
         else:
             CHAPTER_END = self.MOBI_CHAPTER_END
         
-        for index, (url,title,html) in enumerate(self.story.getChapters()):
-            if html:
-                logger.debug('Writing chapter text for: %s' % title)
-                vals={'url':url, 'chapter':title, 'index':"%04d"%(index+1), 'number':index+1}
-                fullhtml = CHAPTER_START.substitute(vals) + html + CHAPTER_END.substitute(vals)
+        for index, chap in enumerate(self.story.getChapters()):
+            if chap.html:
+                logger.debug('Writing chapter text for: %s' % chap.title)
+                vals={'url':chap.url, 'chapter':chap.title, 'index':"%04d"%(index+1), 'number':index+1}
+                fullhtml = CHAPTER_START.substitute(vals) + chap.html + CHAPTER_END.substitute(vals)
                 # ffnet(& maybe others) gives the whole chapter text
                 # as one line.  This causes problems for nook(at
                 # least) when the chapter size starts getting big

fanficfare/writers/writer_txt.py

@@ -154,12 +154,12 @@ End file.
         else:
             CHAPTER_END = self.TEXT_CHAPTER_END
         
-        for index, (url, title,html) in enumerate(self.story.getChapters()):
-            if html:
-                logging.debug('Writing chapter text for: %s' % title)
-                vals={'url':url, 'chapter':title, 'index':"%04d"%(index+1), 'number':index+1}
+        for index, chap in enumerate(self.story.getChapters()):
+            if chap.html:
+                logging.debug('Writing chapter text for: %s' % chap.title)
+                vals={'url':chap.url, 'chapter':chap.title, 'index':"%04d"%(index+1), 'number':index+1}
                 self._write(out,self.lineends(self.wraplines(removeAllEntities(CHAPTER_START.substitute(vals)))))
-                self._write(out,self.lineends(html2text(html,wrap_width=self.wrap_width)))
+                self._write(out,self.lineends(html2text(chap.html,wrap_width=self.wrap_width)))
                 self._write(out,self.lineends(self.wraplines(removeAllEntities(CHAPTER_END.substitute(vals)))))
 
         self._write(out,self.lineends(self.wraplines(FILE_END.substitute(self.story.getAllMetadata()))))

setup.py

@@ -10,13 +10,11 @@ https://github.com/pypa/sampleproject
 # Always prefer setuptools over distutils
 from setuptools import setup, find_packages
 # To use a consistent encoding
-from codecs import open
+import codecs
 from os import path
 
-here = path.abspath(path.dirname(__file__))
-
 # Get the long description from the relevant file
-with open(path.join(here, 'DESCRIPTION.rst'), encoding='utf-8') as f:
+with codecs.open('DESCRIPTION.rst', encoding='utf-8') as f:
     long_description = f.read()
 
 setup(
@@ -25,7 +23,7 @@ setup(
     # Versions should comply with PEP440.  For a discussion on single-sourcing
     # the version across setup.py and the project code, see
     # https://packaging.python.org/en/latest/single_source_version.html
-    version="2.2.10",
+    version="2.2.13",
 
     description='A tool for downloading fanfiction to eBook formats',
     long_description=long_description,

webservice/app.yaml

@@ -1,6 +1,6 @@
-# ffd-retief-hrd fanfictiondownloader fanficfare
+# ffd-retief-hrd fanficfare
 application: fanficfare
-version: 2-2-10
+version: 2-2-13
 runtime: python27
 api_version: 1
 threadsafe: true

webservice/ffstorage.py

@@ -41,6 +41,8 @@ class DownloadMeta(db.Model):
 	completed =  db.BooleanProperty(default=False)
 	date = db.DateTimeProperty(auto_now_add=True)
 	version = db.StringProperty()
+	ch_begin = db.StringProperty()
+	ch_end = db.StringProperty()
 	# data_chunks is implicit from DownloadData def.
 
 class DownloadData(db.Model):

webservice/index.html

@@ -27,26 +27,17 @@
 	      much easier. </p>
 	  </div>
 
-	  <h3>New Name and URL</h3>
-          <p>
-            This version changes the name of the project from
-            FanFictionDownLoader(FFDL) to FanFicFare.  Along with
-            the name change, there are changes to the internal
-            structuring of the project code, but not to the functionality.
-          </p>
-          <p>
-            If you had saved configuration (including your
-            username/password for fic sites) in FFDL, you will need to
-            copy it over to FanFicFare.  Visit
-            the <a href="http://fanfictiondownloader.appspot.com/editconfig">FFDL
-            User Config</a> page, copy your settings and then paste
-            them on
-            the <a href="http://fanficfare.appspot.com/editconfig">FanFicFare
-            User Config</a> page.
-          </p>
 	  <h3>Changes:</h3>
 	  <ul>
-            <li>Updates for mediaminer.org changes.</li>
+            <li>New site: <a href="http://www.quotev.com/">quotev.com</a>, thanks cryzed!</li>
+            <li>Convert dark-solace.org(Elysium fields) to eFiction Base adapter to correct for site changes.</li>
+            <li>Fix eFiction Base Author URLs.</li>
+            <li>Switch chapter #post-12345 chapter URLs to permalinks for XenForoForum Base adapters.</li>
+            <li>Record forum thread tags in XenForoForum Base adapters. Appears on title page by default, but not calibre tags.</li>
+            <li>Change output_filename_safepattern to replace leading '.' for benefit of *nix OSes.</li>
+            <li>Fix mediaminer.org for site changes.</li>
+            <li>Fix to hpfandom.net change story URL when given chapter URL instead of story start URL.</li>
+            <li>Allow forum.questionablequesting.com for questionablequesting.com.</li>
 	  </ul>
 	  <p>
 	    Questions?  Check out our 
@@ -56,7 +47,7 @@
 	    If you have any problems with this application, please
 	    report them in
 	    the <a href="http://groups.google.com/group/fanfic-downloader">FanFicFare Google Group</a>.  The
-	    <a href="http://2-2-9.fanficfare.appspot.com">previous version
+	    <a href="http://2-2-12.fanficfare.appspot.com">previous version
 	    </a> is also available for you to use if necessary.
 	  </p>
 	  <div id='error'>

webservice/main.py

@@ -62,7 +62,7 @@ class UserConfigServer(webapp2.RequestHandler):
     
     def getUserConfig(self,user,url,fileformat):
 
-        configuration = Configuration(adapters.getConfigSectionFor(url),fileformat)
+        configuration = Configuration(adapters.getConfigSectionsFor(url),fileformat)
         
         logging.debug('reading defaults.ini config file')
         configuration.read('fanficfare/defaults.ini')
@@ -366,6 +366,16 @@ class FanfictionDownloader(UserConfigServer):
             self.redirect('/')
             return
 
+        # Allow chapter range with URL.
+        # test1.com?sid=5[4-6]
+        mc = re.match(r"^(?P<url>.*?)(?:\[(?P<begin>\d+)?(?P<comma>[,-])?(?P<end>\d+)?\])?$",url)
+        #print("url:(%s) begin:(%s) end:(%s)"%(mc.group('url'),mc.group('begin'),mc.group('end')))
+        url = mc.group('url')
+        ch_begin = mc.group('begin')
+        ch_end = mc.group('end')
+        if ch_begin and not mc.group('comma'):
+            ch_end = ch_begin
+        
         logging.info("Queuing Download: %s" % url)
         login = self.request.get('login')
         password = self.request.get('password')
@@ -385,6 +395,7 @@ class FanfictionDownloader(UserConfigServer):
                 return
 
             adapter = adapters.getAdapter(configuration,url)
+            adapter.setChaptersRange(ch_begin,ch_end)
             logging.info('Created an adaper: %s' % adapter)
 
             if login or password:
@@ -408,6 +419,8 @@ class FanfictionDownloader(UserConfigServer):
             download.title = story.getMetadata('title')
             download.author = story.getMetadata('author')
             download.url = story.getMetadata('storyUrl')
+            download.ch_begin = ch_begin
+            download.ch_end = ch_end
             download.put()
 
             taskqueue.add(url='/fdowntask',
@@ -490,6 +503,7 @@ class FanfictionDownloaderTask(UserConfigServer):
         try:
             configuration = self.getUserConfig(user,url,format)
             adapter = adapters.getAdapter(configuration,url)
+            adapter.setChaptersRange(download.ch_begin,download.ch_end)
 
             logging.info('Created an adapter: %s' % adapter)