Update translations, bump versions.

Add MassEffect2.in Adapter

MANIFEST.in

@@ -0,0 +1 @@
+include DESCRIPTION.rst

calibre-plugin/__init__.py

@@ -42,7 +42,7 @@ class FanFicFareBase(InterfaceActionBase):
     description         = _('UI plugin to download FanFiction stories from various sites.')
     supported_platforms = ['windows', 'osx', 'linux']
     author              = 'Jim Miller'
-    version             = (2, 2, 8)
+    version             = (2, 2, 12)
     minimum_calibre_version = (1, 48, 0)
 
     #: This field defines the GUI plugin class that contains all the code

calibre-plugin/basicinihighlighter.py

@@ -1,5 +1,5 @@
-#!/usr/bin/env python
-# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+# -*- coding: utf-8 -*-
+
 from __future__ import (unicode_literals, division,
                         print_function)
 

calibre-plugin/common_utils.py

@@ -1,5 +1,5 @@
-#!/usr/bin/env python
-# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+# -*- coding: utf-8 -*-
+
 from __future__ import (unicode_literals, division, absolute_import,
                         print_function)
 

calibre-plugin/config.py

@@ -1,5 +1,5 @@
-#!/usr/bin/env python
-# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+# -*- coding: utf-8 -*-
+
 from __future__ import (unicode_literals, division, absolute_import,
                         print_function)
 
@@ -89,7 +89,7 @@ from calibre_plugins.fanficfare_plugin.dialogs \
             EditTextDialog, IniTextDialog, RejectUrlEntry)
     
 from calibre_plugins.fanficfare_plugin.fanficfare.adapters \
-    import getConfigSections
+    import getSiteSections
 
 from calibre_plugins.fanficfare_plugin.common_utils \
     import ( KeyboardConfigDialog, PrefsViewerDialog )
@@ -259,6 +259,7 @@ class ConfigWidget(QWidget):
         prefs['checkforseriesurlid'] = self.basic_tab.checkforseriesurlid.isChecked()
         prefs['checkforurlchange'] = self.basic_tab.checkforurlchange.isChecked()
         prefs['injectseries'] = self.basic_tab.injectseries.isChecked()
+        prefs['matchtitleauth'] = self.basic_tab.matchtitleauth.isChecked()
         prefs['smarten_punctuation'] = self.basic_tab.smarten_punctuation.isChecked()
         prefs['reject_always'] = self.basic_tab.reject_always.isChecked()
 
@@ -271,6 +272,7 @@ class ConfigWidget(QWidget):
             prefs['addtolists'] = self.readinglist_tab.addtolists.isChecked()
             prefs['addtoreadlists'] = self.readinglist_tab.addtoreadlists.isChecked()
             prefs['addtolistsonread'] = self.readinglist_tab.addtolistsonread.isChecked()
+            prefs['autounnew'] = self.readinglist_tab.autounnew.isChecked()
 
         # personal.ini
         ini = self.personalini_tab.personalini
@@ -280,6 +282,8 @@ class ConfigWidget(QWidget):
             # if they've removed everything, reset to default.
             prefs['personal.ini'] = get_resources('plugin-example.ini')
 
+        prefs['cal_cols_pass_in'] = self.personalini_tab.cal_cols_pass_in.isChecked()
+            
         # Covers tab
         prefs['updatecalcover'] = calcover_save_options[unicode(self.calibrecover_tab.updatecalcover.currentText())]
         # for backward compatibility:
@@ -321,6 +325,8 @@ class ConfigWidget(QWidget):
             colsnewonly[col] = checkbox.isChecked()
         prefs['std_cols_newonly'] = colsnewonly
 
+        prefs['set_author_url'] =self.std_columns_tab.set_author_url.isChecked()
+
         # Custom Columns tab
         # error column
         prefs['errorcol'] = unicode(convert_qvariant(self.cust_columns_tab.errorcol.itemData(self.cust_columns_tab.errorcol.currentIndex())))
@@ -344,13 +350,14 @@ class ConfigWidget(QWidget):
         
         prefs['allow_custcol_from_ini'] = self.cust_columns_tab.allow_custcol_from_ini.isChecked()
 
-        prefs['imapserver'] = unicode(self.imap_tab.imapserver.text())
-        prefs['imapuser'] = unicode(self.imap_tab.imapuser.text())
-        prefs['imappass'] = unicode(self.imap_tab.imappass.text())
-        prefs['imapfolder'] = unicode(self.imap_tab.imapfolder.text())
+        prefs['imapserver'] = unicode(self.imap_tab.imapserver.text()).strip()
+        prefs['imapuser'] = unicode(self.imap_tab.imapuser.text()).strip()
+        prefs['imappass'] = unicode(self.imap_tab.imappass.text()).strip()
+        prefs['imapfolder'] = unicode(self.imap_tab.imapfolder.text()).strip()
         prefs['imapmarkread'] = self.imap_tab.imapmarkread.isChecked()
         prefs['imapsessionpass'] = self.imap_tab.imapsessionpass.isChecked()
         prefs['auto_reject_from_email'] = self.imap_tab.auto_reject_from_email.isChecked()
+        prefs['download_from_email_immediately'] = self.imap_tab.download_from_email_immediately.isChecked()
         
         prefs.save_to_db()
 
@@ -517,6 +524,11 @@ class BasicTab(QWidget):
         self.injectseries.setChecked(prefs['injectseries'])
         self.l.addWidget(self.injectseries)
 
+        self.matchtitleauth = QCheckBox(_("Search by Title/Author(s) for If Story Already Exists?"),self)
+        self.matchtitleauth.setToolTip(_("When checking <i>If Story Already Exists</i> FanFicFare will first match by URL Identifier.  But if not found, it can also search existing books by Title and Author(s)."))
+        self.matchtitleauth.setChecked(prefs['matchtitleauth'])
+        self.l.addWidget(self.matchtitleauth)
+
         rej_gb = groupbox = QGroupBox(_("Reject List"))
         self.l = QVBoxLayout()
         groupbox.setLayout(self.l)
@@ -655,8 +667,13 @@ class PersonalIniTab(QWidget):
         self.defaults.clicked.connect(self.show_defaults)
         self.l.addWidget(self.defaults)
 
+        self.cal_cols_pass_in = QCheckBox(_('Pass Calibre Columns into FanFicFare on Update/Overwrite')%no_trans,self)
+        self.cal_cols_pass_in.setToolTip(_("If checked, when updating/overwriting an existing book, FanFicFare will have the Calibre Columns available to use in replace_metadata, title_page, etc.<br>Click the button below to see the Calibre Column namess.")%no_trans)
+        self.cal_cols_pass_in.setChecked(prefs['cal_cols_pass_in'])
+        self.l.addWidget(self.cal_cols_pass_in)
+        
         self.showcalcols = QPushButton(_('Show Calibre Column Names'), self)
-        self.showcalcols.setToolTip(_("FanFicFare passes the Calibre columns into the download/update process.  This will show you the columns available by name."))
+        self.showcalcols.setToolTip(_("FanFicFare can pass the Calibre Columns into the download/update process.<br>This will show you the columns available by name."))
         self.showcalcols.clicked.connect(self.show_showcalcols)
         self.l.addWidget(self.showcalcols)
 
@@ -766,6 +783,11 @@ class ReadingListTab(QWidget):
         self.addtolistsonread.setChecked(prefs['addtolistsonread'])
         self.l.addWidget(self.addtolistsonread)
             
+        self.autounnew = QCheckBox(_('Automatically run Remove "New" Chapter Marks when marking books "Read".'),self)
+        self.autounnew.setToolTip(_('Menu option to remove from "To Read" lists will also remove "(new)" chapter marks created by personal.ini <i>mark_new_chapters</i> setting.'))
+        self.autounnew.setChecked(prefs['autounnew'])
+        self.l.addWidget(self.autounnew)
+            
         self.l.insertStretch(-1)
         
 class CalibreCoverTab(QWidget):
@@ -913,7 +935,7 @@ class CalibreCoverTab(QWidget):
         
         self.gc_dropdowns = {}
 
-        sitelist = getConfigSections()
+        sitelist = getSiteSections()
         sitelist.sort()
         sitelist.insert(0,_("Default"))
         for site in sitelist:
@@ -1223,7 +1245,6 @@ class CustomColumnsTab(QWidget):
         self.allow_custcol_from_ini.setChecked(prefs['allow_custcol_from_ini'])
         self.l.addWidget(self.allow_custcol_from_ini)
         
-        self.l.addSpacing(5)
         label = QLabel(_("Special column:"))
         label.setWordWrap(True)
         self.l.addWidget(label)
@@ -1305,6 +1326,17 @@ class StandardColumnsTab(QWidget):
             horz.addWidget(newonlycheck)
             
             self.l.addLayout(horz)
+            
+        self.l.addSpacing(5)
+        label = QLabel(_("Other Standard Column Options"))
+        label.setWordWrap(True)
+        self.l.addWidget(label)
+        self.l.addSpacing(5)
+
+        self.set_author_url = QCheckBox(_('Set Calibre Author URL'),self)
+        self.set_author_url.setToolTip(_("Set Calibre Author URL to Author's URL on story site."))
+        self.set_author_url.setChecked(prefs['set_author_url'])
+        self.l.addWidget(self.set_author_url)
         
         self.l.insertStretch(-1)
 
@@ -1383,6 +1415,12 @@ class ImapTab(QWidget):
         self.l.addWidget(self.auto_reject_from_email,row,0,1,-1)
         row+=1
         
+        self.download_from_email_immediately = QCheckBox(_('Download from Email Immediately'),self)
+        self.download_from_email_immediately.setToolTip(_('If checked, FanFicFare will start downloading story URLs from emails immediately.<br>Otherwise the usual Download from URLs dialog will appear.'))
+        self.download_from_email_immediately.setChecked(prefs['download_from_email_immediately'])
+        self.l.addWidget(self.download_from_email_immediately,row,0,1,-1)
+        row+=1
+        
         label = QLabel(_("<b>It's safest if you create a separate email account that you use only "
                          "for your story update notices.  FanFicFare and calibre cannot guarantee that "
                          "malicious code cannot get your email password once you've entered it. "

calibre-plugin/dialogs.py

@@ -1,5 +1,5 @@
-#!/usr/bin/env python
-# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+# -*- coding: utf-8 -*-
+
 from __future__ import (unicode_literals, division,
                         print_function)
 
@@ -258,6 +258,7 @@ class AddNewDialog(SizePersistedDialog):
         self.l.addWidget(self.url)
 
         self.merge = self.newmerge = False
+        self.extraoptions = {}
         
         # elements to hide when doing merge.
         self.mergehide = []

calibre-plugin/fff_plugin.py

@@ -1,5 +1,5 @@
-#!/usr/bin/env python
-# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+# -*- coding: utf-8 -*-
+
 from __future__ import (unicode_literals, division, absolute_import,
                         print_function)
 
@@ -66,7 +66,8 @@ from calibre_plugins.fanficfare_plugin.fanficfare import (
     adapters, exceptions)
 
 from calibre_plugins.fanficfare_plugin.fanficfare.epubutils import (
-    get_dcsource, get_dcsource_chaptercount, get_story_url_from_html)
+    get_dcsource, get_dcsource_chaptercount, get_story_url_from_html,
+    reset_orig_chapters_epub)
 
 from calibre_plugins.fanficfare_plugin.fanficfare.geturls import (
     get_urls_from_page, get_urls_from_html,get_urls_from_text,
@@ -335,6 +336,12 @@ class FanFicFarePlugin(InterfaceAction):
                                                                       image='minusminus.png',
                                                                       triggered=partial(self.update_lists,add=False))
 
+            self.menu.addSeparator()
+            self.get_list_action = self.create_menu_item_ex(self.menu, _('Remove "New" Chapter Marks from Selected books'),
+                                                            unique_name='Remove "(new)" chapter marks created by personal.ini <i>mark_new_chapters</i> setting.',
+                                                            image='edit-undo.png',
+                                                            triggered=self.unnew_books)
+
             self.menu.addSeparator()
             self.get_list_action = self.create_menu_item_ex(self.menu, _('Get Story URLs from Selected Books'),
                                                             unique_name='Get URLs from Selected Books',
@@ -415,6 +422,8 @@ class FanFicFarePlugin(InterfaceAction):
                 return
 
             self.update_reading_lists(self.gui.library_view.get_selected_ids(),add)
+            if not add and prefs['autounnew']:
+                self.unnew_books()
 
     def get_urls_from_imap_menu(self):
 
@@ -451,21 +460,29 @@ class FanFicFarePlugin(InterfaceAction):
             reject_list = set([x for x in url_list if rejecturllist.check(adapters.getNormalStoryURLSite(x)[0])])
         url_list = url_list - reject_list
 
-        self.gui.status_bar.show_message(_('Finished Fetching Story URLs from Email.'),3000)
+        self.gui.status_bar.show_message(_('No Valid Story URLs Found in Unread Emails.'),3000)
         self.restore_cursor()
-        
-        if url_list:
-            self.add_dialog("\n".join(url_list),merge=False)
-        else:
-            
-            msg = _('No Valid Story URLs Found in Unread Emails.')
-            if reject_list:
-                msg = msg + '<p>'+(_('(%d Story URLs Skipped, on Rejected URL List)')%len(reject_list))+'</p>'
-            info_dialog(self.gui, _('Get Story URLs from Email'),
-                        msg,
-                        show=True,
-                        show_copy_button=False)
 
+        if prefs['download_from_email_immediately']:
+            ## do imap fetch w/o GUI elements
+            if url_list:
+                self.prep_downloads(self.add_new_dialog.get_fff_options(),
+                                    "\n".join(url_list))
+            else:
+                self.gui.status_bar.show_message(_('Finished Fetching Story URLs from Email.'),3000)
+                
+        else:
+            if url_list:
+                self.add_dialog("\n".join(url_list),merge=False)
+            else:            
+                msg = _('No Valid Story URLs Found in Unread Emails.')
+                if reject_list:
+                    msg = msg + '<p>'+(_('(%d Story URLs Skipped, on Rejected URL List)')%len(reject_list))+'</p>'
+                info_dialog(self.gui, _('Get Story URLs from Email'),
+                            msg,
+                            show=True,
+                            show_copy_button=False)
+    
     def get_urls_from_page_menu(self,anthology=False):
 
         urltxt = ""
@@ -555,6 +572,70 @@ class FanFicFarePlugin(InterfaceAction):
                         show=True,
                         show_copy_button=False)
 
+    def unnew_books(self):
+        '''Get list of URLs from existing books.'''
+        if not self.is_library_view():
+            self.gui.status_bar.show_message(_('Can only UnNew books in library'),
+                                             3000)
+            return
+        
+        if not self.gui.current_view().selectionModel().selectedRows() :
+            self.gui.status_bar.show_message(_('No Selected Books to Get URLs From'),
+                                             3000)
+            return
+
+        book_list = map( partial(self.make_book_id_only),
+                         self.gui.library_view.get_selected_ids() )
+
+        tdir = PersistentTemporaryDirectory(prefix='fanficfare_')
+        LoopProgressDialog(self.gui,
+                           book_list,
+                           partial(self.get_unnew_books_loop, db=self.gui.current_db, tdir=tdir),
+                           partial(self.get_unnew_books_finish, tdir=tdir),
+                           init_label=_("UnNewing books..."),
+                           win_title=_("UnNew Books"),
+                           status_prefix=_("Books UnNewed"))
+
+    def get_unnew_books_loop(self,book,db=None,tdir=None):
+        
+        if book['calibre_id'] and db.has_format(book['calibre_id'],'EPUB',index_is_id=True):
+            tmp = PersistentTemporaryFile(prefix='%s-'%book['calibre_id'],
+                                          suffix='.epub',
+                                          dir=tdir)
+            db.copy_format_to(book['calibre_id'],'EPUB',tmp,index_is_id=True)
+            
+            unnewtmp = PersistentTemporaryFile(prefix='unnew-%s-'%book['calibre_id'],
+                                               suffix='.epub',
+                                               dir=tdir)
+            book['changed']=reset_orig_chapters_epub(tmp,unnewtmp)
+            if book['changed']:
+                db.add_format_with_hooks(book['calibre_id'],
+                                         'EPUB',
+                                         unnewtmp,
+                                         index_is_id=True)
+                if prefs['deleteotherforms']:
+                    fmts = db.formats(book['calibre_id'], index_is_id=True).split(',')
+                    for fmt in fmts:
+                        if fmt.lower() != formmapping['epub'].lower():
+                            logger.debug("deleteotherforms remove f:"+fmt)
+                            db.remove_format(book['calibre_id'], fmt, index_is_id=True)#, notify=False
+                elif prefs['autoconvert']:
+                    ## 'Convert Book'.auto_convert_auto_add doesn't convert if
+                    ## the format is already there.
+                    fmt = calibre_prefs['output_format']
+                    # delete if there, but not if the format we just made.
+                    if fmt.lower() != 'epub' and db.has_format(book['calibre_id'],fmt,index_is_id=True):
+                        logger.debug("autoconvert remove f:"+fmt)
+                        db.remove_format(book['calibre_id'], fmt, index_is_id=True)#, notify=False
+                
+    def get_unnew_books_finish(self, book_list, tdir=None):
+        remove_dir(tdir)
+        if prefs['autoconvert']:
+            changed_ids = [ x['calibre_id'] for x in book_list if x['changed'] ]
+            if changed_ids:
+                self.gui.status_bar.show_message(_('Starting auto conversion of %d books.')%(len(changed_ids)), 3000)
+                self.gui.iactions['Convert Books'].auto_convert_auto_add(changed_ids)
+
     def reject_list_urls(self):
         if self.is_library_view():
             book_list = map( partial(self.make_book_id_only),
@@ -828,6 +909,7 @@ class FanFicFarePlugin(InterfaceAction):
         options['version'] = self.version
         logger.debug(self.version)
         options['personal.ini'] = get_fff_personalini()
+        options['savemetacol'] = prefs['savemetacol']
 
         #print("prep_downloads:%s"%books)
 
@@ -920,6 +1002,8 @@ class FanFicFarePlugin(InterfaceAction):
             return
 
         adapter = get_fff_adapter(url,fileform)
+        ## chapter range for title_chapter_range_pattern
+        adapter.setChaptersRange(book['begin'],book['end'])
         ## save and share cookiejar and pagecache between all
         ## downloads.
         if 'pagecache' not in options:
@@ -1023,8 +1107,9 @@ class FanFicFarePlugin(InterfaceAction):
 
         # all_metadata duplicates some data, but also includes extra_entries, etc.
         book['all_metadata'] = story.getAllMetadata(removeallentities=True)
-        # get metadata to save in configured column.
-        book['savemetacol'] = story.dump_html_metadata()
+        if prefs['savemetacol'] != '':
+            # get metadata to save in configured column.
+            book['savemetacol'] = story.dump_html_metadata()
         
         book['title'] = story.getMetadata("title", removeallentities=True)
         book['author_sort'] = book['author'] = story.getList("author", removeallentities=True)
@@ -1072,7 +1157,7 @@ class FanFicFarePlugin(InterfaceAction):
                 # try to find by identifier url or uri first.
                 identicalbooks = self.do_id_search(url)
                 # print("identicalbooks:%s"%identicalbooks)
-                if len(identicalbooks) < 1:
+                if len(identicalbooks) < 1 and prefs['matchtitleauth']:
                     # find dups
                     authlist = story.getList("author", removeallentities=True)
                     mi = MetaInformation(story.getMetadata("title", removeallentities=True),
@@ -1202,51 +1287,52 @@ class FanFicFarePlugin(InterfaceAction):
                     #print("calibre_series:%s [%s]"%book['calibre_series'])
 
         if book['good']: # there shouldn't be any !'good' books at this point.
-            
+
             ## Filling calibre_std_* and calibre_cust_* metadata
             book['calibre_columns']={}
-            # std columns
-            mi = db.get_metadata(book['calibre_id'],index_is_id=True)
-            # book['calibre_columns']['calibre_std_identifiers']=\
-            #     {'val':', '.join(["%s:%s"%(k,v) for (k,v) in mi.get_identifiers().iteritems()]),
-            #                      'label':_('Ids')}
-            for k in mi.standard_field_keys():
-            # for k in mi:
-                if k in STD_COLS_SKIP:
-                    continue
-                (label,value,v,fmd) = mi.format_field_extended(k)
-                if not label and k in field_metadata:
-                    label=field_metadata[k]['name']
-                key='calibre_std_'+k
-                
-                # if k == 'user_categories':
-                #     value=u', '.join(mi.get(k))
-                #     label=_('User Categories')
-    
-                if label: # only if it has a human readable name.
-                    if value is None or not book['calibre_id']:
-                        ## if existing book, populate existing calibre column
-                        ## values in metadata, else '' to hide.
-                        value=''                    
-                    book['calibre_columns'][key]={'val':value,'label':label}
-                    #logger.debug("%s(%s): %s"%(label,key,value))
-    
-            # custom columns
-            for k, column in self.gui.library_view.model().custom_columns.iteritems():
-                if k != prefs['savemetacol']:
-                    key='calibre_cust_'+k[1:]
-                    label=column['name']
-                    value=db.get_custom(book['calibre_id'],
-                                        label=column['label'],
-                                        index_is_id=True)
-                    # custom always have name.
-                    if value is None or not book['calibre_id']:
-                        ## if existing book, populate existing calibre column
-                        ## values in metadata, else '' to hide.
-                        value=''
-                    book['calibre_columns'][key]={'val':value,'label':label}
-                    # logger.debug("%s(%s): %s"%(label,key,value))
+            if prefs['cal_cols_pass_in']:
+                # std columns
+                mi = db.get_metadata(book['calibre_id'],index_is_id=True)
+                # book['calibre_columns']['calibre_std_identifiers']=\
+                #     {'val':', '.join(["%s:%s"%(k,v) for (k,v) in mi.get_identifiers().iteritems()]),
+                #                      'label':_('Ids')}
+                for k in mi.standard_field_keys():
+                # for k in mi:
+                    if k in STD_COLS_SKIP:
+                        continue
+                    (label,value,v,fmd) = mi.format_field_extended(k)
+                    if not label and k in field_metadata:
+                        label=field_metadata[k]['name']
+                    key='calibre_std_'+k
+                    
+                    # if k == 'user_categories':
+                    #     value=u', '.join(mi.get(k))
+                    #     label=_('User Categories')
         
+                    if label: # only if it has a human readable name.
+                        if value is None or not book['calibre_id']:
+                            ## if existing book, populate existing calibre column
+                            ## values in metadata, else '' to hide.
+                            value=''                    
+                        book['calibre_columns'][key]={'val':value,'label':label}
+                        #logger.debug("%s(%s): %s"%(label,key,value))
+        
+                # custom columns
+                for k, column in self.gui.library_view.model().custom_columns.iteritems():
+                    if k != prefs['savemetacol']:
+                        key='calibre_cust_'+k[1:]
+                        label=column['name']
+                        value=db.get_custom(book['calibre_id'],
+                                            label=column['label'],
+                                            index_is_id=True)
+                        # custom always have name.
+                        if value is None or not book['calibre_id']:
+                            ## if existing book, populate existing calibre column
+                            ## values in metadata, else '' to hide.
+                            value=''
+                        book['calibre_columns'][key]={'val':value,'label':label}
+                        # logger.debug("%s(%s): %s"%(label,key,value))
+            
             # if still 'good', make a temp file to write the output to.
             # For HTML format users, make the filename inside the zip something reasonable.
             # For crazy long titles/authors, limit it to 200chars.
@@ -1738,7 +1824,7 @@ class FanFicFarePlugin(InterfaceAction):
         # save metadata to configured column
         if 'savemetacol' in book and prefs['savemetacol'] != '' and prefs['savemetacol'] in custom_columns:
             label = custom_columns[prefs['savemetacol']]['label']
-            self.set_custom(db, book_id, 'comment', book['savemetacol'], label=label, commit=True) # book['comment'] book['savemetacol'] = story.dump_html_metadata()
+            self.set_custom(db, book_id, 'comment', book['savemetacol'], label=label, commit=True)
 
         #print("prefs['custom_cols'] %s"%prefs['custom_cols'])
         for col, meta in prefs['custom_cols'].iteritems():
@@ -1840,7 +1926,7 @@ class FanFicFarePlugin(InterfaceAction):
 
         # set author link if found.  All current adapters have authorUrl, except anonymous on AO3.
         # Moved down so author's already in the DB.
-        if 'authorUrl' in book['all_metadata']:
+        if 'authorUrl' in book['all_metadata'] and prefs['set_author_url']:
             authurls = book['all_metadata']['authorUrl'].split(", ")
             authorlist = [ a.replace('&',';') for a in book['author'] ]
             authorids = db.new_api.get_item_ids('authors',authorlist)

calibre-plugin/fff_util.py

@@ -1,5 +1,5 @@
-#!/usr/bin/env python
-# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+# -*- coding: utf-8 -*-
+
 from __future__ import (unicode_literals, division, absolute_import,
                         print_function)
 
@@ -23,12 +23,12 @@ def get_fff_personalini():
 def get_fff_config(url,fileform="epub",personalini=None):
     if not personalini:
         personalini = get_fff_personalini()
-    site='unknown'
+    sections=['unknown']
     try:
-        site = adapters.getConfigSectionFor(url)
+        sections = adapters.getConfigSectionsFor(url)
     except Exception as e:
-        logger.debug("Failed trying to get ini config for url(%s): %s, using section [%s] instead"%(url,e,site))
-    configuration = Configuration(site,fileform)
+        logger.debug("Failed trying to get ini config for url(%s): %s, using section %s instead"%(url,e,sections))
+    configuration = Configuration(sections,fileform)
     configuration.readfp(StringIO(get_resources("plugin-defaults.ini")))
     configuration.readfp(StringIO(personalini))
 

calibre-plugin/inihighlighter.py

@@ -1,5 +1,5 @@
-#!/usr/bin/env python
-# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+# -*- coding: utf-8 -*-
+
 from __future__ import (unicode_literals, division,
                         print_function)
 

calibre-plugin/jobs.py

@@ -1,5 +1,5 @@
-#!/usr/bin/env python
-# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+# -*- coding: utf-8 -*-
+
 from __future__ import (unicode_literals, division, absolute_import,
                         print_function)
 
@@ -156,7 +156,8 @@ def do_download_for_worker(book,options,notification=lambda x,y:x):
                 logger.info("Skipping CALIBREONLY 'update' down inside worker--this shouldn't be happening...")
                 book['comment'] = 'Metadata collected.'
                 book['all_metadata'] = story.getAllMetadata(removeallentities=True)
-                book['savemetacol'] = story.dump_html_metadata()
+                if options['savemetacol'] != '':
+                    book['savemetacol'] = story.dump_html_metadata()
                 
             ## checks were done earlier, it's new or not dup or newer--just write it.
             elif options['collision'] in (ADDNEW, SKIP, OVERWRITE, OVERWRITEALWAYS) or \
@@ -176,7 +177,8 @@ def do_download_for_worker(book,options,notification=lambda x,y:x):
                 writer.writeStory(outfilename=outfile, forceOverwrite=True)
                 book['comment'] = 'Download %s completed, %s chapters.'%(options['fileform'],story.getMetadata("numChapters"))
                 book['all_metadata'] = story.getAllMetadata(removeallentities=True)
-                book['savemetacol'] = story.dump_html_metadata()
+                if options['savemetacol'] != '':
+                    book['savemetacol'] = story.dump_html_metadata()
                 
             ## checks were done earlier, just update it.
             elif 'epub_for_update' in book and options['collision'] in (UPDATE, UPDATEALWAYS):
@@ -190,14 +192,17 @@ def do_download_for_worker(book,options,notification=lambda x,y:x):
                  adapter.oldimgs,
                  adapter.oldcover,
                  adapter.calibrebookmark,
-                 adapter.logfile) = get_update_data(book['epub_for_update'])[0:7]
+                 adapter.logfile,
+                 adapter.oldchaptersmap,
+                 adapter.oldchaptersdata) = get_update_data(book['epub_for_update'])[0:9]
     
                 # dup handling from fff_plugin needed for anthology updates.
                 if options['collision'] == UPDATE:
                     if chaptercount == urlchaptercount:
                         book['comment']=_("Already contains %d chapters.  Reuse as is.")%chaptercount
                         book['all_metadata'] = story.getAllMetadata(removeallentities=True)
-                        book['savemetacol'] = story.dump_html_metadata()
+                        if options['savemetacol'] != '':
+                            book['savemetacol'] = story.dump_html_metadata()
                         book['outfile'] = book['epub_for_update'] # for anthology merge ops.
                         return book
     
@@ -218,8 +223,9 @@ def do_download_for_worker(book,options,notification=lambda x,y:x):
                 book['comment'] = _('Update %s completed, added %s chapters for %s total.')%\
                     (options['fileform'],(urlchaptercount-chaptercount),urlchaptercount)
                 book['all_metadata'] = story.getAllMetadata(removeallentities=True)
-                book['savemetacol'] = story.dump_html_metadata()
-            
+                if options['savemetacol'] != '':
+                    book['savemetacol'] = story.dump_html_metadata()
+
             if options['smarten_punctuation'] and options['fileform'] == "epub" \
                     and calibre_version >= (0, 9, 39):
                 # for smarten punc

calibre-plugin/plugin-defaults.ini

@@ -261,6 +261,11 @@ keep_summary_html:true
 ## kept.  Some sites(AO3) use <a title="Translation">Foreign language</a>
 #keep_title_attr: false
 
+## If a chapter range was given, use this pattern for the book title.
+## replace_metadata and include/exclude will be applied *after* this.
+## Set to empty value to disable.
+title_chapter_range_pattern:${title} (Ch ${first}-${last})
+
 ## Don't like the numbers at the start of chapter titles on some
 ## sites?  You can use strip_chapter_numbers to strip them off.  Just
 ## want to make them all look the same?  Strip them off, then add them
@@ -293,12 +298,31 @@ chapter_title_strip_pattern:^[0-9]+[\.: -]+
 ## etc
 #chapter_title_strip_pattern:^([0-9]+[\.: -]+)?(Chapter *[0-9]+[\.:, -]*)?
 
-## Uses a python template substitution.  The ${index} is the 'chapter'
-## number and ${title} is the chapter title, after applying
-## chapter_title_strip_pattern.  Those are the only variables available.
-## "The Beginning" => "1. The Beginning" 
+## If true, when updating an epub that already has old chapters, new
+## chapters will be marked in the TOC and chapter header by using
+## chapter_title_new_pattern and chapter_title_addnew_pattern to set the chapter.
+mark_new_chapters:false
+
+## chapter title patterns use python template substitution.  The
+## ${index} is the 'chapter' number and ${title} is the chapter title,
+## after applying chapter_title_strip_pattern.  Those are the only
+## variables available.
+
+## The basic pattern used when not using add_chapter_numbers or
+## mark_new_chapters
+chapter_title_def_pattern:${title}
+
+## Pattern used with add_chapter_numbers, but not mark_new_chapters
 chapter_title_add_pattern:${index}. ${title}
 
+## Pattern used with mark_new_chapters, but not add_chapter_numbers
+## (new) is just text and can be changed.
+chapter_title_new_pattern:(new) ${title}
+
+## Pattern used with add_chapter_numbers and mark_new_chapters
+## (new) is just text and can be changed.
+chapter_title_addnew_pattern:${index}. (new) ${title}
+
 ## Uses a python template substitution.  The ${title} is the default
 ## title of a new anthology, <series name> in the case of a series, or
 ## the first book title otherwise.  This is only applied to new
@@ -330,6 +354,74 @@ sort_ships:false
 ## User-agent
 user_agent:FFF/2.X
 
+## Added for [base_xenforoforum], but can be used with other sites,
+## too.  Limit the 'description' to the first X *characters*
+## collected.  Character count includes HTML tags, so it can be
+## non-intuitive.
+#description_limit:1000
+
+[base_efiction]
+## At the time of writing, eFiction Base adapters allow downloading
+## the whole story in bulk using the 'Print' feature. If 'bulk_load'
+## is set to 'true', both metadata and chapters can be loaded in one
+## step
+bulk_load:true
+
+[base_xenforoforum]
+## Currently only forums.spacebattles.com and forums.sufficientvelocity.com
+
+cover_exclusion_regexp:/clear.png
+
+## I saw lots of chapters name simply '1.1' etc during testing.
+strip_chapter_numbers:false
+
+## Copy title to tagsfromtitle for parsing tags.
+add_to_extra_valid_entries:,tagsfromtitle
+
+## '.NOREPL' tells the system to *not* apply title's
+## in/exclude/replace_metadata -- Only works on include_in_ lines.
+include_in_tagsfromtitle:title.NOREPL
+
+tagsfromtitle_label:Tags from Title
+
+## might want to do this, maybe not.  Will often include category, but
+## also often include non-category stuff.
+# include_in_category:tagsfromtitle 
+
+add_to_include_metadata_pre:
+# only keep tagsfromtitle with ( or [ in.
+ tagsfromtitle=~[\[\(]
+ 
+add_to_replace_metadata:
+# remove anything outside () or []
+ tagsfromtitle=>^.*?([\(\[]([^\]\)]+)[\)\]]).*?$=>\1
+# remove () []
+ tagsfromtitle=>[\(\)\[\]]=>
+# change (spaces)slash(or semicolon)(spaces) to comma
+ tagsfromtitle=> *[/;] *=>,
+ tagsfromtitle=> [xX] =>,
+ 
+# remove [] or () blocks and leading/trailing spaces/dashes/colons
+ title=>[-: ]*[\(\[]([^\]\)]+)[\)\]][-: ]*=>
+# remove 'Thread' and the next word, usually "Thread 2", "Thread
+# four", "Thread iv", etc
+ title,tagsfromtitle=>[-: ]*[Tt]hread [^ ]+[-: ]*=>
+
+add_to_extra_titlepage_entries:,tagsfromtitle
+
+## '.SPLIT' tells the system to split by ','
+add_to_include_subject_tags:,tagsfromtitle.SPLIT
+
+## base_xenforoforum reads Published and Updated datetimes from
+## Threadmarks if used, or from the posted & updated times of the
+## 'first' post if no threadmarks.
+datePublished_format:%%Y-%%m-%%d %%H:%%M:%%S
+dateUpdated_format:%%Y-%%m-%%d %%H:%%M:%%S
+
+## Only take the first X characters of the 'first' post to use as
+## the description.
+description_limit:500
+
 ## Each output format has a section that overrides [defaults]
 [html]
 
@@ -578,6 +670,8 @@ extratags: FanFiction,Testing,HTML
 ## doesn't like that.  If do_update_hook is uncommented and set true,
 ## the adapter will discard all existing chapters from the newest one
 ## on when updating to enforce accurate chapters.
+## Starting July 2015, FFF stores chapter URLs in the chapter files.
+## Stories downloaded after that shouldn't need this setting anymore.
 #do_update_hook:false
 
 ## AO3 adapter defines a few extra metadata entries.
@@ -766,11 +860,6 @@ extraships:Spike/Buffy
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
-## Virtually all eFiction Base adapters allow downloading the whole story in
-## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
-## metadata and chapters can be loaded in one step
-bulk_load:true
-
 [dramione.org]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -951,11 +1040,6 @@ extraships:Harry Potter/Hermione Granger
 #username:YourName
 #password:yourpassword
 
-## Virtually all eFiction Base adapters allow downloading the whole story in
-## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
-## metadata and chapters can be loaded in one step
-bulk_load:true
-
 extra_valid_entries: readings,romance
 extra_titlepage_entries: readings,romance
 readings_label: Readings
@@ -1045,6 +1129,12 @@ extra_valid_entries:size
 # don't show twitter icon.
 cover_exclusion_regexp:/res/css/bir.png
 
+[forums.spacebattles.com]
+## see [base_xenforoforum]
+
+[forums.sufficientvelocity.com]
+## see [base_xenforoforum]
+
 [grangerenchanted.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -1103,11 +1193,6 @@ extracategories:Glee RPF
 extracharacters:Darren Criss, Chris Colfer
 extraships:Darren Criss/Chris Colfer
 
-## Virtually all eFiction Base adapters allow downloading the whole story in
-## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
-## metadata and chapters can be loaded in one step
-bulk_load:true
-
 [ksarchive.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:Star Trek
@@ -1127,11 +1212,6 @@ eroticatags_label:Erotica Tags
 extra_titlepage_entries: eroticatags
 
 [lotrfanfiction.com]
-## Virtually all eFiction Base adapters allow downloading the whole story in
-## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
-## metadata and chapters can be loaded in one step
-bulk_load:true
-
 extra_valid_entries: readings
 readings_label: Readings
 
@@ -1185,6 +1265,12 @@ extracategories:NCIS
 extracategories:Buffy: The Vampire Slayer
 extracharacters:Willow
 
+[ninelives.dark-solace.org]
+## Site dedicated to these categories/characters/ships
+extracategories:The Walking Dead
+extracharacters:Carol,Daryl
+extraships:Carol/Daryl
+
 [nocturnal-light.net]
 ## Extra metadata that this adapter knows about.  See [dramione.org]
 ## for examples of how to use them.
@@ -1381,11 +1467,6 @@ extracategories:Transgender
 ## confirm they are adult for adult content.
 #is_adult:true
 
-## Virtually all eFiction Base adapters allow downloading the whole story in
-## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
-## metadata and chapters can be loaded in one step
-bulk_load:true
-
 [thehexfiles.net]
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter
@@ -1405,11 +1486,6 @@ extraships:Harry Potter/Draco Malfoy
 ## personal.ini, not defaults.ini.
 #is_adult:true
 
-## Virtually all eFiction Base adapters allow downloading the whole story in
-## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
-## metadata and chapters can be loaded in one step
-bulk_load:true
-
 ## Site dedicated to these categories/characters/ships
 extracategories:Criminal Minds
 
@@ -1419,11 +1495,6 @@ extracategories:Criminal Minds
 ## personal.ini, not defaults.ini.
 #is_adult:true
 
-## Virtually all eFiction Base adapters allow downloading the whole story in
-## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
-## metadata and chapters can be loaded in one step
-bulk_load:true
-
 extra_valid_entries: readings,challenge
 extra_titlepage_entries: readings,challenge
 challenge_label: Challenge
@@ -1777,10 +1848,27 @@ extraships:InuYasha/Kagome
 ## Site dedicated to these categories/characters/ships
 extracategories:Lord of the Rings
 
-## Virtually all eFiction Base adapters allow downloading the whole story in
-## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
-## metadata and chapters can be loaded in one step
-bulk_load:true
+[www.masseffect2.in]
+## Site dedicated to this fandom.
+extracategories: Mass Effect
+
+## Ucoz.com engine, upon which MassEffect2.in is based, imposes an unspecified limit on request frequency.
+## Reports vary from `5 requests per second' to `2 requests per second for more than 10 per minute'.
+## With default settings, a several-hours IP ban may follow, so set it higher.
+slow_down_sleep_time: 2
+
+## Whether to exclude editor signature from the bottom of chapter text.
+exclude_editor_signature: false
+
+## Stories on the site almost never have cover image, and for the stories which do,
+## this may be adjusted in `personal.ini' before downloading.
+never_make_cover: true
+
+## Titles for ratings identified by 1- or 2-letter codes from `ERATING system'
+## (`система Р.Е.Й.Т.И.Н.Г.').  MassEffect2.in and some other sites adopted it,
+## but changed titles and update them occasionally.
+rating_titles: R=RESTRICTED (16+), E=EXEMPT (18+), I=ART HOUSE, T=To every, A=IN=Иной мир, Nn=Новый мир, G=О\, Господи!
+adult_ratings: E,R
 
 [www.mediaminer.org]
 

calibre-plugin/prefs.py

@@ -1,5 +1,5 @@
-#!/usr/bin/env python
-# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+# -*- coding: utf-8 -*-
+
 from __future__ import (unicode_literals, division, absolute_import,
                         print_function)
 
@@ -51,6 +51,7 @@ PREFS_KEY_SETTINGS = 'settings'
 # take from here.
 default_prefs = {}
 default_prefs['personal.ini'] = get_resources('plugin-example.ini')
+default_prefs['cal_cols_pass_in'] = False
 default_prefs['rejecturls'] = ''
 default_prefs['rejectreasons'] = '''Sucked
 Boring
@@ -75,6 +76,7 @@ default_prefs['lookforurlinhtml'] = False
 default_prefs['checkforseriesurlid'] = True
 default_prefs['checkforurlchange'] = True
 default_prefs['injectseries'] = False
+default_prefs['matchtitleauth'] = True
 default_prefs['smarten_punctuation'] = False
 default_prefs['show_est_time'] = False
 
@@ -83,6 +85,7 @@ default_prefs['read_lists'] = ''
 default_prefs['addtolists'] = False
 default_prefs['addtoreadlists'] = False
 default_prefs['addtolistsonread'] = False
+default_prefs['autounnew'] = False
 
 default_prefs['updatecalcover'] = None
 default_prefs['gencalcover'] = SAVE_YES
@@ -104,6 +107,7 @@ default_prefs['custom_cols_newonly'] = {}
 default_prefs['allow_custcol_from_ini'] = True
 
 default_prefs['std_cols_newonly'] = {}
+default_prefs['set_author_url'] = True
 
 default_prefs['imapserver'] = ''
 default_prefs['imapuser'] = ''
@@ -112,6 +116,7 @@ default_prefs['imapsessionpass'] = False
 default_prefs['imapfolder'] = 'INBOX'
 default_prefs['imapmarkread'] = True
 default_prefs['auto_reject_from_email'] = False
+default_prefs['download_from_email_immediately'] = False
 
 def set_library_config(library_config,db):
     db.prefs.set_namespaced(PREFS_NAMESPACE,

fanficfare/adapters/__init__.py

@@ -135,6 +135,11 @@ import adapter_fanfictionjunkiesde
 import adapter_devianthearts
 import adapter_tgstorytimecom
 import adapter_itcouldhappennet
+import adapter_forumsspacebattlescom
+import adapter_forumssufficientvelocitycom
+import adapter_questionablequestingcom
+import adapter_ninelivesdarksolaceorg
+import adapter_masseffect2in
 
 ## This bit of complexity allows adapters to be added by just adding
 ## importing.  It eliminates the long if/else clauses we used to need
@@ -194,14 +199,24 @@ def getAdapter(config,url,anyurl=False):
     # No adapter found.
     raise exceptions.UnknownSite( url, [cls.getSiteDomain() for cls in __class_list] )
 
-def getConfigSections():
+def getSiteSections():
+    # doesn't include base sections. Sections rather than site DNS because of squidge/peja
     return [cls.getConfigSection() for cls in __class_list]
 
+def getConfigSections():
+    # does include base sections.
+    sections = set()
+    for cls in __class_list:
+        sections.update(cls.getConfigSections())
+    return sections
+
 def get_bulk_load_sites():
     # for now, all eFiction Base adapters are assumed to allow bulk_load.
-    return [cls.getConfigSection().replace('www.','') for cls in
-            filter( lambda x : issubclass(x,base_efiction_adapter.BaseEfictionAdapter),
-                    __class_list)]
+    sections = set()
+    for cls in filter( lambda x : issubclass(x,base_efiction_adapter.BaseEfictionAdapter),
+                       __class_list):
+        sections.update( [ x.replace('www.','') for x in cls.getConfigSections() ] )
+    return sections
 
 def getSiteExamples():
     l=[]
@@ -209,10 +224,10 @@ def getSiteExamples():
         l.append((cls.getConfigSection(),cls.getSiteExampleURLs().split()))
     return l
 
-def getConfigSectionFor(url):
+def getConfigSectionsFor(url):
     (cls,fixedurl) = getClassFor(url)
     if cls:
-        return cls.getConfigSection()
+        return cls.getConfigSections()
 
     # No adapter found.
     raise exceptions.UnknownSite( url, [cls.getSiteDomain() for cls in __class_list] )

fanficfare/adapters/adapter_dramioneorg.py

@@ -38,8 +38,9 @@ class DramioneOrgAdapter(BaseSiteAdapter):
     def __init__(self, config, url):
         BaseSiteAdapter.__init__(self, config, url)
 
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
+        self.decode = ["utf8",
+                       "Windows-1252",]
+                               # 1252 is a superset of iso-8859-1.
                                # Most sites that claim to be
                                # iso-8859-1 (and some that claim to be
                                # utf8) are really windows-1252.

fanficfare/adapters/adapter_fanfictionnet.py

@@ -104,7 +104,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
             data = self._fetchUrl(url)
             #logger.debug("\n===================\n%s\n===================\n"%data)
             soup = self.make_soup(data)
-        except urllib2.HTTPError, e:
+        except urllib2.HTTPError as e:
             if e.code == 404:
                 raise exceptions.StoryDoesNotExist(url)
             else:
@@ -135,11 +135,15 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
                                                 chapcount+1)
                 logger.debug('=Trying newer chapter: %s' % tryurl)
                 newdata = self._fetchUrl(tryurl)
-                if "not found. Please check to see you are not using an outdated url." \
-                        not in newdata:
+                if "not found. Please check to see you are not using an outdated url." not in newdata \
+                        and "This request takes too long to process, it is timed out by the server." not in newdata:
                     logger.debug('=======Found newer chapter: %s' % tryurl)
                     soup = self.make_soup(newdata)
-            except:
+            except urllib2.HTTPError as e:
+                if e.code == 503:
+                    raise e
+            except e:
+                logger.warn("Caught an exception reading URL: %s sleeptime(%s) Exception %s."%(unicode(url),sleeptime,unicode(e)))
                 pass
             
         # Find authorid and URL from... author url.

fanficfare/adapters/adapter_forumsspacebattlescom.py

@@ -0,0 +1,35 @@
+#  -*- coding: utf-8 -*-
+
+# Copyright 2015 FanFicFare team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from base_xenforoforum_adapter import BaseXenForoForumAdapter
+
+def getClass():
+    return ForumsSpacebattlesComAdapter
+
+class ForumsSpacebattlesComAdapter(BaseXenForoForumAdapter):
+
+    def __init__(self, config, url):
+        BaseXenForoForumAdapter.__init__(self, config, url)
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','fsb')
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'forums.spacebattles.com'
+

fanficfare/adapters/adapter_forumssufficientvelocitycom.py

@@ -0,0 +1,34 @@
+#  -*- coding: utf-8 -*-
+
+# Copyright 2015 FanFicFare team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from base_xenforoforum_adapter import BaseXenForoForumAdapter
+
+def getClass():
+    return ForumsSufficientVelocityComAdapter
+
+class ForumsSufficientVelocityComAdapter(BaseXenForoForumAdapter):
+
+    def __init__(self, config, url):
+        BaseXenForoForumAdapter.__init__(self, config, url)
+
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','fsv')
+
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'forums.sufficientvelocity.com'

fanficfare/adapters/adapter_masseffect2in.py

@@ -0,0 +1,719 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2015 FanFicFare team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import bs4
+import datetime
+import logging
+import re
+import urllib2
+
+from ..htmlcleanup import removeEntities, stripHTML
+from .. import exceptions as exceptions
+from base_adapter import BaseSiteAdapter, makeDate
+
+
+_logger = logging.getLogger(__name__)
+
+
+def getClass():
+    """Returns adapter class defined in this module."""
+    return MassEffect2InAdapter
+
+
+class ParsingError(Exception):
+    """Indicates an error while parsing web page content."""
+    def __init__(self, message):
+        Exception.__init__(self)
+        self.message = message
+
+    def __str__(self):
+        return self.message
+
+
+class MassEffect2InAdapter(BaseSiteAdapter):
+    """
+    Provides support for MassEffect2.in site as story source.
+    Can be used as a template for sites build upon Ucoz.com engine (until no base class extracted).
+    Specializations:
+        1) Russian content (date format, genre names, etc.);
+        2) original `E.R.A.T.I.N.G.' rating scale, used by masseffect2.in
+           and some affiliated sites, denoted with images;
+        3) editor signatures an an option to remove them.
+    """
+
+    WORD_PATTERN = re.compile(u'\w+', re.UNICODE)
+    DOCUMENT_ID_PATTERN = re.compile(u'\d+-\d+-\d+-\d+')
+    SITE_LANGUAGE = u'Russian'
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["utf8"]
+
+        self.story.setMetadata('siteabbrev', 'me2in')
+        self.story.setMetadata('storyId', self._getDocumentId(self.url))
+
+        self._setURL(self._makeDocumentUrl(self.story.getMetadata('storyId')))
+
+        self._chapters = {}
+        self._parsingConfiguration = None
+
+    # Must be @staticmethod, not @classmethod!
+    @staticmethod
+    def getSiteDomain():
+        return 'www.masseffect2.in'
+
+    @classmethod
+    def getSiteExampleURLs(cls):
+        return u' '.join([cls._makeDocumentUrl('19-1-0-1234'),
+                          cls._makeDocumentUrl('24-1-0-4321')])
+
+    def getSiteURLPattern(self):
+        return re.escape(self._makeDocumentUrl('')) + self.DOCUMENT_ID_PATTERN.pattern
+
+    def use_pagecache(self):
+        """Allows use of downloaded page cache.  It is essential for this
+        adapter, because the site does not offers chapter URL list, and many
+        pages have to be fetched and parsed repeatedly."""
+        return True
+
+    def extractChapterUrlsAndMetadata(self):
+        """Extracts chapter URLs and story metadata.  Actually downloads all
+        chapters, which is not exactly right, but necessary due to technical
+        limitations of the site."""
+
+        def followChapters(starting, forward=True):
+            if forward:
+                url = starting.getNextChapterUrl()
+            else:
+                url = starting.getPreviousChapterUrl()
+            if url:
+                url = self._makeDocumentUrl(self._getDocumentId(url))
+                following = self._makeChapter(url)
+                # Do not follow links to related, but different stories (prequels or sequels).
+                if not following.isFromStory(starting.getHeading()):
+                    return
+                if forward:
+                    yield following
+                for chapter in followChapters(following, forward):
+                    yield chapter
+                if not forward:
+                    yield following
+
+        try:
+            startingChapter = self._makeChapter(self.url)
+        except urllib2.HTTPError, error:
+            if error.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            raise
+
+        # We only have one date for each chapter and assume the oldest one
+        # to be publication date and the most recent one to be update date.
+        datePublished = datetime.datetime.max
+        dateUpdated = datetime.datetime.min
+        wordCount = 0
+        # We aim at counting chapters, not chapter parts.
+        chapterCount = 0
+        storyInProgress = False
+
+        chapters = \
+            list(followChapters(startingChapter, forward=False)) + \
+            [startingChapter] + \
+            list(followChapters(startingChapter, forward=True))
+
+        headings = [chapter.getHeading() for chapter in chapters]
+        largestCommonPrefix = _getLargestCommonPrefix(*headings)
+        prefixLength = len(largestCommonPrefix)
+        storyTitleEnd, chapterTitleStart = prefixLength, prefixLength
+        match = re.search(u'[:\.\s]*(?P<chapter>глава\s+)?$', largestCommonPrefix, re.IGNORECASE | re.UNICODE)
+        if match:
+            storyTitleEnd -= len(match.group())
+            label = match.group('chapter')
+            if label:
+                chapterTitleStart -= len(label)
+        storyTitle = largestCommonPrefix[:storyTitleEnd]
+        self.story.setMetadata('title', storyTitle)
+
+        garbagePattern = re.compile(u'(?P<start>^)?[:\.\s]*(?(start)|$)', re.UNICODE)
+        indexPattern = re.compile(u'(?:глава\s)?(?:(?<!\d)(?P<index>\d{1,3})(?=\D|$))', re.IGNORECASE | re.UNICODE)
+
+        for chapter in chapters:
+            url = chapter.getUrl()
+            self._chapters[url] = chapter
+            _logger.debug(u"Processing chapter `%s'.", url)
+
+            try:
+                authorName = chapter.getAuthorName()
+                if authorName:
+                    self.story.extendList('author', [authorName])
+                    authorId = chapter.getAuthorId()
+                    if authorId:
+                        authorUrl = 'http://%s/index/%s' % (self.getSiteDomain(), authorId)
+                    else:
+                        authorId = u''
+                        authorUrl = u''
+                    self.story.extendList('authorId', [authorId])
+                    self.story.extendList('authorUrl', [authorUrl])
+
+                if not self.story.getMetadata('rating'):
+                    ratingTitle = chapter.getRatingTitle()
+                    if ratingTitle:
+                        self.story.setMetadata('rating', ratingTitle)
+
+                if not self.story.getMetadata('description'):
+                    summary = chapter.getSummary()
+                    if summary:
+                        self.story.setMetadata('description', summary)
+
+                datePublished = min(datePublished, chapter.getDate())
+                dateUpdated = max(dateUpdated, chapter.getDate())
+
+                self.story.extendList('genre', chapter.getGenres())
+                self.story.extendList('characters', chapter.getCharacters())
+                self.story.extendList('ships', chapter.getPairings())
+
+                wordCount += self._getWordCount(chapter.getTextElement())
+
+                # Chapter status usually represents the story status, so we want the last chapter status.
+                # Some chapters may have no status attribute.
+                chapterInProgress = chapter.isInProgress()
+                if chapterInProgress is not None:
+                    storyInProgress = chapterInProgress
+
+                # If any chapter is adult, consider the whole story adult.
+                if chapter.isAdult():
+                    self.story.setMetadata('is_adult', True)
+                    warning = chapter.getWarning()
+                    if warning:
+                        self.story.extendList('warnings', [warning])
+
+                chapterTitle = re.sub(garbagePattern, u'', chapter.getHeading()[chapterTitleStart:])
+
+                match = re.search(indexPattern, chapterTitle)
+                if match:
+                    index = int(match.group('index'))
+                    chapterCount = max(chapterCount, index)
+                else:
+                    chapterCount += 1
+
+                self.chapterUrls.append((chapterTitle, url))
+            except ParsingError, error:
+                raise exceptions.FailedToDownload(u"Failed to download chapter `%s': %s" % (url, error))
+
+        # Some metadata are handled separately due to format conversions.
+        self.story.setMetadata('status', 'In Progress' if storyInProgress else 'Completed')
+        self.story.setMetadata('datePublished', datePublished)
+        self.story.setMetadata('dateUpdated', dateUpdated)
+        self.story.setMetadata('numWords', str(wordCount))
+        self.story.setMetadata('numChapters', chapterCount)
+
+        # Site-specific metadata.
+        self.story.setMetadata('language', self.SITE_LANGUAGE)
+
+    def getChapterText(self, url):
+        """Grabs the text for an individual chapter."""
+        if url not in self._chapters:
+            raise exceptions.FailedToDownload(u"No chapter `%s' present!" % url)
+        chapter = self._chapters[url]
+        return self.utf8FromSoup(url, chapter.getTextElement())
+
+    def _makeChapter(self, url):
+        """Creates a chapter object given a URL."""
+        document = self.make_soup(self._fetchUrl(url))
+        chapter = Chapter(self._getParsingConfiguration(), url, document)
+        return chapter
+
+    def _getWordCount(self, element):
+        """Returns word count in plain text extracted from chapter body."""
+        text = stripHTML(element)
+        count = len(re.findall(self.WORD_PATTERN, text))
+        return count
+
+    def _getParsingConfiguration(self):
+        if not self._parsingConfiguration:
+            self._parsingConfiguration = {}
+
+            adultRatings = self.getConfigList('adult_ratings')
+            if not adultRatings:
+                raise exceptions.PersonalIniFailed(
+                    u"Missing `adult_ratings' setting", u"MassEffect2.in", u"?")
+            adultRatings = set(adultRatings)
+            self._parsingConfiguration['adultRatings'] = adultRatings
+
+            ratingTitleDescriptions = self.getConfigList('rating_titles')
+            if ratingTitleDescriptions:
+                ratingTitles = {}
+                for ratingDescription in ratingTitleDescriptions:
+                    parts = ratingDescription.split(u'=')
+                    if len(parts) < 2:
+                        _logger.warning(
+                            u"Invalid `rating_titles' setting, missing `=' in `%s'."
+                            % ratingDescription)
+                        continue
+                    labels = parts[:-1]
+                    title = parts[-1]
+                    for label in labels:
+                        ratingTitles[label] = title
+                        # Duplicate label aliasing in adult rating set.
+                        if label in adultRatings:
+                            adultRatings.add(*labels)
+                self._parsingConfiguration['adultRatings'] = list(adultRatings)
+                self._parsingConfiguration['ratingTitles'] = ratingTitles
+            else:
+                raise exceptions.PersonalIniFailed(
+                    u"Missing `rating_titles' setting", u"MassEffect2.in", u"?")
+
+            self._parsingConfiguration['excludeEditorSignature'] = \
+                self.getConfig('exclude_editor_signature', False)
+
+        return self._parsingConfiguration
+
+    def _getDocumentId(self, url):
+        """Extract document ID from MassEffect2.in URL."""
+        match = re.search(self.DOCUMENT_ID_PATTERN, url)
+        if not match:
+            raise ValueError(u"Failed to extract document ID from `'" % url)
+        documentId = url[match.start():match.end()]
+        return documentId
+
+    @classmethod
+    def _makeDocumentUrl(cls, documentId):
+        """Make a chapter URL given a document ID."""
+        return 'http://%s/publ/%s' % (cls.getSiteDomain(), documentId)
+
+
+class Chapter(object):
+    """Represents a lazily-parsed chapter of a story."""
+    def __init__(self, configuration, url, document):
+        self._configuration = configuration
+        self._url = url
+        self._document = document
+        # Lazy-loaded:
+        self._heading = None
+        self._date = None
+        self._author = None
+        self._attributes = None
+        self._textElement = None
+        self._infoBar = None
+
+    def getHeading(self):
+        return self._extractHeading()
+
+    def getSummary(self):
+        attributes = self.__getAttributes()
+        if 'summary' in attributes:
+            return attributes['summary']
+
+    def getAuthorId(self):
+        author = self._getAuthor()
+        if author:
+            return author['id']
+
+    def getAuthorName(self):
+        author = self._getAuthor()
+        if author:
+            return author['name']
+
+    def getDate(self):
+        return self.__getDate()
+
+    def getRatingTitle(self):
+        attributes = self.__getAttributes()
+        if 'rating' in attributes:
+            return attributes['rating']['title']
+
+    def isAdult(self):
+        attributes = self.__getAttributes()
+        if 'rating' in attributes and attributes['rating']['isAdult']:
+            return True
+        if 'warning' in attributes:
+            return True
+        return False
+
+    def getWarning(self):
+        attributes = self.__getAttributes()
+        if 'warning' in attributes:
+            return attributes['warning']
+
+    def getCharacters(self):
+        return self._getListAttribute('characters')
+
+    def getPairings(self):
+        return self._getListAttribute('pairings')
+
+    def getGenres(self):
+        return self._getListAttribute('genres')
+
+    def isInProgress(self):
+        attributes = self.__getAttributes()
+        if 'isInProgress' in attributes:
+            return attributes['isInProgress']
+
+    def getUrl(self):
+        return self._url
+
+    def getTextElement(self):
+        return self._getTextElement()
+
+    def getPreviousChapterUrl(self):
+        """Download chapters following `Previous chapter' links.
+        Returns a list of chapters' URLs."""
+        return self._getSiblingChapterUrl({'class': 'fl tal'})
+
+    def getNextChapterUrl(self):
+        """Download chapters following `Next chapter' links.
+        Returns a list of chapters' URLs."""
+        return self._getSiblingChapterUrl({'class': 'tar fr'})
+
+    def isFromStory(self, storyTitle, prefixThreshold=-1):
+        """Check if this chapter is from a story different from the given one.
+        Prefix threshold specifies how long common story title prefix shall be
+        for chapters from one story: negative value means implementation-defined
+        optimum, zero inhibits the check, and positive value adjusts threshold."""
+
+        def getFirstWord(string):
+            match = re.search(u'^\s*\w+', string, re.UNICODE)
+            return string[match.start():match.end()]
+
+        thisStoryTitle = self.getHeading()
+        if prefixThreshold != 0:
+            if prefixThreshold < 0:
+                prefixThreshold = min(
+                    len(getFirstWord(storyTitle)), len(getFirstWord(thisStoryTitle)))
+            else:
+                prefixThreshold = min(
+                    prefixThreshold, len(storyTitle), len(thisStoryTitle))
+            result = len(_getLargestCommonPrefix(storyTitle, thisStoryTitle)) >= prefixThreshold
+            return result
+        else:
+            return storyTitle != thisStoryTitle
+
+    def _getListAttribute(self, name):
+        """Return an attribute value as a list or an empty list if the attribute is absent."""
+        attributes = self.__getAttributes()
+        if name in attributes:
+            return attributes[name]
+        return []
+
+    def _extractHeading(self):
+        """Extract header text from the document."""
+        return stripHTML(
+            self._document.find('div', {'class': 'eTitle'}).string)
+
+    def __getHeading(self):
+        """Lazily parse and return heading."""
+        if not self._heading:
+            self._heading = self._extractHeading()
+        return self._heading
+
+    def _getAuthor(self):
+        """Lazily parse and return author's information."""
+        if not self._author:
+            self._author = self._parseAuthor()
+        return self._author
+
+    def _parseAuthor(self):
+        """Locate and parse chapter author's information to a dictionary with author's `id' and `name'."""
+        try:
+            authorLink = self._getInfoBarElement() \
+                .find('i', {'class': 'icon-user'}) \
+                .findNextSibling('a')
+        except AttributeError:
+            raise ParsingError(u'Failed to locate author link.')
+        match = re.search(u'(8-\d+)', authorLink['onclick'])
+        if not match:
+            raise ParsingError(u'Failed to extract author ID.')
+        authorId = match.group(0)
+        authorName = stripHTML(authorLink.text)
+        return {
+            'id': authorId,
+            'name': authorName
+        }
+
+    def __getDate(self):
+        """Lazily parse chapter date."""
+        if not self._date:
+            self._date = self._parseDate()
+        return self._date
+
+    def _parseDate(self):
+        """Locate and parse chapter date."""
+        try:
+            dateText = self._getInfoBarElement() \
+                .find('i', {'class': 'icon-eye'}) \
+                .findPreviousSibling(text=True) \
+                .strip(u'| \n')
+        except AttributeError:
+            raise ParsingError(u'Failed to locate date.')
+        date = makeDate(dateText, '%d.%m.%Y')
+        return date
+
+    def _getInfoBarElement(self):
+        """Locate informational bar element, containing chapter date and author, on the page."""
+        if not self._infoBar:
+            self._infoBar = self._document.find('td', {'class': 'eDetails2'})
+            if not self._infoBar:
+                raise ParsingError(u'No informational bar found.')
+        return self._infoBar
+
+    def __getAttributes(self):
+        """Lazily parse attributes."""
+        if not self._attributes:
+            self._attributes = self._parseAttributes()
+        return self._attributes
+
+    def _parseAttributes(self):
+        """Parse chapter attribute block and return it as a dictionary with standard entries."""
+
+        attributes = {}
+        attributesText = u''
+        try:
+            starter = self._document \
+                .find('div', {'class': 'comm-div'}) \
+                .findNextSibling('div', {'class': 'cb'})
+            bound = starter.findNextSibling('div', {'class': 'cb'})
+
+            def processElement(element):
+                """Return textual representation an *inline* element of chapter attribute block."""
+                result = u''
+                if isinstance(element, bs4.Tag):
+                    if element.name in ('b', 'strong', 'font', 'br'):
+                        result += u"\n"
+                    if element.name == 's':
+                        result += u"<s>%s</s>" % stripHTML(element)
+                    else:
+                        result += stripHTML(element)
+                else:
+                    result += removeEntities(element)
+                return result
+
+            elements = starter.nextSiblingGenerator()
+            for element in elements:
+                if isinstance(element, bs4.Tag):
+                    if element == bound:
+                        break
+                    else:
+                        if element.name in ('div', 'p'):
+                            attributesText += u"\n"
+                            for child in element.childGenerator():
+                                attributesText += processElement(child)
+                            continue
+                attributesText += processElement(element)
+
+            elements = starter.nextGenerator()
+            for element in elements:
+                if isinstance(element, bs4.Tag):
+                    if element == bound:
+                        break
+                    elif element.name == 'img':
+                        rating = self._parseRatingFromImage(element)
+                        if rating:
+                            attributes['rating'] = rating
+                            break
+        except AttributeError or TypeError:
+            raise ParsingError(u'Failed to locate and collect attributes.')
+
+        separators = u"\r\n :;."
+        freestandingText = u''
+        for line in attributesText.split(u'\n'):
+            if line.count(u':') != 1:
+                freestandingText += line
+                continue
+            key, value = line.split(u':', 1)
+            key = key.strip(separators).lower()
+            value = value.strip().strip(separators)
+            parsed = self._parseAttribute(key, value)
+            for parsedKey, parsedValue in parsed.iteritems():
+                attributes[parsedKey] = parsedValue
+
+        freestandingText = freestandingText.strip()
+        if 'summary' not in attributes and freestandingText:
+            attributes['summary'] = freestandingText
+
+        if 'rating' not in attributes:
+            _logger.warning(u"Failed to locate or recognize rating for `%s'!", self.getUrl())
+
+        return attributes
+
+    # Most, but not all, URLs of rating icons match this.
+    RATING_LABEL_PATTERN = re.compile(u'/(?P<rating>[ERATINnG]+)\.png$')
+
+    def _parseRatingFromImage(self, element):
+        """Given an image element, try to parse story rating from it."""
+        # Although deprecated, `has_key()' is required here.
+        if not element.has_attr('src'):
+            return
+        source = element['src']
+        if 'REITiNG' in source:
+            match = re.search(self.RATING_LABEL_PATTERN, source)
+            if not match:
+                return
+            label = match.group('rating')
+            if label in self._configuration['ratingTitles']:
+                return {
+                    'label': label,
+                    'title': self._configuration['ratingTitles'][label],
+                    'isAdult': label in self._configuration['adultRatings']
+                }
+            else:
+                _logger.warning(u"No title found for rating label `%s'!" % label)
+        # TODO: conduct a research on such abnormal URLs.
+        elif source == 'http://www.masseffect2.in/_fr/10/1360399.png':
+            label = 'Nn'
+            return {
+                'label': 'Nn',
+                'title': self._configuration['ratingTitles'][label],
+                'isAdult': label in self._configuration['adultRatings']
+            }
+
+    # Various `et cetera' and `et al' forms in Russian texts.
+    # Intended to be used with whole strings!
+    ETC_PATTERN = re.compile(
+        u'''[и&]\s(?:
+              (?:т\.?\s?[пд]?\.?)|
+              (?:др(?:угие|\.)?)|
+              (?:пр(?:очие|\.)?)|
+              # Note: identically looking letters `K' and `o'
+              # below are from Latin and Cyrillic alphabets.
+              (?:ко(?:мпания)?|[KК][oо°])
+            )$
+        ''',
+        re.IGNORECASE + re.UNICODE + re.VERBOSE)
+
+    # `Author's Notes' and its variants in Russian.
+    ANNOTATION_PATTERN = re.compile(u'аннотация|описание|(?:(?:за|при)мечание\s)?(?:от\s)?автора', re.UNICODE)
+
+    def _parseAttribute(self, key, value):
+        """
+        Parse a single a single record in chapter attributes for chapter metadata.
+        Return a dictionary of canonical attributes and values (i. e. multiple attributes may be discovered).
+        """
+
+        def refineCharacter(name):
+            """Refines character name from stop-words and distortions."""
+            strippedName = name.strip()
+            nameOnly = re.sub(self.ETC_PATTERN, u'', strippedName)
+            # TODO: extract canonical name (even ME-specific?).
+            canonicalName = nameOnly
+            return canonicalName
+
+        if re.match(u'жанры?', key, re.UNICODE):
+            genres = filter(bool, map(unicode.strip, re.split(u'[,;/]', value)))
+            return {'genres': genres}
+        elif key == u'статус':
+            isInProgress = value == u'в процессе'
+            return {'isInProgress': isInProgress}
+        elif key == u'персонажи':
+            participants = map(refineCharacter, re.split(u'[,;]', value))
+            characters = []
+            pairings = []
+            for participant in participants:
+                if u'/' in participant:
+                    pairings.append(participant)
+                else:
+                    characters.append(participant)
+            return {
+                'characters': characters,
+                'pairings': pairings
+            }
+        elif key == u'предупреждение':
+            return {'warning': value}
+        elif re.match(self.ANNOTATION_PATTERN, key):
+            if not value.endswith(u'.'):
+                value += u'.'
+            # Capitalize would make value[1:] lowercase, which we don't want.
+            value = value[:1].upper() + value[1:]
+            return {'summary': value}
+        else:
+            _logger.info(u"Unrecognized attribute `%s' ignored.", key)
+            return {}
+
+    def _getTextElement(self):
+        """Locate chapter body text element on the page."""
+        if not self._textElement:
+            self._textElement = self.__collectTextElements()
+        return self._textElement
+
+    def __collectTextElements(self):
+        """Return all elements containing parts of chapter text (which may be
+        <p>aragraphs, <div>isions or plain text nodes) under a single root."""
+        starter = self._document.find('div', {'id': u'article'})
+        if starter is None:
+            # FIXME: This will occur if the method is called more than once.
+            # The reason is elements appended to `root' are removed from the document.
+            # BS 4.4 implements cloning via `copy.copy()', but supporting it for BS 4.3
+            # would be error-prone (due to relying on BS internals) and is not needed.
+            if self._textElement:
+                _logger.debug(u"You may not call this function more than once!")
+            raise ParsingError(u'Failed to locate text.')
+        collection = [starter]
+        for element in starter.nextSiblingGenerator():
+            if element is None:
+                break
+            if isinstance(element, bs4.Tag) and element.name == 'tr':
+                break
+            collection.append(element)
+        root = bs4.Tag(name='td')
+        for element in collection:
+            root.append(element)
+
+        if self._configuration['excludeEditorSignature']:
+            root = self._excludeEditorSignature(root)
+
+        return root
+
+    def _getSiblingChapterUrl(self, selector):
+        """Locate a link to a sibling chapter, either previous or next one, and return its URL."""
+        block = self._document \
+            .find('td', {'class': 'eDetails1'}) \
+            .find('div', selector)
+        if not block:
+            return
+        link = block.find('a')
+        if not link:
+            return
+        return link['href']
+
+    # Editor signature always starts with something like this.
+    SIGNED_PATTERN = re.compile(u'отредактирова(?:но|ла?)[:.\s]', re.IGNORECASE + re.UNICODE)
+
+    def _excludeEditorSignature(self, root):
+        """Exclude editor signature from within `root' element."""
+        for textNode in root.findAll(text=True):
+            if re.match(self.SIGNED_PATTERN, textNode.string):
+                editorLink = textNode.findNext('a')
+                if editorLink:
+                    editorLink.extract()
+                # Seldom editor link has inner formatting, which is sibling DOM-wise.
+                editorName = textNode.findNext('i')
+                if editorName:
+                    editorName.extract()
+                textNode.extract()
+                # We could try removing container element, but there is a risk
+                # of removing text ending with it.  Better play safe here.
+                break
+        return root
+
+
+def _getLargestCommonPrefix(*args):
+    """Returns largest common prefix of all unicode arguments, ignoring case.
+    :rtype : unicode
+    """
+    from itertools import takewhile, izip
+    toLower = lambda xs: map(lambda x: x.lower(), xs)
+    allSame = lambda xs: len(set(toLower(xs))) == 1
+    return u''.join([i[0] for i in takewhile(allSame, izip(*args))])

fanficfare/adapters/adapter_mediaminerorg.py

@@ -22,7 +22,6 @@ import re
 import urllib
 import urllib2
 
-from .. import BeautifulSoup as bs
 from ..htmlcleanup import stripHTML
 from .. import exceptions as exceptions
 
@@ -42,7 +41,12 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
         # get storyId from url--url validation guarantees query correct
         m = re.match(self.getSiteURLPattern(),url)
         if m:
-            self.story.setMetadata('storyId',m.group('id'))
+            if m.group('id'):
+                self.story.setMetadata('storyId',m.group('id'))
+            elif m.group('id2'):
+                self.story.setMetadata('storyId',m.group('id2'))
+            elif m.group('id3'):
+                self.story.setMetadata('storyId',m.group('id2'))
             
             # normalized story URL.
             self._setURL('http://' + self.getSiteDomain() + '/fanfic/view_st.php/'+self.story.getMetadata('storyId'))
@@ -62,8 +66,17 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
     def getSiteURLPattern(self):
         ##  http://www.mediaminer.org/fanfic/view_st.php/76882
         ##  http://www.mediaminer.org/fanfic/view_ch.php/167618/594087#fic_c
+        ##  http://www.mediaminer.org/fanfic/view_ch.php?submit=View+Chapter&id=105816&cid=357151
+        ##  http://www.mediaminer.org/fanfic/view_ch.php?cid=612153&submit=View+Chapter&id=171668
         return re.escape("http://"+self.getSiteDomain())+\
-            "/fanfic/view_(st|ch)\.php/"+r"(?P<id>\d+)(/\d+(#fic_c)?)?$"
+            r"/fanfic/view_(st|ch)\.php"+\
+            r"(/(?P<id>\d+)(/\d+(#fic_c)?)?/?|"+\
+            r"\?((submit=View(\+| )Chapter|id=(?P<id2>\d+)|cid=\d+)&?)+)"
+
+    # Override stripURLParameters so the id parameter won't get stripped
+    @classmethod
+    def stripURLParameters(cls, url):
+        return url
 
     def extractChapterUrlsAndMetadata(self):
 
@@ -71,7 +84,7 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
         logger.debug("URL: "+url)
 
         try:
-            data = self._fetchUrl(url)
+            data = self._fetchUrl(url+'/') # trailing / gets 'chapter list' page even for one-shots.
         except urllib2.HTTPError, e:
             if e.code == 404:
                 raise exceptions.StoryDoesNotExist(self.url)
@@ -79,7 +92,7 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
                 raise e
 
         # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
+        soup = self.make_soup(data)
 
         # [ A - All Readers ], strip '[' ']'
         ## Above title because we remove the smtxt font to get title.
@@ -106,18 +119,12 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
         title = soup.find('td',{'class':'ffh'})
         for font in title.findAll('font'):
             font.extract() # removes 'font' tags from inside the td.        
-        if title.has_key('colspan'):
+        if title.has_attr('colspan'):
             titlet = stripHTML(title)
         else:
             ## No colspan, it's part chapter title--even if it's a one-shot.
             titlet = ':'.join(stripHTML(title).split(':')[:-1]) # strip trailing 'Chapter X' or chapter title
         self.story.setMetadata('title',titlet)
-        ## The story title is difficult to reliably parse from the
-        ## story pages.  Getting it from the author page is, but costs
-        ## another fetch.
-        # authsoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
-        # titlea = authsoup.find('a',{'href':'/fanfic/view_st.php/'+self.story.getMetadata('storyId')})
-        # self.story.setMetadata('title',titlea.text)
 
         # save date from first for later.
         firstdate=None
@@ -137,7 +144,9 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
                 # save date from first for later.
                 if not firstdate:
                     firstdate = m.group(3)
-                self.chapterUrls.append((chapter,'http://'+self.host+'/fanfic/view_ch.php/'+self.story.getMetadata('storyId')+'/'+option['value']))
+                # http://www.mediaminer.org/fanfic/view_ch.php?cid=376587&submit=View+Chapter&id=105816
+                # self.chapterUrls.append((chapter,'http://'+self.host+'/fanfic/view_ch.php/'+self.story.getMetadata('storyId')+'/'+option['value']))
+                self.chapterUrls.append((chapter,'http://'+self.host+'/fanfic/view_ch.php?submit=View Chapter&id='+self.story.getMetadata('storyId')+'&cid='+option['value']))
         self.story.setMetadata('numChapters',len(self.chapterUrls))
 
         # category
@@ -193,38 +202,37 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
         logger.debug('Getting chapter text from: %s' % url)
 
         data=self._fetchUrl(url)
-        soup = bs.BeautifulStoneSoup(data,
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        soup = self.make_soup(data)
 
-        anchor = soup.find('a',{'name':'fic_c'})
+        header = soup.find('div',{'class':'post-meta clearfix '})
+        # print("data:%s"%data)
 
-        if None == anchor:
+        chapter=self.make_soup('<div class="story"></div>').find('div')
+
+        if None == header:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
 
         ## find divs with align=left, those are paragraphs in newer stories.
-        divlist = anchor.findAllNext('div',{'align':'left'})
+        divlist = header.findAllNext('div',{'align':'left'})
         if divlist:
             for div in divlist:
                 div.name='p' # convert to <p> mediaminer uses div with
                              # a margin for paragraphs.
-                anchor.append(div) # cheat!  stuff all the content
-                                   # divs into anchor just as a
-                                   # holder.
+                chapter.append(div)
                 del div['style']
                 del div['align']
-            anchor.name='div'
-            return self.utf8FromSoup(url,anchor)
+            return self.utf8FromSoup(url,chapter)
         
         else:
             logger.debug('Using kludgey text find for older mediaminer story.')
             ## Some older mediaminer stories are unparsable with BeautifulSoup.
             ## Really nasty formatting.  Sooo... Cheat!  Parse it ourselves a bit first.
             ## Story stuff falls between:
-            data = "<div id='HERE'>" + data[data.find('<a name="fic_c">'):] +"</div>"
-            soup = bs.BeautifulStoneSoup(data,
-                                         selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+            data = "<div id='HERE'>" + data[data.find('<div class="adWrap">'):data.find('<div class="addthis_sharing_toolbox">')] +"</div>"
+            soup = self.make_soup(data)
             for tag in soup.findAll('td',{'class':'ffh'}) + \
                     soup.findAll('div',{'class':'acl'}) + \
+                    soup.findAll('div',{'class':'adWrap'}) + \
                     soup.findAll('div',{'class':'footer smtxt'}) + \
                     soup.findAll('table',{'class':'tbbrdr'}):
                 tag.extract() # remove tag from soup.

fanficfare/adapters/adapter_nhamagicalworldsus.py

@@ -16,222 +16,24 @@
 #
 
 # Software: eFiction
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-from .. import BeautifulSoup as bs
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return NHAMagicalWorldsUsAdapter
+from base_efiction_adapter import BaseEfictionAdapter
 
 # Class name has to be unique.  Our convention is camel case the
 # sitename with Adapter at the end.  www is skipped.
-class NHAMagicalWorldsUsAdapter(BaseSiteAdapter):
+class NHAMagicalWorldsUsAdapter(BaseEfictionAdapter):
 
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-        
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-        
-		
-        
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
-        
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','nha')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = " %d/%m/%y"
-            
-    @staticmethod # must be @staticmethod, don't remove it.
+    @staticmethod
     def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
         return 'nha.magical-worlds.us'
 
     @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-        url = self.url
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-            
-        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
-        if m != None:
-            if self.is_adult or self.getConfig("is_adult"):
-                # We tried the default and still got a warning, so
-                # let's pull the warning number from the 'continue'
-                # link and reload data.
-                addurl = m.group(1)
-                # correct stupid & error in url.
-                addurl = addurl.replace("&","&")
-                url = self.url+'&index=1'+addurl
-                logger.debug("URL 2nd try: "+url)
-
-                try:
-                    data = self._fetchUrl(url)
-                except urllib2.HTTPError, e:
-                    if e.code == 404:
-                        raise exceptions.StoryDoesNotExist(self.url)
-                    else:
-                        raise e    
-            else:
-                raise exceptions.AdultCheckRequired(self.url)
-            
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-            
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = bs.BeautifulSoup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-            
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-        asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')))
-		
-        try:
-            # in case link points somewhere other than the first chapter
-            a = soup.findAll('option')[1]['value']
-            self.story.setMetadata('storyId',a.split('=',)[1])
-            url = 'http://'+self.host+'/'+a
-            soup = bs.BeautifulSoup(self._fetchUrl(url))
-        except:
-            pass
-		
-        for info in asoup.findAll('table', {'width' : '100%', 'bordercolor' : re.compile(r'#')}):
-            a = info.find('a')
-            if 'viewstory.php?sid='+self.story.getMetadata('storyId') == a['href'] or \
-                    ('viewstory.php?sid='+self.story.getMetadata('storyId')+'&') in a['href']:
-                self.story.setMetadata('title',stripHTML(a))
-                break
-		
-
-        # Find the chapters:
-        chapters=soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+'&chapter=\d+$'))
-        if len(chapters) == 0:
-            self.chapterUrls.append((self.story.getMetadata('title'),url))
-        else:
-            for chapter in chapters:
-                # just in case there's tags, like <i> in chapter titles.
-                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d):
-            try:
-                return d.name
-            except:
-                return ""
-				
-        cats = info.findAll('a',href=re.compile('categories.php'))
-        for cat in cats:
-            self.story.addToList('category',cat.string)
-					
-        a = info.find('a', href=re.compile(r'viewuser.php'))
-        val = a.nextSibling
-        svalue = ""
-        while not defaultGetattr(val) == 'br':
-            val = val.nextSibling
-        val = val.nextSibling
-        while not defaultGetattr(val) == 'br':
-            svalue += unicode(val)
-            val = val.nextSibling
-        self.setDescription(url,svalue)
+    def getSiteAbbrev(self):
+        return 'nha'
         
-        #does not provide convenient way to get word count
-        labels = info.findAll('i')
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = stripHTML(labelspan)
-
-            if 'Rating' in label:
-                self.story.setMetadata('rating', value.split(' -')[0])
-
-            if 'Genres' in label:
-                genres = value.string.split(', ')
-                for genre in genres:
-                    if 'None' not in genre:
-                        self.story.addToList('genre',genre.split(' -')[0])
-            
-            if 'Characters' in label:
-                chars = value.string.split(', ')
-                for char in chars:
-                    if 'None' not in char:
-                        self.story.addToList('characters',char.split(' -')[0])
-
-            if 'Warnings' in label:
-                warnings = value.string.split(', ')
-                for warning in warnings:
-                    if 'None' not in warning:
-                        self.story.addToList('warnings',warning.split(' -')[0])
-
-            if 'Completed' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(value.split(' -')[0], self.dateformat))
-            
-            if 'Updated' in label:
-                # there's a stray [ at the end.
-                #value = value[0:-1]
-                self.story.setMetadata('dateUpdated', makeDate(value.split(' -')[0], self.dateformat))
-
-            
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-		
-        data = self._fetchUrl(url)
-
-        soup = bs.BeautifulSoup(data, selfClosingTags=('br','hr','span','center')) # some chapters seem to be hanging up on those tags, so it is safer to close them
-        
-        story = soup.find('div', {"id" : "story"})
-
-        if None == story:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    @classmethod
+    def getDateFormat(self):
+        return "%d/%m/%y" 
+                           
+def getClass():
+    return NHAMagicalWorldsUsAdapter
     
-        return self.utf8FromSoup(url,story)

fanficfare/adapters/adapter_ninelivesdarksolaceorg.py

@@ -0,0 +1,38 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2015 Fanficdownloader team, 2015 FanFicFare team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Software: eFiction
+import re
+from base_efiction_adapter import BaseEfictionAdapter
+
+class NineLivesDarkSolaceAdapter(BaseEfictionAdapter):
+
+    @staticmethod
+    def getSiteDomain():
+        return 'ninelives.dark-solace.org'
+
+    @classmethod
+    def getSiteAbbrev(self):
+        return '9lvs'
+        
+    @classmethod
+    def getDateFormat(self):
+        return "%B %d, %Y" 
+                           
+def getClass():
+    return NineLivesDarkSolaceAdapter
+    

fanficfare/adapters/adapter_portkeyorg.py

@@ -258,6 +258,7 @@ class PortkeyOrgAdapter(BaseSiteAdapter): # XXX
         data = self._fetchUrl(url)
 
         data = data.replace("HTML>","div>")
+        data = data.replace("html>","div>")
         
         soup = bs.BeautifulSoup(data)
 

fanficfare/adapters/adapter_questionablequestingcom.py

@@ -0,0 +1,35 @@
+#  -*- coding: utf-8 -*-
+
+# Copyright 2015 FanFicFare team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from base_xenforoforum_adapter import BaseXenForoForumAdapter
+
+def getClass():
+    return QuestionablequestingComAdapter
+
+class QuestionablequestingComAdapter(BaseXenForoForumAdapter):
+
+    def __init__(self, config, url):
+        BaseXenForoForumAdapter.__init__(self, config, url)
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','qq')
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'questionablequesting.com'
+

fanficfare/adapters/adapter_spikeluvercom.py

@@ -106,7 +106,9 @@ class SpikeluverComAdapter(BaseSiteAdapter):
 
         listbox_tag = soup.find('div', {'class': 'listbox'})
         for span_tag in listbox_tag('span'):
-            key = span_tag.string.strip(' :')
+            key = span_tag.string
+            if key:
+                key = key.strip(' :')
             try:
                 value = stripHTML(span_tag.nextSibling)
             # This can happen with some fancy markup in the summary. Just
@@ -135,8 +137,10 @@ class SpikeluverComAdapter(BaseSiteAdapter):
                         contents.append(sibling)
 
                 # Remove the preceding break line tag and other crud
-                contents.pop()
-                contents.pop()
+                if contents:
+                    contents.pop()
+                if contents:
+                    contents.pop()
                 self.story.setMetadata('description', ''.join(contents))
 
             elif key == 'Rated':

fanficfare/adapters/base_adapter.py

@@ -104,6 +104,8 @@ class BaseSiteAdapter(Configurable):
         self.chapterFirst = None
         self.chapterLast = None
         self.oldchapters = None
+        self.oldchaptersmap = None
+        self.oldchaptersdata = None
         self.oldimgs = None
         self.oldcover = None # (data of existing cover html, data of existing cover image)
         self.calibrebookmark = None
@@ -140,28 +142,6 @@ class BaseSiteAdapter(Configurable):
         '''
         self.get_cookiejar().load(filename, ignore_discard=True, ignore_expires=True)
         
-    # def save_cookiejar(self,filename):
-    #     '''
-    #     Assumed to be a FileCookieJar if self.cookiejar set.
-    #     Takes file *name*.
-    #     '''
-    #     self.get_cookiejar().save(filename, ignore_discard=True, ignore_expires=True)
-
-    # def save_pagecache(self,filename):
-    #     '''
-    #     Writes pickle of pagecache to file *name*
-    #     '''
-    #     with open(filename, 'wb') as f:
-    #         pickle.dump(self.get_pagecache(),
-    #                     f,protocol=pickle.HIGHEST_PROTOCOL)
-        
-    # def load_pagecache(self,filename):
-    #     '''
-    #     Reads pickle of pagecache from file *name*
-    #     '''
-    #     with open(filename, 'rb') as f:
-    #         self.set_pagecache(pickle.load(f))
-
     def get_pagecache(self):
         return self.pagecache
     
@@ -185,9 +165,9 @@ class BaseSiteAdapter(Configurable):
         else:
             return None
 
-    def _set_to_pagecache(self,cachekey,data):
+    def _set_to_pagecache(self,cachekey,data,redirectedurl):
         if self.use_pagecache():
-            self.get_pagecache()[cachekey] = data
+            self.get_pagecache()[cachekey] = (data,redirectedurl)
 
     def use_pagecache(self):
         '''
@@ -257,7 +237,8 @@ class BaseSiteAdapter(Configurable):
         cachekey=self._get_cachekey(url, parameters, headers)
         if usecache and self._has_cachekey(cachekey):
             logger.debug("#####################################\npagecache HIT: %s"%cachekey)
-            return self._get_from_pagecache(cachekey)
+            data,redirecturl = self._get_from_pagecache(cachekey)
+            return data
         
         logger.debug("#####################################\npagecache MISS: %s"%cachekey)
         self.do_sleep(extrasleep)
@@ -272,13 +253,23 @@ class BaseSiteAdapter(Configurable):
                          data=urllib.urlencode(parameters),
                          headers=headers)
         data = self._decode(self.opener.open(req,None,float(self.getConfig('connect_timeout',30.0))).read())
-        self._set_to_pagecache(cachekey,data)
+        self._set_to_pagecache(cachekey,data,url)
         return data
 
     def _fetchUrlRaw(self, url,
                      parameters=None,
                      extrasleep=None,
                      usecache=True):
+        
+        return self._fetchUrlRawOpened(url,
+                                       parameters,
+                                       extrasleep,
+                                       usecache)[0]
+        
+    def _fetchUrlRawOpened(self, url,
+                           parameters=None,
+                           extrasleep=None,
+                           usecache=True):
         '''
         When should cache be cleared or not used? logins...
         
@@ -289,16 +280,25 @@ class BaseSiteAdapter(Configurable):
         cachekey=self._get_cachekey(url, parameters)
         if usecache and self._has_cachekey(cachekey):
             logger.debug("#####################################\npagecache HIT: %s"%cachekey)
-            return self._get_from_pagecache(cachekey)
+            data,redirecturl = self._get_from_pagecache(cachekey)
+            class FakeOpened:
+                def __init__(self,data,url):
+                    self.data=data
+                    self.url=url
+                def geturl(self): return self.url
+                def read(self): return self.data
+            return (data,FakeOpened(data,redirecturl))
         
         logger.debug("#####################################\npagecache MISS: %s"%cachekey)
         self.do_sleep(extrasleep)
         if parameters != None:
-            data = self.opener.open(url.replace(' ','%20'),urllib.urlencode(parameters),float(self.getConfig('connect_timeout',30.0))).read()
+            opened = self.opener.open(url.replace(' ','%20'),urllib.urlencode(parameters),float(self.getConfig('connect_timeout',30.0)))
         else:
-            data = self.opener.open(url.replace(' ','%20'),None,float(self.getConfig('connect_timeout',30.0))).read()
-        self._set_to_pagecache(cachekey,data)
-        return data
+            opened = self.opener.open(url.replace(' ','%20'),None,float(self.getConfig('connect_timeout',30.0)))
+        data = opened.read()
+        self._set_to_pagecache(cachekey,data,opened.url)
+        
+        return (data,opened)
 
     def set_sleep(self,val):
         logger.debug("\n===========\n set sleep time %s\n==========="%val)
@@ -312,20 +312,30 @@ class BaseSiteAdapter(Configurable):
         elif self.getConfig('slow_down_sleep_time'):
             time.sleep(float(self.getConfig('slow_down_sleep_time')))
         
-    # parameters is a dict()
     def _fetchUrl(self, url,
                   parameters=None,
                   usecache=True,
                   extrasleep=None):
+        return self._fetchUrlOpened(url,
+                                    parameters,
+                                    usecache,
+                                    extrasleep)[0]
+
+    # parameters is a dict()
+    def _fetchUrlOpened(self, url,
+                        parameters=None,
+                        usecache=True,
+                        extrasleep=None):
 
         excpt=None
         for sleeptime in [0, 0.5, 4, 9]:
             time.sleep(sleeptime)	
             try:
-                return self._decode(self._fetchUrlRaw(url,
+                (data,opened)=self._fetchUrlRawOpened(url,
                                                       parameters=parameters,
                                                       usecache=usecache,
-                                                      extrasleep=extrasleep))
+                                                      extrasleep=extrasleep)
+                return (self._decode(data),opened)
             except u2.HTTPError, he:
                 excpt=he
                 if he.code == 404:
@@ -345,6 +355,7 @@ class BaseSiteAdapter(Configurable):
             self.chapterFirst=int(first)-1
         if last:
             self.chapterLast=int(last)-1
+        self.story.set_chapters_range(first,last)
     
     # Does the download the first time it's called.
     def getStory(self):
@@ -352,21 +363,40 @@ class BaseSiteAdapter(Configurable):
             self.getStoryMetadataOnly(get_cover=True)
 
             for index, (title,url) in enumerate(self.chapterUrls):
+                newchap = False
                 if (self.chapterFirst!=None and index < self.chapterFirst) or \
                         (self.chapterLast!=None and index > self.chapterLast):
                     self.story.addChapter(url,
                                           removeEntities(title),
                                           None)
                 else:
-                    if self.oldchapters and index < len(self.oldchapters):
+                    data = None
+                    if self.oldchaptersmap:
+                        if url in self.oldchaptersmap:
+                            data = self.utf8FromSoup(None,
+                                                     self.oldchaptersmap[url],
+                                                     partial(cachedfetch,self._fetchUrlRaw,self.oldimgs))
+                    elif self.oldchapters and index < len(self.oldchapters):
                         data = self.utf8FromSoup(None,
                                                  self.oldchapters[index],
                                                  partial(cachedfetch,self._fetchUrlRaw,self.oldimgs))
-                    else:
+
+                    # if already marked new -- ie, origtitle and title don't match
+                    # logger.debug("self.oldchaptersdata[url]:%s"%(self.oldchaptersdata[url]))
+                    newchap = (self.oldchaptersdata is not None and
+                               url in self.oldchaptersdata and (
+                            self.oldchaptersdata[url]['chapterorigtitle'] !=
+                            self.oldchaptersdata[url]['chaptertitle']) )
+                    
+                    if not data:
                         data = self.getChapterText(url)
+                        # if had to fetch and has existing chapters
+                        newchap = bool(self.oldchapters or self.oldchaptersmap)
+                        
                     self.story.addChapter(url,
                                           removeEntities(title),
-                                          removeEntities(data))
+                                          removeEntities(data),
+                                          newchap)
             self.storyDone = True
             
             # include image, but no cover from story, add default_cover_image cover.
@@ -399,7 +429,10 @@ class BaseSiteAdapter(Configurable):
             self.doExtractChapterUrlsAndMetadata(get_cover=get_cover)
             
             if not self.story.getMetadataRaw('dateUpdated'):
-                self.story.setMetadata('dateUpdated',self.story.getMetadataRaw('datePublished'))
+                if self.story.getMetadataRaw('datePublished'):
+                    self.story.setMetadata('dateUpdated',self.story.getMetadataRaw('datePublished'))
+                else:
+                    self.story.setMetadata('dateUpdated',self.story.getMetadataRaw('dateCreated'))                
 
             self.metadataDone = True
         return self.story
@@ -409,7 +442,10 @@ class BaseSiteAdapter(Configurable):
             self.story.load_html_metadata(metahtml)
             self.metadataDone = True
             if not self.story.getMetadataRaw('dateUpdated'):
-                self.story.setMetadata('dateUpdated',self.story.getMetadataRaw('datePublished'))
+                if self.story.getMetadataRaw('datePublished'):
+                    self.story.setMetadata('dateUpdated',self.story.getMetadataRaw('datePublished'))
+                else:
+                    self.story.setMetadata('dateUpdated',self.story.getMetadataRaw('dateCreated'))                
     
     def hookForUpdates(self,chaptercount):
         "Usually not needed."
@@ -427,6 +463,11 @@ class BaseSiteAdapter(Configurable):
         "Only needs to be overriden if != site domain."
         return cls.getSiteDomain()
     
+    @classmethod
+    def getConfigSections(cls):
+        "Only needs to be overriden if has additional ini sections."
+        return [cls.getConfigSection()]
+    
     @classmethod
     def stripURLParameters(cls,url):
         "Only needs to be overriden if URL contains more than one parameter"
@@ -476,6 +517,13 @@ class BaseSiteAdapter(Configurable):
 
     def setDescription(self,url,svalue):
         #print("\n\nsvalue:\n%s\n"%svalue)
+        strval = u"%s"%svalue # works for either soup or string
+        if self.hasConfig('description_limit'):
+            limit = int(self.getConfig('description_limit'))
+            if limit and len(strval) > limit:
+                svalue = strval[:limit]
+
+        #print(u"[[[[[\n\n%s\n\n]]]]]]]]"%svalue) # works for either soup or string
         if self.getConfig('keep_summary_html'):
             if isinstance(svalue,basestring):
                 # bs4/html5lib add html, header and body tags, which

fanficfare/adapters/base_efiction_adapter.py

@@ -70,6 +70,11 @@ class BaseEfictionAdapter(BaseSiteAdapter):
         self.triedAcceptWarnings = False
         self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
 
+    @classmethod
+    def getConfigSections(cls):
+        "Only needs to be overriden if has additional ini sections."
+        return ['base_efiction',cls.getConfigSection()]
+    
     @classmethod
     def getAcceptDomains(cls):
         return [cls.getSiteDomain(),'www.' + cls.getSiteDomain()]
@@ -264,7 +269,6 @@ class BaseEfictionAdapter(BaseSiteAdapter):
                 else:
                     super(NameOfMyAdapter, self).handleMetadata(key, value)
         """
-        # logger.debug("metadata: '%s' == '%s'" % (key, value))
         if value == 'None':
             return
         elif key == 'Summary':
@@ -287,7 +291,7 @@ class BaseEfictionAdapter(BaseSiteAdapter):
                 self.story.addToList('challenge', val)
         elif key == 'Chapters':
             self.story.setMetadata('numChapters', int(value))
-        elif key == 'Rating':
+        elif key == 'Rating' or key == 'Rated':
             self.story.setMetadata('rating', value)
         elif key == 'Word count':
             self.story.setMetadata('numWords', value)

fanficfare/adapters/base_xenforoforum_adapter.py

@@ -0,0 +1,229 @@
+#  -*- coding: utf-8 -*-
+
+# Copyright 2015 FanFicFare team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+logger = logging.getLogger(__name__)
+
+class BaseXenForoForumAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["utf8",
+                       "Windows-1252"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+							   
+							   
+        # get storyId from url--url validation guarantees query is only sid=1234
+        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])        
+        
+        # get storyId from url--url validation guarantees query correct
+        m = re.match(self.getSiteURLPattern(),url)
+        if m:
+            self.story.setMetadata('storyId',m.group('id'))
+
+            # normalized story URL.
+            self._setURL(self.getURLPrefix() + '/'+m.group('tp')+'/'+self.story.getMetadata('storyId')+'/')
+        else:
+            raise exceptions.InvalidStoryURL(url,
+                                             self.getSiteDomain(),
+                                             self.getSiteExampleURLs())
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','fsb')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%b %d, %Y at %I:%M %p"
+            
+    @classmethod
+    def getConfigSections(cls):
+        "Only needs to be overriden if has additional ini sections."
+        return ['base_xenforoforum',cls.getConfigSection()]
+    
+    @classmethod
+    def getURLPrefix(cls):
+        # The site domain.  Does have www here, if it uses it.
+        return 'https://' + cls.getSiteDomain() 
+
+    @classmethod
+    def getSiteExampleURLs(cls):
+        return cls.getURLPrefix()+"/threads/some-story-name.123456/"
+
+    def getSiteURLPattern(self):
+        return r"https?://"+re.escape(self.getSiteDomain())+r"/(?P<tp>threads|posts)/(.+\.)?(?P<id>\d+)/?"
+        
+    def use_pagecache(self):
+        '''
+        adapters that will work with the page cache need to implement
+        this and change it to True.
+        '''
+        return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        useurl = self.url
+        logger.info("url: "+useurl)
+
+        try:
+            (data,opened) = self._fetchUrlOpened(useurl)
+            useurl = opened.geturl()
+            logger.info("use useurl: "+useurl)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = self.make_soup(data)
+
+        a = soup.find('h3',{'class':'userText'}).find('a')
+        self.story.addToList('authorId',a['href'].split('/')[1])
+        self.story.addToList('authorUrl',self.getURLPrefix()+'/'+a['href'])
+        self.story.addToList('author',a.text)
+
+        h1 = soup.find('div',{'class':'titleBar'}).h1
+        self.story.setMetadata('title',stripHTML(h1))
+        
+        if '#' in useurl:
+            anchorid = useurl.split('#')[1]
+            soup = soup.find('li',id=anchorid)
+        else:
+            # try threadmarks if no '#' in , require at least 2.
+            threadmarksa = soup.find('a',{'class':'threadmarksTrigger'})
+            if threadmarksa:
+                soupmarks = self.make_soup(self._fetchUrl(self.getURLPrefix()+'/'+threadmarksa['href']))
+                markas = soupmarks.find('ol',{'class':'overlayScroll'}).find_all('a')
+                if len(markas) > 1:
+                    for (atag,url,name) in [ (x,x['href'],stripHTML(x)) for x in markas ]:
+                        date = self.make_date(atag.find_next_sibling('div',{'class':'extra'}))
+                        if not self.story.getMetadataRaw('datePublished') or date < self.story.getMetadataRaw('datePublished'):
+                            self.story.setMetadata('datePublished', date)
+                        if not self.story.getMetadataRaw('dateUpdated') or date > self.story.getMetadataRaw('dateUpdated'):
+                            self.story.setMetadata('dateUpdated', date)
+                            
+                        self.chapterUrls.append((name,self.getURLPrefix()+'/'+url))
+                        
+            soup = soup.find('li',{'class':'message'}) # limit first post for date stuff below. ('#' posts above)
+                
+        # Now go hunting for the 'chapter list'.
+        bq = soup.find('blockquote') # assume first posting contains TOC urls.
+        
+        bq.name='div'
+
+        for iframe in bq.find_all('iframe'):
+            iframe.extract() # calibre book reader & editor don't like iframes to youtube.
+
+        for qdiv in bq.find_all('div',{'class':'quoteExpand'}):
+            qdiv.extract() # Remove <div class="quoteExpand">click to expand</div>
+            
+        self.setDescription(useurl,bq)
+
+        # otherwise, use first post links--include first post since
+        # that's often also the first chapter.
+        if not self.chapterUrls:
+            self.chapterUrls.append(("First Post",useurl))
+            for (url,name) in [ (x['href'],stripHTML(x)) for x in bq.find_all('a') ]:
+                logger.debug("found chapurl:%s"%url)
+                if not url.startswith('http'):
+                    url = self.getURLPrefix()+'/'+url
+    
+                if ( url.startswith(self.getURLPrefix()) or
+                     url.startswith('http://'+self.getSiteDomain()) or
+                     url.startswith('https://'+self.getSiteDomain()) ) and ('/posts/' in url or '/threads/' in url):
+                    # brute force way to deal with SB's http->https change when hardcoded http urls.
+                    url = url.replace('http://'+self.getSiteDomain(),self.getURLPrefix())
+                    logger.debug("used chapurl:%s"%(url))
+                    self.chapterUrls.append((name,url))
+                    if url == useurl and 'First Post' == self.chapterUrls[0][0]:
+                        # remove "First Post" if included in list.
+                        logger.debug("delete dup 'First Post' chapter: %s %s"%self.chapterUrls[0])
+                        del self.chapterUrls[0]
+                        
+            # Didn't use threadmarks, so take created/updated dates
+            # from the 'first' posting created and updated.
+            date = self.make_date(soup.find('a',{'class':'datePermalink'}))
+            if date:
+                self.story.setMetadata('datePublished', date)
+                self.story.setMetadata('dateUpdated', date) # updated overwritten below if found.
+        
+            date = self.make_date(soup.find('div',{'class':'editDate'}))
+            if date:
+                self.story.setMetadata('dateUpdated', date) 
+            
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+    def make_date(self,parenttag): # forums use a BS thing where dates
+                                  # can appear different if recent.
+        datestr=None
+        try:
+            datetag = parenttag.find('span',{'class':'DateTime'})
+            if datetag:
+                datestr = datetag['title']
+            else:
+                datetag = parenttag.find('abbr',{'class':'DateTime'})
+                if datetag:
+                    datestr="%s at %s"%(datetag['data-datestring'],datetag['data-timestring'])
+            # Apr 24, 2015 at 4:39 AM
+            # May 1, 2015 at 5:47 AM
+            datestr = re.sub(r' (\d[^\d])',r' 0\1',datestr) # add leading 0 for single digit day & hours.
+            return makeDate(datestr, self.dateformat)
+        except:
+            logger.debug('No date found in %s'%parenttag)
+            return None
+        
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+        logger.debug('Getting chapter text from: %s' % url)
+
+        origurl = url
+        (data,opened) = self._fetchUrlOpened(url)
+        url = opened.geturl()
+        if '#' in origurl and '#' not in url:
+            url = url + origurl[origurl.index('#'):]
+        logger.debug("chapter URL redirected to: %s"%url)
+
+        soup = self.make_soup(data)
+
+        if '#' in url:
+            anchorid = url.split('#')[1]
+            soup = soup.find('li',id=anchorid)
+        bq = soup.find('blockquote')
+
+        bq.name='div'
+
+        for iframe in bq.find_all('iframe'):
+            iframe.extract() # calibre book reader & editor don't like iframes to youtube.
+
+        for qdiv in bq.find_all('div',{'class':'quoteExpand'}):
+            qdiv.extract() # Remove <div class="quoteExpand">click to expand</div>
+
+        return self.utf8FromSoup(url,bq)

fanficfare/cli.py

@@ -16,7 +16,7 @@
 #
 
 from optparse import OptionParser
-from os.path import expanduser, isfile, join, dirname
+from os.path import expanduser, join, dirname
 from os import access, R_OK
 from subprocess import call
 import ConfigParser
@@ -41,12 +41,14 @@ try:
     # running under calibre
     from calibre_plugins.fanfictiondownloader_plugin.fanficfare import adapters, writers, exceptions
     from calibre_plugins.fanfictiondownloader_plugin.fanficfare.configurable import Configuration
-    from calibre_plugins.fanfictiondownloader_plugin.fanficfare.epubutils import get_dcsource_chaptercount, get_update_data
+    from calibre_plugins.fanfictiondownloader_plugin.fanficfare.epubutils import (
+        get_dcsource_chaptercount, get_update_data, reset_orig_chapters_epub)
     from calibre_plugins.fanfictiondownloader_plugin.fanficfare.geturls import get_urls_from_page
 except ImportError:
     from fanficfare import adapters, writers, exceptions
     from fanficfare.configurable import Configuration
-    from fanficfare.epubutils import get_dcsource_chaptercount, get_update_data
+    from fanficfare.epubutils import (
+        get_dcsource_chaptercount, get_update_data, reset_orig_chapters_epub)
     from fanficfare.geturls import get_urls_from_page
 
 
@@ -87,6 +89,9 @@ def main(argv=None, parser=None, passed_defaultsini=None, passed_personalini=Non
     parser.add_option('-u', '--update-epub',
                       action='store_true', dest='update',
                       help='Update an existing epub with new chapters, give epub filename instead of storyurl.', )
+    parser.add_option('--unnew',
+                      action='store_true', dest='unnew',
+                      help='Remove (new) chapter marks left by mark_new_chapters setting.', )
     parser.add_option('--update-cover',
                       action='store_true', dest='updatecover',
                       help='Update cover in an existing epub, otherwise existing cover (if any) is used on update.  Only valid with --update-epub.', )
@@ -129,6 +134,9 @@ def main(argv=None, parser=None, passed_defaultsini=None, passed_personalini=Non
     if options.update and options.format != 'epub':
         parser.error('-u/--update-epub only works with epub')
 
+    if options.unnew and options.format != 'epub':
+        parser.error('--unnew only works with epub')
+
     # for passing in a file list
     if options.infile:
         urls=[]
@@ -168,6 +176,12 @@ def do_download(arg,
     # Attempt to update an existing epub.
     chaptercount = None
     output_filename = None
+
+    if options.unnew:
+        # remove mark_new_chapters marks
+        reset_orig_chapters_epub(arg,arg)
+        return
+    
     if options.update:
         try:
             url, chaptercount = get_dcsource_chaptercount(arg)
@@ -184,7 +198,7 @@ def do_download(arg,
         url = arg
         
     try:
-        configuration = Configuration(adapters.getConfigSectionFor(url), options.format)
+        configuration = Configuration(adapters.getConfigSectionsFor(url), options.format)
     except exceptions.UnknownSite, e:
         if options.list or options.normalize:
             # list for page doesn't have to be a supported site.
@@ -200,25 +214,18 @@ def do_download(arg,
     if passed_defaultsini:
         configuration.readfp(passed_defaultsini)
 
-    if isfile(join(dirname(__file__), 'defaults.ini')):
-        conflist.append(join(dirname(__file__), 'defaults.ini'))
-        
-    if isfile(join(homepath, 'defaults.ini')):
-        conflist.append(join(homepath, 'defaults.ini'))
-    if isfile(join(homepath2, 'defaults.ini')):
-        conflist.append(join(homepath2, 'defaults.ini'))
-    if isfile('defaults.ini'):
-        conflist.append('defaults.ini')
+    # don't need to check existance for our selves.
+    conflist.append(join(dirname(__file__), 'defaults.ini'))
+    conflist.append(join(homepath, 'defaults.ini'))
+    conflist.append(join(homepath2, 'defaults.ini'))
+    conflist.append('defaults.ini')
 
     if passed_personalini:
         configuration.readfp(passed_personalini)
 
-    if isfile(join(homepath, 'personal.ini')):
-        conflist.append(join(homepath, 'personal.ini'))
-    if isfile(join(homepath2, 'personal.ini')):
-        conflist.append(join(homepath2, 'personal.ini'))
-    if isfile('personal.ini'):
-        conflist.append('personal.ini')
+    conflist.append(join(homepath, 'personal.ini'))
+    conflist.append(join(homepath2, 'personal.ini'))
+    conflist.append('personal.ini')
 
     if options.configfile:
         conflist.extend(options.configfile)
@@ -324,7 +331,9 @@ def do_download(arg,
                  adapter.oldimgs,
                  adapter.oldcover,
                  adapter.calibrebookmark,
-                 adapter.logfile) = (get_update_data(output_filename))[0:7]
+                 adapter.logfile,
+                 adapter.oldchaptersmap,
+                 adapter.oldchaptersdata) = (get_update_data(output_filename))[0:9]
 
                 print 'Do update - epub(%d) vs url(%d)' % (chaptercount, urlchaptercount)
 

fanficfare/configurable.py

@@ -17,6 +17,7 @@
 
 import ConfigParser, re
 import exceptions
+import codecs
 from ConfigParser import DEFAULTSECT, MissingSectionHeaderError, ParsingError
 
 # All of the writers(epub,html,txt) and adapters(ffnet,twlt,etc)
@@ -77,10 +78,12 @@ formatsections = ['html','txt','epub','mobi']
 othersections = ['defaults','overrides']
 
 def get_valid_sections():
-    sites = adapters.getConfigSections()
+    sites = adapters.getConfigSections() 
     sitesections = list(othersections)
     for section in sites:
         sitesections.append(section)
+        # also allows [www.base_efiction] and [www.base_forum]. Not
+        # likely to matter.
         if section.startswith('www.'):
             # add w/o www if has www
             sitesections.append(section[4:])
@@ -130,6 +133,7 @@ def get_valid_set_options():
                'replace_hr':(None,None,boollist),
                'sort_ships':(None,None,boollist),
                'strip_chapter_numbers':(None,None,boollist),
+               'mark_new_chapters':(None,None,boollist),
                'titlepage_use_table':(None,None,boollist),
                
                'use_ssl_unverified_context':(None,None,boollist),
@@ -202,6 +206,7 @@ def get_valid_keywords():
     return list(['(in|ex)clude_metadata_(pre|post)',
                  'add_chapter_numbers',
                  'add_genre_when_multi_category',
+                 'adult_ratings',
                  'allow_unsafe_filename',
                  'always_overwrite',
                  'anthology_tags',
@@ -210,8 +215,12 @@ def get_valid_keywords():
                  'bulk_load',
                  'chapter_end',
                  'chapter_start',
-                 'chapter_title_add_pattern',
                  'chapter_title_strip_pattern',
+                 'chapter_title_def_pattern',
+                 'chapter_title_add_pattern',
+                 'chapter_title_new_pattern',
+                 'chapter_title_addnew_pattern',
+                 'mark_new_chapters',
                  'check_next_chapter',
                  'skip_author_cover',
                  'collect_series',
@@ -224,8 +233,10 @@ def get_valid_keywords():
                  'datePublished_format',
                  'dateUpdated_format',
                  'default_cover_image',
+                 'description_limit',
                  'do_update_hook',
                  'exclude_notes',
+                 'exclude_editor_signature',
                  'extra_logpage_entries',
                  'extra_subject_tags',
                  'extra_titlepage_entries',
@@ -277,6 +288,7 @@ def get_valid_keywords():
                  'output_filename_safepattern',
                  'password',
                  'post_process_cmd',
+                 'rating_titles',
                  'remove_transparency',
                  'replace_br_with_p',
                  'replace_hr',
@@ -331,7 +343,8 @@ def make_generate_cover_settings(param):
 
 class Configuration(ConfigParser.SafeConfigParser):
 
-    def __init__(self, site, fileform):
+    def __init__(self, sections, fileform):
+        site = sections[-1] # first section is site DN.
         ConfigParser.SafeConfigParser.__init__(self)
 
         self.linenos=dict() # key by section or section,key -> lineno
@@ -339,6 +352,11 @@ class Configuration(ConfigParser.SafeConfigParser):
         ## [injected] section has even less priority than [defaults]
         self.sectionslist = ['defaults','injected']
 
+        ## add other sections (not including site DN) after defaults,
+        ## but before site-specific.
+        for section in sections[:-1]:
+            self.addConfigSection(section)
+        
         if site.startswith("www."):
             sitewith = site
             sitewithout = site.replace("www.","")
@@ -348,8 +366,13 @@ class Configuration(ConfigParser.SafeConfigParser):
         
         self.addConfigSection(sitewith)
         self.addConfigSection(sitewithout)
+        
         if fileform:
             self.addConfigSection(fileform)
+            ## add other sections:fileform (not including site DN)
+            ## after fileform, but before site-specific:fileform.
+            for section in sections[:-1]:
+                self.addConfigSection(section+":"+fileform)
             self.addConfigSection(sitewith+":"+fileform)
             self.addConfigSection(sitewithout+":"+fileform)
         self.addConfigSection("overrides")
@@ -437,6 +460,30 @@ class Configuration(ConfigParser.SafeConfigParser):
         else:
             return self.linenos.get(section,None)
     
+    ## Copied from Python 2.7 library so as to make read utf8.
+    def read(self, filenames):
+        """Read and parse a filename or a list of filenames.
+        Files that cannot be opened are silently ignored; this is
+        designed so that you can specify a list of potential
+        configuration file locations (e.g. current directory, user's
+        home directory, systemwide directory), and all existing
+        configuration files in the list will be read.  A single
+        filename may also be given.
+        Return list of successfully read files.
+        """
+        if isinstance(filenames, basestring):
+            filenames = [filenames]
+        read_ok = []
+        for filename in filenames:
+            try:
+                fp = codecs.open(filename,encoding='utf-8')
+            except IOError:
+                continue
+            self._read(fp, filename)
+            fp.close()
+            read_ok.append(filename)
+        return read_ok
+    
     ## Copied from Python 2.7 library so as to make it save linenos too.
     #
     # Regular expressions for parsing section headers and options.

fanficfare/defaults.ini

@@ -268,6 +268,11 @@ keep_summary_html:true
 ## kept.  Some sites(AO3) use <a title="Translation">Foreign language</a>
 #keep_title_attr: false
 
+## If a chapter range was given, use this pattern for the book title.
+## replace_metadata and include/exclude will be applied *after* this.
+## Set to empty value to disable.
+title_chapter_range_pattern:${title} (Ch ${first}-${last})
+
 ## Don't like the numbers at the start of chapter titles on some
 ## sites?  You can use strip_chapter_numbers to strip them off.  Just
 ## want to make them all look the same?  Strip them off, then add them
@@ -300,12 +305,31 @@ chapter_title_strip_pattern:^[0-9]+[\.: -]+
 ## etc
 #chapter_title_strip_pattern:^([0-9]+[\.: -]+)?(Chapter *[0-9]+[\.:, -]*)?
 
-## Uses a python template substitution.  The ${index} is the 'chapter'
-## number and ${title} is the chapter title, after applying
-## chapter_title_strip_pattern.  Those are the only variables available.
-## "The Beginning" => "1. The Beginning" 
+## If true, when updating an epub that already has old chapters, new
+## chapters will be marked in the TOC and chapter header by using
+## chapter_title_new_pattern and chapter_title_addnew_pattern to set the chapter.
+mark_new_chapters:false
+
+## chapter title patterns use python template substitution.  The
+## ${index} is the 'chapter' number and ${title} is the chapter title,
+## after applying chapter_title_strip_pattern.  Those are the only
+## variables available.
+
+## The basic pattern used when not using add_chapter_numbers or
+## mark_new_chapters
+chapter_title_def_pattern:${title}
+
+## Pattern used with add_chapter_numbers, but not mark_new_chapters
 chapter_title_add_pattern:${index}. ${title}
 
+## Pattern used with mark_new_chapters, but not add_chapter_numbers
+## (new) is just text and can be changed.
+chapter_title_new_pattern:(new) ${title}
+
+## Pattern used with add_chapter_numbers and mark_new_chapters
+## (new) is just text and can be changed.
+chapter_title_addnew_pattern:${index}. (new) ${title}
+
 ## Reorder ships so b/a and c/b/a become a/b and a/b/c. Only separates
 ## on '/', so use replace_metadata to change separator first if
 ## needed. Something like: ships=>[ ]*(/|&amp;|&)[ ]*=>/ You can use
@@ -327,6 +351,74 @@ sort_ships:false
 ## User-agent
 user_agent:FFF/2.X
 
+## Added for [base_xenforoforum], but can be used with other sites,
+## too.  Limit the 'description' to the first X *characters*
+## collected.  Character count includes HTML tags, so it can be
+## non-intuitive.
+#description_limit:1000
+
+[base_efiction]
+## At the time of writing, eFiction Base adapters allow downloading
+## the whole story in bulk using the 'Print' feature. If 'bulk_load'
+## is set to 'true', both metadata and chapters can be loaded in one
+## step
+bulk_load:true
+
+[base_xenforoforum]
+## Currently only forums.spacebattles.com and forums.sufficientvelocity.com
+
+cover_exclusion_regexp:/clear.png
+
+## I saw lots of chapters name simply '1.1' etc during testing.
+strip_chapter_numbers:false
+
+## Copy title to tagsfromtitle for parsing tags.
+add_to_extra_valid_entries:,tagsfromtitle
+
+## '.NOREPL' tells the system to *not* apply title's
+## in/exclude/replace_metadata -- Only works on include_in_ lines.
+include_in_tagsfromtitle:title.NOREPL
+
+tagsfromtitle_label:Tags from Title
+
+## might want to do this, maybe not.  Will often include category, but
+## also often include non-category stuff.
+# include_in_category:tagsfromtitle 
+
+add_to_include_metadata_pre:
+# only keep tagsfromtitle with ( or [ in.
+ tagsfromtitle=~[\[\(]
+ 
+add_to_replace_metadata:
+# remove anything outside () or []
+ tagsfromtitle=>^.*?([\(\[]([^\]\)]+)[\)\]]).*?$=>\1
+# remove () []
+ tagsfromtitle=>[\(\)\[\]]=>
+# change (spaces)slash(or semicolon)(spaces) to comma
+ tagsfromtitle=> *[/;] *=>,
+ tagsfromtitle=> [xX] =>,
+ 
+# remove [] or () blocks and leading/trailing spaces/dashes/colons
+ title=>[-: ]*[\(\[]([^\]\)]+)[\)\]][-: ]*=>
+# remove 'Thread' and the next word, usually "Thread 2", "Thread
+# four", "Thread iv", etc
+ title,tagsfromtitle=>[-: ]*[Tt]hread [^ ]+[-: ]*=>
+
+add_to_extra_titlepage_entries:,tagsfromtitle
+
+## '.SPLIT' tells the system to split by ','
+add_to_include_subject_tags:,tagsfromtitle.SPLIT
+
+## base_xenforoforum reads Published and Updated datetimes from
+## Threadmarks if used, or from the posted & updated times of the
+## 'first' post if no threadmarks.
+datePublished_format:%%Y-%%m-%%d %%H:%%M:%%S
+dateUpdated_format:%%Y-%%m-%%d %%H:%%M:%%S
+
+## Only take the first X characters of the 'first' post to use as
+## the description.
+description_limit:500
+
 ## Each output format has a section that overrides [defaults]
 [html]
 
@@ -582,6 +674,8 @@ extratags: FanFiction,Testing,HTML
 ## doesn't like that.  If do_update_hook is uncommented and set true,
 ## the adapter will discard all existing chapters from the newest one
 ## on when updating to enforce accurate chapters.
+## Starting July 2015, FFF stores chapter URLs in the chapter files.
+## Stories downloaded after that shouldn't need this setting anymore.
 #do_update_hook:false
 
 ## AO3 adapter defines a few extra metadata entries.
@@ -770,11 +864,6 @@ extraships:Spike/Buffy
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
-## Virtually all eFiction Base adapters allow downloading the whole story in
-## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
-## metadata and chapters can be loaded in one step
-bulk_load:true
-
 [dramione.org]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -937,11 +1026,6 @@ extraships:Harry Potter/Hermione Granger
 #username:YourName
 #password:yourpassword
 
-## Virtually all eFiction Base adapters allow downloading the whole story in
-## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
-## metadata and chapters can be loaded in one step
-bulk_load:true
-
 extra_valid_entries: readings,romance
 extra_titlepage_entries: readings,romance
 readings_label: Readings
@@ -1031,6 +1115,12 @@ extra_valid_entries:size
 # don't show twitter icon.
 cover_exclusion_regexp:/res/css/bir.png
 
+[forums.spacebattles.com]
+## see [base_xenforoforum]
+
+[forums.sufficientvelocity.com]
+## see [base_xenforoforum]
+
 [grangerenchanted.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -1089,11 +1179,6 @@ extracategories:Glee RPF
 extracharacters:Darren Criss, Chris Colfer
 extraships:Darren Criss/Chris Colfer
 
-## Virtually all eFiction Base adapters allow downloading the whole story in
-## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
-## metadata and chapters can be loaded in one step
-bulk_load:true
-
 [ksarchive.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:Star Trek
@@ -1113,11 +1198,6 @@ eroticatags_label:Erotica Tags
 extra_titlepage_entries: eroticatags
 
 [lotrfanfiction.com]
-## Virtually all eFiction Base adapters allow downloading the whole story in
-## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
-## metadata and chapters can be loaded in one step
-bulk_load:true
-
 extra_valid_entries: readings
 readings_label: Readings
 
@@ -1171,6 +1251,12 @@ extracategories:NCIS
 extracategories:Buffy: The Vampire Slayer
 extracharacters:Willow
 
+[ninelives.dark-solace.org]
+## Site dedicated to these categories/characters/ships
+extracategories:The Walking Dead
+extracharacters:Carol,Daryl
+extraships:Carol/Daryl
+
 [nocturnal-light.net]
 ## Extra metadata that this adapter knows about.  See [dramione.org]
 ## for examples of how to use them.
@@ -1367,11 +1453,6 @@ extracategories:Transgender
 ## confirm they are adult for adult content.
 #is_adult:true
 
-## Virtually all eFiction Base adapters allow downloading the whole story in
-## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
-## metadata and chapters can be loaded in one step
-bulk_load:true
-
 [thehexfiles.net]
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter
@@ -1391,11 +1472,6 @@ extraships:Harry Potter/Draco Malfoy
 ## personal.ini, not defaults.ini.
 #is_adult:true
 
-## Virtually all eFiction Base adapters allow downloading the whole story in
-## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
-## metadata and chapters can be loaded in one step
-bulk_load:true
-
 ## Site dedicated to these categories/characters/ships
 extracategories:Criminal Minds
 
@@ -1405,11 +1481,6 @@ extracategories:Criminal Minds
 ## personal.ini, not defaults.ini.
 #is_adult:true
 
-## Virtually all eFiction Base adapters allow downloading the whole story in
-## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
-## metadata and chapters can be loaded in one step
-bulk_load:true
-
 extra_valid_entries: readings,challenge
 extra_titlepage_entries: readings,challenge
 challenge_label: Challenge
@@ -1757,10 +1828,27 @@ extraships:InuYasha/Kagome
 ## Site dedicated to these categories/characters/ships
 extracategories:Lord of the Rings
 
-## Virtually all eFiction Base adapters allow downloading the whole story in
-## bulk using the 'Print' feature. If 'bulk_load' is set to 'true', both
-## metadata and chapters can be loaded in one step
-bulk_load:true
+[www.masseffect2.in]
+## Site dedicated to this fandom.
+extracategories: Mass Effect
+
+## Ucoz.com engine, upon which MassEffect2.in is based, imposes an unspecified limit on request frequency.
+## Reports vary from `5 requests per second' to `2 requests per second for more than 10 per minute'.
+## With default settings, a several-hours IP ban may follow, so set it higher.
+slow_down_sleep_time: 2
+
+## Whether to exclude editor signature from the bottom of chapter text.
+exclude_editor_signature: false
+
+## Stories on the site almost never have cover image, and for the stories which do,
+## this may be adjusted in `personal.ini' before downloading.
+never_make_cover: true
+
+## Titles for ratings identified by 1- or 2-letter codes from `ERATING system'
+## (`система Р.Е.Й.Т.И.Н.Г.').  MassEffect2.in and some other sites adopted it,
+## but changed titles and update them occasionally.
+rating_titles: R=RESTRICTED (16+), E=EXEMPT (18+), I=ART HOUSE, T=To every, A=IN=Иной мир, Nn=Новый мир, G=О\, Господи!
+adult_ratings: E,R
 
 [www.mediaminer.org]
 

fanficfare/epubutils.py

@@ -1,18 +1,17 @@
-#!/usr/bin/env python
-# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
-from __future__ import (unicode_literals, division, absolute_import,
-                        print_function)
+# -*- coding: utf-8 -*-
 
 __license__   = 'GPL v3'
-__copyright__ = '2014, Jim Miller'
+__copyright__ = '2015, Jim Miller'
 __docformat__ = 'restructuredtext en'
 
 import logging
 logger = logging.getLogger(__name__)
 
 import re, os, traceback
-from zipfile import ZipFile
+from collections import defaultdict
+from zipfile import ZipFile, ZIP_STORED, ZIP_DEFLATED
 from xml.dom.minidom import parseString
+from StringIO import StringIO
 
 import bs4 as bs
 
@@ -90,7 +89,9 @@ def get_update_data(inputio,
 
     filecount = 0
     soups = [] # list of xhmtl blocks
+    urlsoups = {} # map of xhtml blocks by url
     images = {} # dict() longdesc->data
+    datamaps = defaultdict(dict) # map of data maps by url
     if getfilecount:
         # spin through the manifest--only place there are item tags.
         for item in contentdom.getElementsByTagName("item"):
@@ -124,20 +125,45 @@ def get_update_data(inputio,
                                 logger.warn("Image %s not found!\n(originally:%s)"%(newsrc,longdesc))
                                 logger.warn("Exception: %s"%(unicode(e)))
                                 traceback.print_exc()
-                        soup = soup.find('body')
+                        bodysoup = soup.find('body')
                         # ffdl epubs have chapter title h3
-                        h3 = soup.find('h3')
+                        h3 = bodysoup.find('h3')
                         if h3:
                             h3.extract()
                         # TtH epubs have chapter title h2
-                        h2 = soup.find('h2')
+                        h2 = bodysoup.find('h2')
                         if h2:
                             h2.extract()
 
-                        for skip in soup.findAll(attrs={'class':'skip_on_ffdl_update'}):
+                        for skip in bodysoup.findAll(attrs={'class':'skip_on_ffdl_update'}):
                             skip.extract()
+
+                        ## <meta name="chapterurl" content="${url}"></meta>
+                        #print("look for meta chapurl")
+                        currenturl = None
+                        chapurl = soup.find('meta',{'name':'chapterurl'})
+                        if chapurl:
+                            if chapurl['content'] not in urlsoups: # keep first found if more than one.
+                            #print("Found chapurl['content']:%s"%chapurl['content'])
+                                currenturl = chapurl['content']
+                                urlsoups[chapurl['content']] = bodysoup
+                        else:
+                            # for older pre-meta.  Only temp.
+                            chapa = bodysoup.find('a',{'class':'chapterurl'})
+                            if chapa and chapa['href'] not in urlsoups: # keep first found if more than one.
+                                urlsoups[chapa['href']] = bodysoup
+                                currenturl = chapa['href']
+                                chapa.extract()
+
+                        chapterorigtitle = soup.find('meta',{'name':'chapterorigtitle'})
+                        if chapterorigtitle:
+                            datamaps[currenturl]['chapterorigtitle'] = chapterorigtitle['content']
                             
-                        soups.append(soup)
+                        chaptertitle = soup.find('meta',{'name':'chaptertitle'})
+                        if chaptertitle:
+                            datamaps[currenturl]['chaptertitle'] = chaptertitle['content']
+                            
+                        soups.append(bodysoup)
                         
                     filecount+=1
 
@@ -148,7 +174,8 @@ def get_update_data(inputio,
                     
     #for k in images.keys():
         #print("\tlongdesc:%s\n\tData len:%s\n"%(k,len(images[k])))
-    return (source,filecount,soups,images,oldcover,calibrebookmark,logfile)
+    # print("datamaps:%s"%datamaps)
+    return (source,filecount,soups,images,oldcover,calibrebookmark,logfile,urlsoups,datamaps)
 
 def get_path_part(n):
     relpath = os.path.dirname(n)
@@ -192,3 +219,105 @@ def get_story_url_from_html(inputio,_is_good_url=None):
                 if _is_good_url == None or _is_good_url(ahref):
                     return ahref
     return None
+
+def reset_orig_chapters_epub(inputio,outfile):
+    inputepub = ZipFile(inputio, 'r') # works equally well with a path or a blob
+
+    ## build zip in memory in case updating in place(CLI).
+    zipio = StringIO()
+    
+    ## Write mimetype file, must be first and uncompressed.
+    ## Older versions of python(2.4/5) don't allow you to specify
+    ## compression by individual file.
+    ## Overwrite if existing output file.
+    outputepub = ZipFile(zipio, 'w', compression=ZIP_STORED)
+    outputepub.debug = 3
+    outputepub.writestr("mimetype", "application/epub+zip")
+    outputepub.close()
+
+    ## Re-open file for content.
+    outputepub = ZipFile(zipio, "a", compression=ZIP_DEFLATED)
+    outputepub.debug = 3
+
+    changed = False
+    
+    tocncxdom = parseString(inputepub.read('toc.ncx'))
+    ## spin through file contents.
+    for zf in inputepub.namelist():
+        if zf not in ['mimetype','toc.ncx'] :
+            entrychanged = False
+            data = inputepub.read(zf)
+            # if isinstance(data,unicode):
+            #     logger.debug("\n\n\ndata is unicode\n\n\n")
+            if re.match(r'.*/file\d+\.xhtml',zf):
+                data = data.decode('utf-8')
+                soup = bs.BeautifulSoup(data,"html5lib")
+                
+                chapterorigtitle = None
+                tag = soup.find('meta',{'name':'chapterorigtitle'})
+                if tag:
+                    chapterorigtitle = tag['content']
+
+                # toctitle is separate for add_chapter_numbers:toconly users.
+                chaptertoctitle = None
+                tag = soup.find('meta',{'name':'chaptertoctitle'})
+                if tag:
+                    chaptertoctitle = tag['content']
+                elif chapterorigtitle:
+                    chaptertoctitle = chapterorigtitle
+                    
+                chaptertitle = None
+                tag = soup.find('meta',{'name':'chaptertitle'})
+                if tag:
+                    chaptertitle = tag['content']
+
+                if chaptertitle and chapterorigtitle and chapterorigtitle != chaptertitle:
+                    origdata = data
+                    # print("\n%s\n%s\n"%(chapterorigtitle,chaptertitle))
+                    data = data.replace(u'<meta name="chaptertitle" content="'+chaptertitle+u'"></meta>',
+                                        u'<meta name="chaptertitle" content="'+chapterorigtitle+u'"></meta>')
+                    data = data.replace(u'<title>'+chaptertitle+u'</title>',u'<title>'+chapterorigtitle+u'</title>')
+                    data = data.replace(u'<h3>'+chaptertitle+u'</h3>',u'<h3>'+chapterorigtitle+u'</h3>')
+
+                    entrychanged = ( origdata != data )
+                    changed = changed or entrychanged
+                    
+                    if entrychanged:
+                        ## go after the TOC entry, too.
+                        # <navPoint id="file0005" playOrder="6">
+                        #   <navLabel>
+                        #     <text>5. (new) Chapter 4</text>
+                        #   </navLabel>
+                        #   <content src="OEBPS/file0005.xhtml"/>
+                        # </navPoint>
+                        for contenttag in tocncxdom.getElementsByTagName("content"):
+                            if contenttag.getAttribute('src') == zf:
+                                texttag = contenttag.parentNode.getElementsByTagName('navLabel')[0].getElementsByTagName('text')[0]
+                                texttag.childNodes[0].replaceWholeText(chaptertoctitle)
+                                # logger.debug("text label:%s"%texttag.toxml())
+                                continue
+                    
+                outputepub.writestr(zf,data.encode('utf-8'))
+            else:
+                # possibly binary data, thus no .encode().
+                outputepub.writestr(zf,data)
+
+    outputepub.writestr('toc.ncx',tocncxdom.toxml(encoding='utf-8'))
+    outputepub.close()
+    # declares all the files created by Windows.  otherwise, when
+    # it runs in appengine, windows unzips the files as 000 perms.
+    for zf in outputepub.filelist:
+        zf.create_system = 0
+        
+    # only *actually* write if changed.
+    if changed:
+        if isinstance(outfile,basestring):
+            with open(outfile,"wb") as outputio:
+                outputio.write(zipio.getvalue())
+        else:
+            outfile.write(zipio.getvalue())
+
+    inputepub.close()
+    zipio.close()
+            
+    return changed

fanficfare/geturls.py

@@ -22,6 +22,9 @@ import re
 import urllib2 as u2
 import urlparse
 
+import logging
+logger = logging.getLogger(__name__)
+
 from BeautifulSoup import BeautifulSoup 
 from gziphttp import GZipProcessor
 
@@ -82,17 +85,17 @@ def get_urls_from_html(data,url=None,configuration=None,normalize=False,restrict
     soup = BeautifulSoup(data)
     if restrictsearch:
         soup = soup.find(*restrictsearch)
-        #print("restrict search:%s"%soup)
+        #logger.debug("restrict search:%s"%soup)
     
     for a in soup.findAll('a'):
         if a.has_key('href'):
-            #print("a['href']:%s"%a['href'])
+            #logger.debug("a['href']:%s"%a['href'])
             href = form_url(url,a['href'])
-            #print("1 urlhref:%s"%href)
+            #logger.debug("1 urlhref:%s"%href)
             # this (should) catch normal story links, some javascript
             # 'are you old enough' links, and 'Report This' links.
             if 'story.php' in a['href']:
-                #print("trying:%s"%a['href'])
+                #logger.debug("trying:%s"%a['href'])
                 m = re.search(r"(?P<sid>(view)?story\.php\?(sid|psid|no|story|stid)=\d+)",a['href'])
                 if m != None:
                     href = form_url(a['href'] if '//' in a['href'] else url,
@@ -100,15 +103,15 @@ def get_urls_from_html(data,url=None,configuration=None,normalize=False,restrict
                     
             try:
                 href = href.replace('&index=1','')
-                #print("2 urlhref:%s"%href)
+                #logger.debug("2 urlhref:%s"%href)
                 adapter = adapters.getAdapter(configuration,href)
-                #print("found adapter")
+                #logger.debug("found adapter")
                 if adapter.story.getMetadata('storyUrl') not in urls:
                     urls[adapter.story.getMetadata('storyUrl')] = [href]
                 else:
                     urls[adapter.story.getMetadata('storyUrl')].append(href)
             except Exception, e:
-                #print e
+                #logger.debug e
                 pass
 
     # Simply return the longest URL with the assumption that it contains the
@@ -173,7 +176,8 @@ def form_url(parenturl,url):
      return returl
        
 def get_urls_from_imap(srv,user,passwd,folder,markread=True):
-    
+
+    logger.debug("get_urls_from_imap srv:(%s)"%srv)
     mail = imaplib.IMAP4_SSL(srv)
     mail.login(user, passwd)
     mail.list()
@@ -182,8 +186,8 @@ def get_urls_from_imap(srv,user,passwd,folder,markread=True):
     
     result, data = mail.uid('search', None, "UNSEEN")
     
-    #print("result:%s"%result)
-    #print("data:%s"%data)
+    #logger.debug("result:%s"%result)
+    #logger.debug("data:%s"%data)
     urls=set()
     
     #latest_email_uid = data[0].split()[-1]
@@ -191,8 +195,8 @@ def get_urls_from_imap(srv,user,passwd,folder,markread=True):
 
         result, data = mail.uid('fetch', email_uid, '(BODY.PEEK[])') #RFC822
     
-        #print("result:%s"%result)
-        #print("data:%s"%data)
+        #logger.debug("result:%s"%result)
+        #logger.debug("data:%s"%data)
     
         raw_email = data[0][1]
         
@@ -201,28 +205,28 @@ def get_urls_from_imap(srv,user,passwd,folder,markread=True):
     
         email_message = email.message_from_string(raw_email)
      
-        #print "To:%s"%email_message['To']
-        #print "From:%s"%email_message['From']
-        #print "Subject:%s"%email_message['Subject']
+        #logger.debug "To:%s"%email_message['To']
+        #logger.debug "From:%s"%email_message['From']
+        #logger.debug "Subject:%s"%email_message['Subject']
     
-    #    print("payload:%s"%email_message.get_payload())
+    #    logger.debug("payload:%s"%email_message.get_payload())
 
         urllist=[]
         for part in email_message.walk():
             try:
-            #print("part mime:%s"%part.get_content_type())
+            #logger.debug("part mime:%s"%part.get_content_type())
                 if part.get_content_type() == 'text/plain':
                     urllist.extend(get_urls_from_text(part.get_payload(decode=True)))
                 if part.get_content_type() == 'text/html':
                     urllist.extend(get_urls_from_html(part.get_payload(decode=True)))
             except Exception as e:
-                print("Failed to read email content: %s"%e)
-        #print "urls:%s"%get_urls_from_text(get_first_text_block(email_message))
+                logger.error("Failed to read email content: %s"%e)
+        #logger.debug "urls:%s"%get_urls_from_text(get_first_text_block(email_message))
 
         if urllist and markread:
             #obj.store(data[0].replace(' ',','),'+FLAGS','\Seen')
             r,d = mail.uid('store',email_uid,'+FLAGS','(\\SEEN)')
-            #print("seen result:%s->%s"%(email_uid,r))
+            #logger.debug("seen result:%s->%s"%(email_uid,r))
                 
         [ urls.add(x) for x in urllist ]
     

fanficfare/story.py

@@ -16,6 +16,7 @@
 #
 
 import os, re
+from collections import namedtuple
 import urlparse
 import string
 import json
@@ -32,6 +33,8 @@ import exceptions
 from htmlcleanup import conditionalRemoveEntities, removeAllEntities
 from configurable import Configurable, re_compile
 
+Chapter = namedtuple('Chapter', 'url title html origtitle toctitle new')
+
 SPACE_REPLACE=u'\s'
 SPLIT_META=u'\,'
 
@@ -412,7 +415,9 @@ class Story(Configurable):
         except:
             self.metadata = {'version':'4.4'}
         self.in_ex_cludes = {}
-        self.chapters = [] # chapters will be tuples of (title,html)
+        self.chapters = [] # chapters will be namedtuple of Chapter(url,title,html,etc)
+        self.chapter_first = None
+        self.chapter_last = None
         self.imgurls = []
         self.imgtuples = []
 
@@ -441,8 +446,12 @@ class Story(Configurable):
                 iel = []
                 self.in_ex_cludes[ie] = set_in_ex_clude(ies)
 
+    def set_chapters_range(self,first=None,last=None):
+        self.chapter_first=first
+        self.chapter_last=last
+                
     def join_list(self, key, vallist):
-        return self.getConfig("join_string_"+key,u", ").replace(SPACE_REPLACE,' ').join(map(unicode, vallist))
+        return self.getConfig("join_string_"+key,u", ").replace(SPACE_REPLACE,' ').join(map(unicode, [ x for x in vallist if x is not None ]))
 
     def setMetadata(self, key, value, condremoveentities=True):
 
@@ -465,7 +474,7 @@ class Story(Configurable):
 
         if key == 'dateUpdated' and value:
             # Last Update tags for Bill.
-            self.addToList('lastupdate',value.strftime("Last Update Year/Month: %Y/%m"))
+            self.addToList('lastupdate',value.strftime("Last Update Year/Month: %Y/%m"),clear=True)
             self.addToList('lastupdate',value.strftime("Last Update: %Y/%m/%d"))
 
 
@@ -641,6 +650,14 @@ class Story(Configurable):
                 if key in ("datePublished","dateUpdated"):
                     value = value.strftime(self.getConfig(key+"_format","%Y-%m-%d"))
 
+                if key == "title" and (self.chapter_first or self.chapter_last) and self.getConfig("title_chapter_range_pattern"):
+                    first = self.chapter_first or "1"
+                    last = self.chapter_last or self.getMetadata("numChapters")
+                    templ = string.Template(self.getConfig("title_chapter_range_pattern"))
+                    value = templ.substitute({'title':value,
+                                              'first':commaGroups(first),
+                                              'last':commaGroups(last)})
+
             if doreplacements:
                 value=self.doReplacements(value,key)
             if removeallentities and value != None:
@@ -757,8 +774,12 @@ class Story(Configurable):
         # includelist prevents infinite recursion of include_in_'s
         if self.hasConfig("include_in_"+listname) and listname not in includelist:
             for k in self.getConfigList("include_in_"+listname):
+                ldorepl = doreplacements
+                if k.endswith('.NOREPL'):
+                    k = k[:-len('.NOREPL')]
+                    ldorepl = False
                 retlist.extend(self.getList(k,removeallentities=False,
-                                            doreplacements=doreplacements,includelist=includelist+[listname]))
+                                            doreplacements=ldorepl,includelist=includelist+[listname]))
         else:
 
             if not self.isList(listname):
@@ -781,7 +802,11 @@ class Story(Configurable):
 
             retlist = filter( lambda x : x!=None and x!='' ,retlist)
 
-        if listname == 'genre' and self.getConfig('add_genre_when_multi_category') and len(self.getList('category')) > 1:
+        if listname == 'genre' and self.getConfig('add_genre_when_multi_category') and len(self.getList('category',
+                                                                                                        removeallentities=False,
+                                                                                                        # to avoid inf loops if genre/cat substs
+                                                                                                        doreplacements=False
+                                                                                                        )) > 1:
             retlist.append(self.getConfig('add_genre_when_multi_category'))
 
         # reorder ships so b/a and c/b/a become a/b and a/b/c.  Only on '/',
@@ -809,7 +834,16 @@ class Story(Configurable):
 
         # metadata all go into dc:subject tags, but only if they are configured.
         for (name,value) in self.getAllMetadata(removeallentities=removeallentities,keeplists=True).iteritems():
-            if name in tags_list:
+            if name+'.SPLIT' in tags_list:
+                flist=[]
+                if isinstance(value,list):
+                    for tag in value:
+                        flist.extend(tag.split(','))
+                else:
+                    flist.extend(value)
+                for tag in flist:
+                    subjectset.add(tag)
+            elif name in tags_list:
                 if isinstance(value,list):
                     for tag in value:
                         subjectset.add(tag)
@@ -823,24 +857,63 @@ class Story(Configurable):
 
         return list(subjectset | set(self.getConfigList("extratags")))
 
-    def addChapter(self, url, title, html):
+    def addChapter(self, url, title, html, newchap=False):
+        # logger.debug("addChapter(%s,%s)"%(url,newchap))
         if self.getConfig('strip_chapter_numbers') and \
                 self.getConfig('chapter_title_strip_pattern'):
             title = re.sub(self.getConfig('chapter_title_strip_pattern'),"",title)
-        self.chapters.append( (url,title,html) )
+        self.chapters.append( Chapter(url,title,html,title,title,newchap) )
 
     def getChapters(self,fortoc=False):
-        "Chapters will be tuples of (title,html)"
+        "Chapters will be Chapter namedtuples"
         retval = []
-        ## only add numbers if more than one chapter.
-        if len(self.chapters) > 1 and \
-                (self.getConfig('add_chapter_numbers') == "true" \
-                     or (self.getConfig('add_chapter_numbers') == "toconly" and fortoc)) \
-                     and self.getConfig('chapter_title_add_pattern'):
-            for index, (url,title,html) in enumerate(self.chapters):
-                retval.append( (url,
-                                string.Template(self.getConfig('chapter_title_add_pattern')).substitute({'index':index+1,'title':title}),
-                                html) )
+
+        ## only add numbers if more than one chapter.  Ditto (new) marks.
+        if len(self.chapters) > 1:
+            addnums = ( self.getConfig('add_chapter_numbers') == "true" 
+                        or (self.getConfig('add_chapter_numbers') == "toconly" and fortoc) )
+            
+            marknew = self.getConfig('mark_new_chapters')=='true'
+
+            defpattern = self.getConfig('chapter_title_def_pattern','${title}') # default val in case of missing defaults.ini
+            if addnums and marknew:
+                pattern = self.getConfig('chapter_title_add_pattern')
+                newpattern = self.getConfig('chapter_title_addnew_pattern')
+            elif addnums:
+                pattern = self.getConfig('chapter_title_add_pattern')
+                newpattern = pattern
+            elif marknew:
+                pattern = defpattern
+                newpattern = self.getConfig('chapter_title_new_pattern')
+            else:
+                pattern = defpattern
+                newpattern = pattern
+
+            if self.getConfig('add_chapter_numbers') in ["true","toconly"]:
+                tocpattern = self.getConfig('chapter_title_add_pattern')
+            else:
+                tocpattern = defpattern
+
+            # logger.debug("Patterns: (%s)(%s)"%(pattern,newpattern))
+            templ = string.Template(pattern)
+            newtempl = string.Template(newpattern)
+            toctempl = string.Template(tocpattern)            
+            
+            for index, chap in enumerate(self.chapters):
+                if chap.new:
+                    usetempl = newtempl
+                else:
+                    usetempl = templ
+                # logger.debug("chap.url, chap.new: (%s)(%s)"%(chap.url,chap.new))
+                retval.append( Chapter(chap.url,
+                                       # 'new'
+                                       usetempl.substitute({'index':index+1,'title':chap.title}),
+                                       chap.html,
+                                       # 'orig'
+                                       templ.substitute({'index':index+1,'title':chap.title}),
+                                       # 'toc'
+                                       toctempl.substitute({'index':index+1,'title':chap.title}),
+                                       chap.new) )
         else:
             retval = self.chapters
 
@@ -907,6 +980,7 @@ class Story(Configurable):
                 #print("\n===========\nparsedUrl.path:%s\ntoppath:%s\nimgurl:%s\n\n"%(parsedUrl.path,toppath,imgurl))
 
         # apply coverexclusion to explicit covers, too.  Primarily for ffnet imageu.
+        #print("[[[[[\n\n %s %s \n\n]]]]]]]"%(imgurl,coverexclusion))
         if cover and coverexclusion and re.search(coverexclusion,imgurl):
             return (None,None)
 

fanficfare/writers/base_writer.py

@@ -148,12 +148,12 @@ class BaseStoryWriter(Configurable):
             
             self._write(out,START.substitute(self.story.getAllMetadata()))
 
-            for index, (url,title,html) in enumerate(self.story.getChapters(fortoc=True)):
-                if html:
-                    self._write(out,ENTRY.substitute({'chapter':title,
+            for index, chap in enumerate(self.story.getChapters(fortoc=True)):
+                if chap.html:
+                    self._write(out,ENTRY.substitute({'chapter':chap.title,
                                                       'number':index+1,
                                                       'index':"%04d"%(index+1),
-                                                      'url':url}))
+                                                      'url':chap.url}))
 
             self._write(out,END.substitute(self.story.getAllMetadata()))
 

fanficfare/writers/writer_epub.py

@@ -28,7 +28,7 @@ import re
 from xml.dom.minidom import parse, parseString, getDOMImplementation
 
 from base_writer import *
-from ..htmlcleanup import stripHTML
+from ..htmlcleanup import stripHTML,removeEntities
 
 logger = logging.getLogger(__name__)
 
@@ -133,6 +133,10 @@ ${value}<br />
 <head>
 <title>${chapter}</title>
 <link href="stylesheet.css" type="text/css" charset="UTF-8" rel="stylesheet"/>
+<meta name="chapterurl" content="${url}"></meta>
+<meta name="chapterorigtitle" content="${origchapter}"></meta>
+<meta name="chaptertoctitle" content="${tocchapter}"></meta>
+<meta name="chaptertitle" content="${chapter}"></meta>
 </head>
 <body>
 <h3>${chapter}</h3>
@@ -502,13 +506,13 @@ div { margin: 0pt; padding: 0pt; }
             items.append(("log_page","OEBPS/log_page.xhtml","application/xhtml+xml","Update Log"))
             itemrefs.append("log_page")
             
-        for index, (url,title,html) in enumerate(self.story.getChapters(fortoc=True)):
-            if html:
+        for index, chap in enumerate(self.story.getChapters(fortoc=True)):
+            if chap.html:
                 i=index+1
                 items.append(("file%04d"%i,
                               "OEBPS/file%04d.xhtml"%i,
                               "application/xhtml+xml",
-                              title))
+                              chap.title))
                 itemrefs.append("file%04d"%i)
 
         manifest = contentdom.createElement("manifest")
@@ -650,19 +654,21 @@ div { margin: 0pt; padding: 0pt; }
         else:
             CHAPTER_END = self.EPUB_CHAPTER_END
         
-        for index, (url,title,html) in enumerate(self.story.getChapters()):
-            if html:
-                logger.debug('Writing chapter text for: %s' % title)
-                vals={'url':url, 'chapter':title, 'index':"%04d"%(index+1), 'number':index+1}
-                fullhtml = CHAPTER_START.substitute(vals) + html + CHAPTER_END.substitute(vals)
+        for index, chap in enumerate(self.story.getChapters()): # (url,title,html)
+            if chap.html:
+                logger.debug('Writing chapter text for: %s' % chap.title)
+                vals={'url':removeEntities(chap.url),
+                      'chapter':chap.title,
+                      'origchapter':chap.origtitle,
+                      'tocchapter':chap.toctitle,
+                      'index':"%04d"%(index+1),
+                      'number':index+1}
+                fullhtml = CHAPTER_START.substitute(vals) + \
+                    chap.html + CHAPTER_END.substitute(vals)
                 # ffnet(& maybe others) gives the whole chapter text
                 # as one line.  This causes problems for nook(at
                 # least) when the chapter size starts getting big
                 # (200k+)
-                #fullhtml = fullhtml.replace('</p>','</p>\n').replace('<br />','<br />\n')
-                # The replaces above added tons of extra newlines
-                # during *each* epub update.  The regexp version adds
-                # only one and removes any extra.
                 fullhtml = re.sub(r'(</p>|<br ?/>)\n*',r'\1\n',fullhtml)
 
                 outputepub.writestr("OEBPS/file%04d.xhtml"%(index+1),fullhtml.encode('utf-8'))

fanficfare/writers/writer_html.py

@@ -128,12 +128,12 @@ ${output_css}
         else:
             CHAPTER_END = self.HTML_CHAPTER_END
         
-        for index, (url,title,html) in enumerate(self.story.getChapters()):
-            if html:
-                logging.debug('Writing chapter text for: %s' % title)
-                vals={'url':url, 'chapter':title, 'index':"%04d"%(index+1), 'number':index+1}
+        for index, chap in enumerate(self.story.getChapters()):
+            if chap.html:
+                logging.debug('Writing chapter text for: %s' % chap.title)
+                vals={'url':chap.url, 'chapter':chap.title, 'index':"%04d"%(index+1), 'number':index+1}
                 self._write(out,CHAPTER_START.substitute(vals))
-                self._write(out,html)
+                self._write(out,chap.html)
                 self._write(out,CHAPTER_END.substitute(vals))
 
         self._write(out,FILE_END.substitute(self.story.getAllMetadata()))

fanficfare/writers/writer_mobi.py

@@ -161,11 +161,11 @@ ${value}<br />
         else:
             CHAPTER_END = self.MOBI_CHAPTER_END
         
-        for index, (url,title,html) in enumerate(self.story.getChapters()):
-            if html:
-                logger.debug('Writing chapter text for: %s' % title)
-                vals={'url':url, 'chapter':title, 'index':"%04d"%(index+1), 'number':index+1}
-                fullhtml = CHAPTER_START.substitute(vals) + html + CHAPTER_END.substitute(vals)
+        for index, chap in enumerate(self.story.getChapters()):
+            if chap.html:
+                logger.debug('Writing chapter text for: %s' % chap.title)
+                vals={'url':chap.url, 'chapter':chap.title, 'index':"%04d"%(index+1), 'number':index+1}
+                fullhtml = CHAPTER_START.substitute(vals) + chap.html + CHAPTER_END.substitute(vals)
                 # ffnet(& maybe others) gives the whole chapter text
                 # as one line.  This causes problems for nook(at
                 # least) when the chapter size starts getting big

fanficfare/writers/writer_txt.py

@@ -154,12 +154,12 @@ End file.
         else:
             CHAPTER_END = self.TEXT_CHAPTER_END
         
-        for index, (url, title,html) in enumerate(self.story.getChapters()):
-            if html:
-                logging.debug('Writing chapter text for: %s' % title)
-                vals={'url':url, 'chapter':title, 'index':"%04d"%(index+1), 'number':index+1}
+        for index, chap in enumerate(self.story.getChapters()):
+            if chap.html:
+                logging.debug('Writing chapter text for: %s' % chap.title)
+                vals={'url':chap.url, 'chapter':chap.title, 'index':"%04d"%(index+1), 'number':index+1}
                 self._write(out,self.lineends(self.wraplines(removeAllEntities(CHAPTER_START.substitute(vals)))))
-                self._write(out,self.lineends(html2text(html,wrap_width=self.wrap_width)))
+                self._write(out,self.lineends(html2text(chap.html,wrap_width=self.wrap_width)))
                 self._write(out,self.lineends(self.wraplines(removeAllEntities(CHAPTER_END.substitute(vals)))))
 
         self._write(out,self.lineends(self.wraplines(FILE_END.substitute(self.story.getAllMetadata()))))

setup.py

@@ -10,13 +10,11 @@ https://github.com/pypa/sampleproject
 # Always prefer setuptools over distutils
 from setuptools import setup, find_packages
 # To use a consistent encoding
-from codecs import open
+import codecs
 from os import path
 
-here = path.abspath(path.dirname(__file__))
-
 # Get the long description from the relevant file
-with open(path.join(here, 'DESCRIPTION.rst'), encoding='utf-8') as f:
+with codecs.open('DESCRIPTION.rst', encoding='utf-8') as f:
     long_description = f.read()
 
 setup(
@@ -25,7 +23,7 @@ setup(
     # Versions should comply with PEP440.  For a discussion on single-sourcing
     # the version across setup.py and the project code, see
     # https://packaging.python.org/en/latest/single_source_version.html
-    version="2.2.8",
+    version="2.2.12",
 
     description='A tool for downloading fanfiction to eBook formats',
     long_description=long_description,

webservice/app.yaml

@@ -1,6 +1,6 @@
-# ffd-retief-hrd fanfictiondownloader fanficfare
+# ffd-retief-hrd fanficfare
 application: fanficfare
-version: 2-2-8
+version: 2-2-12
 runtime: python27
 api_version: 1
 threadsafe: true

webservice/ffstorage.py

@@ -41,6 +41,8 @@ class DownloadMeta(db.Model):
 	completed =  db.BooleanProperty(default=False)
 	date = db.DateTimeProperty(auto_now_add=True)
 	version = db.StringProperty()
+	ch_begin = db.StringProperty()
+	ch_end = db.StringProperty()
 	# data_chunks is implicit from DownloadData def.
 
 class DownloadData(db.Model):

webservice/index.html

@@ -46,7 +46,8 @@
           </p>
 	  <h3>Changes:</h3>
 	  <ul>
-            <li>Fix add_genre_when_multi_category when genre is empty.</li>
+            <li>New site: <a href="https://forum.questionablequesting.com/forums/creative-writing.19/">forum.questionablequesting.com</a> base_xenforoforum adapter</li>
+            <li>Add minor fix for portkey.org chapter parsing.</li>
 	  </ul>
 	  <p>
 	    Questions?  Check out our 
@@ -56,7 +57,7 @@
 	    If you have any problems with this application, please
 	    report them in
 	    the <a href="http://groups.google.com/group/fanfic-downloader">FanFicFare Google Group</a>.  The
-	    <a href="http://2-2-6.fanficfare.appspot.com">previous version
+	    <a href="http://2-2-11.fanficfare.appspot.com">previous version
 	    </a> is also available for you to use if necessary.
 	  </p>
 	  <div id='error'>

webservice/main.py

@@ -62,7 +62,7 @@ class UserConfigServer(webapp2.RequestHandler):
     
     def getUserConfig(self,user,url,fileformat):
 
-        configuration = Configuration(adapters.getConfigSectionFor(url),fileformat)
+        configuration = Configuration(adapters.getConfigSectionsFor(url),fileformat)
         
         logging.debug('reading defaults.ini config file')
         configuration.read('fanficfare/defaults.ini')
@@ -366,6 +366,16 @@ class FanfictionDownloader(UserConfigServer):
             self.redirect('/')
             return
 
+        # Allow chapter range with URL.
+        # test1.com?sid=5[4-6]
+        mc = re.match(r"^(?P<url>.*?)(?:\[(?P<begin>\d+)?(?P<comma>[,-])?(?P<end>\d+)?\])?$",url)
+        #print("url:(%s) begin:(%s) end:(%s)"%(mc.group('url'),mc.group('begin'),mc.group('end')))
+        url = mc.group('url')
+        ch_begin = mc.group('begin')
+        ch_end = mc.group('end')
+        if ch_begin and not mc.group('comma'):
+            ch_end = ch_begin
+        
         logging.info("Queuing Download: %s" % url)
         login = self.request.get('login')
         password = self.request.get('password')
@@ -385,6 +395,7 @@ class FanfictionDownloader(UserConfigServer):
                 return
 
             adapter = adapters.getAdapter(configuration,url)
+            adapter.setChaptersRange(ch_begin,ch_end)
             logging.info('Created an adaper: %s' % adapter)
 
             if login or password:
@@ -408,6 +419,8 @@ class FanfictionDownloader(UserConfigServer):
             download.title = story.getMetadata('title')
             download.author = story.getMetadata('author')
             download.url = story.getMetadata('storyUrl')
+            download.ch_begin = ch_begin
+            download.ch_end = ch_end
             download.put()
 
             taskqueue.add(url='/fdowntask',
@@ -490,6 +503,7 @@ class FanfictionDownloaderTask(UserConfigServer):
         try:
             configuration = self.getUserConfig(user,url,format)
             adapter = adapters.getAdapter(configuration,url)
+            adapter.setChaptersRange(download.ch_begin,download.ch_end)
 
             logging.info('Created an adapter: %s' % adapter)