Bump versions.

Fix for #120 
(ProceedQuestion got an unexpected keyword argument 'log_viewer_unique_name')

calibre-plugin/__init__.py

@@ -7,15 +7,21 @@ __license__   = 'GPL v3'
 __copyright__ = '2016, Jim Miller'
 __docformat__ = 'restructuredtext en'
 
-import sys
+import sys, os
 if sys.version_info >= (2, 7):
     import logging
     logger = logging.getLogger(__name__)
     loghandler=logging.StreamHandler()
     loghandler.setFormatter(logging.Formatter("FFF: %(levelname)s: %(asctime)s: %(filename)s(%(lineno)d): %(message)s"))
     logger.addHandler(loghandler)
-    loghandler.setLevel(logging.DEBUG)
-    logger.setLevel(logging.DEBUG)
+
+    from calibre.constants import DEBUG
+    if os.environ.get('CALIBRE_WORKER', None) is not None or DEBUG:
+        loghandler.setLevel(logging.DEBUG)
+        logger.setLevel(logging.DEBUG)
+    else:
+        loghandler.setLevel(logging.CRITICAL)
+        logger.setLevel(logging.CRITICAL)
 
 # pulls in translation files for _() strings
 try:
@@ -42,7 +48,7 @@ class FanFicFareBase(InterfaceActionBase):
     description         = _('UI plugin to download FanFiction stories from various sites.')
     supported_platforms = ['windows', 'osx', 'linux']
     author              = 'Jim Miller'
-    version             = (2, 3, 0)
+    version             = (2, 3, 4)
     minimum_calibre_version = (1, 48, 0)
 
     #: This field defines the GUI plugin class that contains all the code

calibre-plugin/config.py

@@ -332,6 +332,7 @@ class ConfigWidget(QWidget):
         # Custom Columns tab
         # error column
         prefs['errorcol'] = unicode(convert_qvariant(self.cust_columns_tab.errorcol.itemData(self.cust_columns_tab.errorcol.currentIndex())))
+        prefs['save_all_errors'] = self.cust_columns_tab.save_all_errors.isChecked()
 
         # metadata column
         prefs['savemetacol'] = unicode(convert_qvariant(self.cust_columns_tab.savemetacol.itemData(self.cust_columns_tab.savemetacol.currentIndex())))
@@ -1340,6 +1341,7 @@ class CustomColumnsTab(QWidget):
         tooltip=_("When an update or overwrite of an existing story fails, record the reason in this column.\n(Text and Long Text columns only.)")
         label.setToolTip(tooltip)
         horz.addWidget(label)
+
         self.errorcol = QComboBox(self)
         self.errorcol.setToolTip(tooltip)
         self.errorcol.addItem('','none')
@@ -1348,6 +1350,15 @@ class CustomColumnsTab(QWidget):
                 self.errorcol.addItem(column['name'],key)
         self.errorcol.setCurrentIndex(self.errorcol.findData(prefs['errorcol']))
         horz.addWidget(self.errorcol)
+        
+        self.save_all_errors = QCheckBox(_('Save All Errors'),self)
+        self.save_all_errors.setToolTip(_('If unchecked, these errors will not be saved:%s')%(
+                '\n'+
+                '\n'.join((_("Not Overwriting, web site is not newer."),
+                           _("Already contains %d chapters.").replace('%d','X')))))
+        self.save_all_errors.setChecked(prefs['save_all_errors'])
+        horz.addWidget(self.save_all_errors)
+        
         self.l.addLayout(horz)
         
         horz = QHBoxLayout()
@@ -1363,6 +1374,10 @@ class CustomColumnsTab(QWidget):
                 self.savemetacol.addItem(column['name'],key)
         self.savemetacol.setCurrentIndex(self.savemetacol.findData(prefs['savemetacol']))
         horz.addWidget(self.savemetacol)
+
+        label = QLabel('')
+        horz.addWidget(label) # empty spacer for alignment with error column line.
+        
         self.l.addLayout(horz)
 
         #print("prefs['custom_cols'] %s"%prefs['custom_cols'])

calibre-plugin/dialogs.py

@@ -21,19 +21,19 @@ from datetime import datetime
 try:
     from PyQt5 import QtWidgets as QtGui
     from PyQt5 import QtCore
-    from PyQt5.Qt import (QDialog, QTableWidget, QVBoxLayout, QHBoxLayout, QGridLayout,
-                          QPushButton, QFont, QLabel, QCheckBox, QIcon, QLineEdit,
-                          QComboBox, QProgressDialog, QTimer, QDialogButtonBox,
-                          QPixmap, Qt, QAbstractItemView, QTextEdit, pyqtSignal,
-                          QGroupBox, QFrame, QTextBrowser, QSize, QAction)
+    from PyQt5.Qt import (QDialog, QWidget, QTableWidget, QVBoxLayout, QHBoxLayout,
+                          QGridLayout, QPushButton, QFont, QLabel, QCheckBox, QIcon,
+                          QLineEdit, QComboBox, QProgressDialog, QTimer, QDialogButtonBox,
+                          QScrollArea, QPixmap, Qt, QAbstractItemView, QTextEdit,
+                          pyqtSignal, QGroupBox, QFrame, QTextBrowser, QSize, QAction)
 except ImportError as e:
     from PyQt4 import QtGui
     from PyQt4 import QtCore
-    from PyQt4.Qt import (QDialog, QTableWidget, QVBoxLayout, QHBoxLayout, QGridLayout,
-                          QPushButton, QFont, QLabel, QCheckBox, QIcon, QLineEdit,
-                          QComboBox, QProgressDialog, QTimer, QDialogButtonBox,
-                          QPixmap, Qt, QAbstractItemView, QTextEdit, pyqtSignal,
-                          QGroupBox, QFrame, QTextBrowser, QSize, QAction)
+    from PyQt4.Qt import (QDialog, QWidget, QTableWidget, QVBoxLayout, QHBoxLayout,
+                          QGridLayout, QPushButton, QFont, QLabel, QCheckBox, QIcon,
+                          QLineEdit, QComboBox, QProgressDialog, QTimer, QDialogButtonBox,
+                          QScrollArea, QPixmap, Qt, QAbstractItemView, QTextEdit,
+                          pyqtSignal, QGroupBox, QFrame, QTextBrowser, QSize, QAction)
 
 try:
     from calibre.gui2 import QVariant
@@ -158,9 +158,10 @@ class RejectUrlEntry:
         return retval
 
 class NotGoingToDownload(Exception):
-    def __init__(self,error,icon='dialog_error.png'):
+    def __init__(self,error,icon='dialog_error.png',showerror=True):
         self.error=error
         self.icon=icon
+        self.showerror=showerror
 
     def __str__(self):
         return self.error
@@ -580,14 +581,34 @@ class UserPassDialog(QDialog):
         self.status=False
         self.hide()
 
-class LoopProgressDialog(QProgressDialog):
+def LoopProgressDialog(gui,
+                       book_list,
+                       foreach_function,
+                       finish_function,
+                       init_label=_("Fetching metadata for stories..."),
+                       win_title=_("Downloading metadata for stories"),
+                       status_prefix=_("Fetched metadata for")):
+    ld = _LoopProgressDialog(gui,
+                             book_list,
+                             foreach_function,
+                             init_label,
+                             win_title,
+                             status_prefix)
+    
+    # Mac OS X gets upset if the finish_function is called from inside
+    # the real _LoopProgressDialog class.
+    
+    # reflect old behavior.
+    if not ld.wasCanceled():
+        finish_function(book_list)
+        
+class _LoopProgressDialog(QProgressDialog):
     '''
     ProgressDialog displayed while fetching metadata for each story.
     '''
     def __init__(self, gui,
                  book_list,
                  foreach_function,
-                 finish_function,
                  init_label=_("Fetching metadata for stories..."),
                  win_title=_("Downloading metadata for stories"),
                  status_prefix=_("Fetched metadata for")):
@@ -598,7 +619,6 @@ class LoopProgressDialog(QProgressDialog):
         self.setMinimumWidth(500)
         self.book_list = book_list
         self.foreach_function = foreach_function
-        self.finish_function = finish_function
         self.status_prefix = status_prefix
         self.i = 0
         self.start_time = datetime.now()
@@ -638,15 +658,16 @@ class LoopProgressDialog(QProgressDialog):
             self.foreach_function(book)
 
         except NotGoingToDownload as d:
+            book['status']=_('Skipped')
             book['good']=False
+            book['showerror']=d.showerror
             book['comment']=unicode(d)
             book['icon'] = d.icon
 
         except Exception as e:
             book['good']=False
             book['comment']=unicode(e)
-            logger.error("Exception: %s:%s"%(book,unicode(e)))
-            traceback.print_exc()
+            logger.error("Exception: %s:%s"%(book,unicode(e)),exc_info=True)
 
         self.updateStatus()
         self.i += 1
@@ -658,8 +679,6 @@ class LoopProgressDialog(QProgressDialog):
 
     def do_when_finished(self):
         self.hide()
-        # Queues a job to process these books in the background.
-        self.finish_function(self.book_list)
 
 def time_duration_format(seconds):
     """
@@ -1431,6 +1450,15 @@ class ViewLog(SizePersistedDialog):
 
         self.lineno = None
 
+        scrollable = QScrollArea()
+        scrollcontent = QWidget()
+        scrollable.setWidget(scrollcontent)
+        scrollable.setWidgetResizable(True)
+        self.l.addWidget(scrollable)
+
+        self.sl = QVBoxLayout()
+        scrollcontent.setLayout(self.sl)
+
         ## error = (lineno, msg)
         for (lineno, error_msg) in errors:
             # print('adding label for error:%s: %s'%(lineno, error_msg))
@@ -1441,7 +1469,7 @@ class ViewLog(SizePersistedDialog):
             label.setStyleSheet("QLabel { margin-left: 2em; color : blue; } QLabel:hover { color: red; }");
             label.setToolTip(_('Click to go to line %s')%lineno)
             label.mouseReleaseEvent = partial(self.label_clicked, lineno=lineno)
-            self.l.addWidget(label)
+            self.sl.addWidget(label)
 
         # html='<p>'+'</p><p>'.join([ '(lineno: %s) %s'%e for e in errors ])+'</p>'
 
@@ -1451,7 +1479,7 @@ class ViewLog(SizePersistedDialog):
         # self.tb.setHtml(html)
         # l.addWidget(self.tb)
 
-        self.l.insertStretch(-1)
+        self.sl.insertStretch(-1)
 
         horz = QHBoxLayout()
 

calibre-plugin/fff_plugin.py

@@ -73,7 +73,7 @@ except:
 
 from calibre.library.field_metadata import FieldMetadata
 field_metadata = FieldMetadata()
-    
+
 from calibre_plugins.fanficfare_plugin.common_utils import (
     set_plugin_icon_resources, get_icon, create_menu_action_unique,
     get_library_uuid)
@@ -497,11 +497,11 @@ class FanFicFarePlugin(InterfaceAction):
                         },"\n".join(url_list))
             else:
                 self.gui.status_bar.show_message(_('Finished Fetching Story URLs from Email.'),3000)
-                
+
         else:
             if url_list:
                 self.add_dialog("\n".join(url_list),merge=False)
-            else:            
+            else:
                 msg = _('No Valid Story URLs Found in Unread Emails.')
                 if reject_list:
                     msg = msg + '<p>'+(_('(%d Story URLs Skipped, on Rejected URL List)')%len(reject_list))+'</p>'
@@ -509,7 +509,7 @@ class FanFicFarePlugin(InterfaceAction):
                             msg,
                             show=True,
                             show_copy_button=False)
-    
+
     def get_urls_from_page_menu(self,anthology=False):
 
         urltxt = ""
@@ -534,7 +534,7 @@ class FanFicFarePlugin(InterfaceAction):
 
         self.gui.status_bar.show_message(_('Finished Fetching Story URLs from Page.'),3000)
         self.restore_cursor()
-        
+
         if url_list:
             self.add_dialog("\n".join(url_list),merge=d.anthology,anthology_url=url)
         else:
@@ -606,7 +606,7 @@ class FanFicFarePlugin(InterfaceAction):
             self.gui.status_bar.show_message(_('Can only UnNew books in library'),
                                              3000)
             return
-        
+
         if not self.gui.current_view().selectionModel().selectedRows() :
             self.gui.status_bar.show_message(_('No Selected Books to Get URLs From'),
                                              3000)
@@ -632,7 +632,7 @@ class FanFicFarePlugin(InterfaceAction):
                                           suffix='.epub',
                                           dir=tdir)
             db.copy_format_to(book['calibre_id'],'EPUB',tmp,index_is_id=True)
-            
+
             unnewtmp = PersistentTemporaryFile(prefix='unnew-%s-'%book['calibre_id'],
                                                suffix='.epub',
                                                dir=tdir)
@@ -656,7 +656,7 @@ class FanFicFarePlugin(InterfaceAction):
                     if fmt.lower() != 'epub' and db.has_format(book['calibre_id'],fmt,index_is_id=True):
                         logger.debug("autoconvert remove f:"+fmt)
                         db.remove_format(book['calibre_id'], fmt, index_is_id=True)#, notify=False
-                
+
     def get_unnew_books_finish(self, book_list, tdir=None):
         remove_dir(tdir)
         if prefs['autoconvert']:
@@ -787,7 +787,7 @@ class FanFicFarePlugin(InterfaceAction):
 
         self.busy_cursor()
         self.gui.status_bar.show_message(_('Fetching Story URLs for Series...'))
-        
+
         # get list from identifiers:url/uri if present, but only if
         # it's *not* a valid story URL.
         mergeurl = self.get_story_url(db,book_id)
@@ -798,7 +798,7 @@ class FanFicFarePlugin(InterfaceAction):
 
         self.gui.status_bar.show_message(_('Finished Fetching Story URLs for Series.'),3000)
         self.restore_cursor()
-        
+
         #print("urlmapfile:%s"%urlmapfile)
 
         # AddNewDialog collects URLs, format and presents buttons.
@@ -958,7 +958,7 @@ class FanFicFarePlugin(InterfaceAction):
                 init_label=_("Fetching metadata for stories...")
                 win_title=_("Downloading metadata for stories")
                 status_prefix=_("Fetched metadata for")
-                
+
             self.gui.status_bar.show_message(status_bar, 3000)
             LoopProgressDialog(self.gui,
                                books,
@@ -1072,7 +1072,7 @@ class FanFicFarePlugin(InterfaceAction):
             ## Getting metadata from configured column.
             custom_columns = self.gui.library_view.model().custom_columns
             if ( collision in (CALIBREONLYSAVECOL) and
-                 prefs['savemetacol'] != '' and 
+                 prefs['savemetacol'] != '' and
                  prefs['savemetacol'] in custom_columns ):
 
                 savedmeta_book_id = book['calibre_id']
@@ -1081,13 +1081,13 @@ class FanFicFarePlugin(InterfaceAction):
                     identicalbooks = self.do_id_search(url)
                     if len(identicalbooks) == 1:
                         savedmeta_book_id = identicalbooks.pop()
-                        
+
                 if savedmeta_book_id:
                     label = custom_columns[prefs['savemetacol']]['label']
                     savedmetadata = db.get_custom(savedmeta_book_id, label=label, index_is_id=True)
                 else:
                     savedmetadata = None
-                    
+
                 if savedmetadata:
                     # sets flag inside story so getStoryMetadataOnly won't hit server.
                     adapter.setStoryMetadata(savedmetadata)
@@ -1123,19 +1123,19 @@ class FanFicFarePlugin(InterfaceAction):
                         if userpass.status:
                             adapter.username = userpass.user.text()
                             adapter.password = userpass.passwd.text()
-        
+
                     except exceptions.AdultCheckRequired:
                         if question_dialog(self.gui, _('Are You an Adult?'), '<p>'+
                                            _("%s requires that you be an adult.  Please confirm you are an adult in your locale:")%url,
                                            show_copy_button=False):
                             adapter.is_adult=True
-        
+
                 # let other exceptions percolate up.
                 story = adapter.getStoryMetadataOnly(get_cover=False)
                 book['title'] = story.getMetadata('title')
                 book['author'] = [story.getMetadata('author')]
                 book['url'] = story.getMetadata('storyUrl')
-        
+
             ## Check reject list.  Redundant with below for when story
             ## URL changes, but also kept here to avoid network hit in
             ## most common case where given url is story url.
@@ -1167,7 +1167,7 @@ class FanFicFarePlugin(InterfaceAction):
                         book['icon']='rotate-right.png'
                         book['status'] = _('Skipped')
                         return
-    
+
         ################################################################################################################################################33
 
         book['is_adult'] = adapter.is_adult
@@ -1176,7 +1176,7 @@ class FanFicFarePlugin(InterfaceAction):
 
         book['icon'] = 'plus.png'
         book['status'] = _('Add')
-        
+
         if not bgmeta:
             # set PI version instead of default.
             if 'version' in options:
@@ -1187,7 +1187,7 @@ class FanFicFarePlugin(InterfaceAction):
             if prefs['savemetacol'] != '':
                 # get metadata to save in configured column.
                 book['savemetacol'] = story.dump_html_metadata()
-            
+
             book['title'] = story.getMetadata("title", removeallentities=True)
             book['author_sort'] = book['author'] = story.getList("author", removeallentities=True)
             book['publisher'] = story.getMetadata("site")
@@ -1198,7 +1198,7 @@ class FanFicFarePlugin(InterfaceAction):
             else:
                 book['comments']=''
             book['series'] = story.getMetadata("series", removeallentities=True)
-    
+
             if story.getMetadataRaw('datePublished'):
                 book['pubdate'] = story.getMetadataRaw('datePublished').replace(tzinfo=local_tz)
             if story.getMetadataRaw('dateUpdated'):
@@ -1207,7 +1207,7 @@ class FanFicFarePlugin(InterfaceAction):
                 book['timestamp'] = story.getMetadataRaw('dateCreated').replace(tzinfo=local_tz)
             else:
                 book['timestamp'] = None # need *something* there for calibre.
-    
+
         if not merge:# skip all the collision code when d/ling for merging.
             if collision in (CALIBREONLY, CALIBREONLYSAVECOL):
                 book['icon'] = 'metadata.png'
@@ -1318,7 +1318,7 @@ class FanFicFarePlugin(InterfaceAction):
                         urlchaptercount = int(story.getMetadata('numChapters').replace(',',''))
                         if chaptercount == urlchaptercount:
                             if collision == UPDATE:
-                                raise NotGoingToDownload(_("Already contains %d chapters.")%chaptercount,'edit-undo.png')
+                                raise NotGoingToDownload(_("Already contains %d chapters.")%chaptercount,'edit-undo.png',showerror=False)
                         elif chaptercount > urlchaptercount:
                             raise NotGoingToDownload(_("Existing epub contains %d chapters, web site only has %d. Use Overwrite to force update.") % (chaptercount,urlchaptercount),'dialog_error.png')
                         elif chaptercount == 0:
@@ -1329,17 +1329,17 @@ class FanFicFarePlugin(InterfaceAction):
                     logger.debug("OVERWRITE file: "+db.format_abspath(book_id, formmapping[fileform], index_is_id=True))
                     fileupdated=datetime.fromtimestamp(os.stat(db.format_abspath(book_id, formmapping[fileform], index_is_id=True))[8])
                     logger.debug("OVERWRITE file updated: %s"%fileupdated)
-                    book['updated']=fileupdated
+                    book['fileupdated']=fileupdated
                     if not bgmeta:
                         # check make sure incoming is newer.
                         lastupdated=story.getMetadataRaw('dateUpdated')
                         logger.debug("OVERWRITE site updated: %s"%lastupdated)
-    
+
                         # updated doesn't have time (or is midnight), use dates only.
                         # updated does have time, use full timestamps.
                         if (lastupdated.time() == time.min and fileupdated.date() > lastupdated.date()) or \
                                 (lastupdated.time() != time.min and fileupdated > lastupdated):
-                            raise NotGoingToDownload(_("Not Overwriting, web site is not newer."),'edit-undo.png')
+                            raise NotGoingToDownload(_("Not Overwriting, web site is not newer."),'edit-undo.png',showerror=False)
 
                 # For update, provide a tmp file copy of the existing epub so
                 # it can't change underneath us.  Now also overwrite for logpage preserve.
@@ -1377,19 +1377,19 @@ class FanFicFarePlugin(InterfaceAction):
                     if not label and k in field_metadata:
                         label=field_metadata[k]['name']
                     key='calibre_std_'+k
-                    
+
                     # if k == 'user_categories':
                     #     value=u', '.join(mi.get(k))
                     #     label=_('User Categories')
-        
+
                     if label: # only if it has a human readable name.
                         if value is None or not book['calibre_id']:
                             ## if existing book, populate existing calibre column
                             ## values in metadata, else '' to hide.
-                            value=''                    
+                            value=''
                         book['calibre_columns'][key]={'val':value,'label':label}
                         #logger.debug("%s(%s): %s"%(label,key,value))
-        
+
                 # custom columns
                 for k, column in self.gui.library_view.model().custom_columns.iteritems():
                     if k != prefs['savemetacol']:
@@ -1405,7 +1405,7 @@ class FanFicFarePlugin(InterfaceAction):
                             value=''
                         book['calibre_columns'][key]={'val':value,'label':label}
                         # logger.debug("%s(%s): %s"%(label,key,value))
-            
+
             # if still 'good', make a temp file to write the output to.
             # For HTML format users, make the filename inside the zip something reasonable.
             # For crazy long titles/authors, limit it to 200chars.
@@ -1472,10 +1472,20 @@ class FanFicFarePlugin(InterfaceAction):
             htmllog = htmllog + '</table></body></html>'
 
             payload = ([], book_list, options)
-            self.gui.proceed_question(self.update_error_column,
-                                      payload, htmllog,
-                                      _('FanFicFare log'), _('FanFicFare download ended'), msg,
-                                      show_copy_button=False)
+
+            # log_viewer_unique_name implemented here: https://github.com/kovidgoyal/calibre/compare/v2.56.0...v2.57.0
+            if calibre_version >= (2, 57, 0):
+                self.gui.proceed_question(self.update_error_column,
+                                          payload, htmllog,
+                                          _('FanFicFare log'), _('FanFicFare download ended'), msg,
+                                          show_copy_button=False,
+                                          log_viewer_unique_name="FanFicFare log viewer")
+            else:
+                self.gui.proceed_question(self.update_error_column,
+                                          payload, htmllog,
+                                          _('FanFicFare log'), _('FanFicFare download ended'), msg,
+                                          show_copy_button=False)
+
             return
 
         cookiejarfile = PersistentTemporaryFile(suffix='.cookiejar',
@@ -1512,7 +1522,7 @@ class FanFicFarePlugin(InterfaceAction):
         custom_columns = self.gui.library_view.model().custom_columns
         if book['calibre_id'] and prefs['errorcol'] != '' and prefs['errorcol'] in custom_columns:
             label = custom_columns[prefs['errorcol']]['label']
-            if not book['good']:
+            if not book['good'] and (book['showerror'] or prefs['save_all_errors']):
                 logger.debug("record/update error message column %s %s"%(book['title'],book['url']))
                 self.set_custom(db, book['calibre_id'], 'comment', book['comment'], label=label, commit=True) # book['comment']
             else:
@@ -1695,24 +1705,32 @@ class FanFicFarePlugin(InterfaceAction):
                 if 'status' in book:
                     status = book['status']
                 else:
-                    status = 'Good'
+                    status = _('Good')
                 htmllog = htmllog + '<tr><td>' + '</td><td>'.join([escapehtml(status),escapehtml(book['title']),escapehtml(", ".join(book['author'])),escapehtml(book['comment']),book['url']]) + '</td></tr>'
 
             for book in bad_list:
                 if 'status' in book:
                     status = book['status']
                 else:
-                    status = 'Bad'
+                    status = _('Bad')
                 htmllog = htmllog + '<tr><td>' + '</td><td>'.join([escapehtml(status),escapehtml(book['title']),escapehtml(", ".join(book['author'])),escapehtml(book['comment']),book['url']]) + '</td></tr>'
 
             htmllog = htmllog + '</table></body></html>'
 
             do_update_func = self.do_download_list_update
 
-        self.gui.proceed_question(do_update_func,
-                                  payload, htmllog,
-                                  _('FanFicFare log'), _('FanFicFare download complete'), msg,
-                                  show_copy_button=False)
+        # log_viewer_unique_name implemented here: https://github.com/kovidgoyal/calibre/compare/v2.56.0...v2.57.0
+        if calibre_version >= (2, 57, 0):
+            self.gui.proceed_question(do_update_func,
+                                      payload, htmllog,
+                                      _('FanFicFare log'), _('FanFicFare download complete'), msg,
+                                      show_copy_button=False,
+                                      log_viewer_unique_name="FanFicFare log viewer")
+        else:
+            self.gui.proceed_question(do_update_func,
+                                      payload, htmllog,
+                                      _('FanFicFare log'), _('FanFicFare download complete'), msg,
+                                      show_copy_button=False)
 
     def do_download_merge_update(self, payload):
 
@@ -1750,6 +1768,7 @@ class FanFicFarePlugin(InterfaceAction):
 
         self.get_epubmerge_plugin().do_merge(tmp.name,
                                              [ x['outfile'] for x in good_list ],
+                                             tags=mergebook['tags'],
                                              titleopt=mergebook['title'],
                                              keepmetadatafiles=True,
                                              source=mergebook['url'])
@@ -1795,7 +1814,7 @@ class FanFicFarePlugin(InterfaceAction):
                                status_prefix=_("Updated"))
 
     def update_error_column_loop(self,book,db=None,label=None):
-        if book['calibre_id'] and label:
+        if book['calibre_id'] and label and (book['showerror'] or prefs['save_all_errors']):
             logger.debug("add/update bad %s %s %s"%(book['title'],book['url'],book['comment']))
             self.set_custom(db, book['calibre_id'], 'comment', book['comment'], label=label, commit=True)
 
@@ -2042,7 +2061,7 @@ class FanFicFarePlugin(InterfaceAction):
             prefs['gencalcover'] == SAVE_YES ## yes, always
             or (prefs['gencalcover'] == SAVE_YES_UNLESS_IMG ## yes, unless image.
                 and book['all_metadata']['cover_image'] not in ('specific','first','default')) ):
-        
+
             cover_generated = False # flag for polish below.
             # Yes, should do gencov.  Which?
             if prefs['calibre_gen_cover'] and HAS_CALGC:
@@ -2055,19 +2074,19 @@ class FanFicFarePlugin(InterfaceAction):
                 cover_generated = True
             elif prefs['plugin_gen_cover'] and 'Generate Cover' in self.gui.iactions:
                 # plugin, if available.
-    
+
                 #logger.debug("Do Generate Cover added:%s gcnewonly:%s"%(book['added'],prefs['gcnewonly']))
-    
+
                 # force a refresh if generating cover so complex composite
                 # custom columns are current and correct
                 db.refresh_ids([book_id])
-    
+
                 gc_plugin = self.gui.iactions['Generate Cover']
                 setting_name = None
                 if prefs['allow_gc_from_ini']:
                     if not configuration: # might already have it from allow_custcol_from_ini
                         configuration = get_fff_config(book['url'],options['fileform'])
-    
+
                     # template => regexp to match => GC Setting to use.
                     # generate_cover_settings:
                     # ${category} => Buffy:? the Vampire Slayer => Buffy
@@ -2076,25 +2095,25 @@ class FanFicFarePlugin(InterfaceAction):
                     #         (template,regexp,setting) = map( lambda x: x.strip(), line.split("=>") )
                     for (template,regexp,setting) in configuration.get_generate_cover_settings():
                         value = Template(template).safe_substitute(book['all_metadata']).encode('utf8')
-                        print("%s(%s) => %s => %s"%(template,value,regexp,setting))
+                        # print("%s(%s) => %s => %s"%(template,value,regexp,setting))
                         if re.search(regexp,value):
                             setting_name = setting
                             break
-    
+
                     if setting_name:
                         logger.debug("Generate Cover Setting from generate_cover_settings(%s)"%setting_name)
                         if setting_name not in gc_plugin.get_saved_setting_names():
                             logger.info("GC Name %s not found, discarding! (check personal.ini for typos)"%setting_name)
                             setting_name = None
-    
+
                 if not setting_name and book['all_metadata']['site'] in prefs['gc_site_settings']:
                     setting_name =  prefs['gc_site_settings'][book['all_metadata']['site']]
                     logger.debug("Generate Cover Setting from site(%s)"%setting_name)
-    
+
                 if not setting_name and 'Default' in prefs['gc_site_settings']:
                     setting_name =  prefs['gc_site_settings']['Default']
                     logger.debug("Generate Cover Setting from Default(%s)"%setting_name)
-    
+
                 if setting_name:
                     logger.debug("Running Generate Cover with settings %s."%setting_name)
                     ## fetch updated mi object from
@@ -2103,14 +2122,14 @@ class FanFicFarePlugin(InterfaceAction):
                     realmi = db.get_metadata(book_id, index_is_id=True)
                     gc_plugin.generate_cover_for_book(realmi,saved_setting_name=setting_name)
                     cover_generated = True
-    
+
             if cover_generated and prefs['gc_polish_cover'] and \
                     options['fileform'] == "epub":
                 # set cover inside epub from calibre's polish feature
                 from calibre.ebooks.oeb.polish.main import polish, ALL_OPTS
                 from calibre.utils.logging import Log
                 from collections import namedtuple
-    
+
                 # Couldn't find a better way to get the cover path.
                 cover_path = os.path.join(db.library_path,
                                           db.path(book_id, index_is_id=True),
@@ -2121,7 +2140,7 @@ class FanFicFarePlugin(InterfaceAction):
                 opts.update(data)
                 O = namedtuple('Options', ' '.join(ALL_OPTS.iterkeys()))
                 opts = O(**opts)
-    
+
                 log = Log(level=Log.DEBUG)
                 outfile = db.format_abspath(book_id,
                                             formmapping[options['fileform']],
@@ -2212,6 +2231,10 @@ class FanFicFarePlugin(InterfaceAction):
         book['comments'] = '' # note this is the book comments.
 
         book['good'] = True
+        book['showerror'] = True # False when NotGoingToDownload is
+                                 # not-overwrite / not-update / skip
+                                 # -- what some would consider 'not an
+                                 # error'
         book['calibre_id'] = None
         book['begin'] = None
         book['end'] = None
@@ -2231,7 +2254,7 @@ class FanFicFarePlugin(InterfaceAction):
             book = self.convert_url_to_book(url)
             if book['url'] in uniqueurls:
                 book['good'] = False
-                book['comment'] = "Same story already included."
+                book['comment'] = _("Same story already included.")
             uniqueurls.add(book['url'])
             book['listorder']=i # BG d/l jobs don't come back in order.
                                 # Didn't matter until anthologies & 'marked' successes
@@ -2483,7 +2506,7 @@ class FanFicFarePlugin(InterfaceAction):
 
     def restore_cursor(self):
         QApplication.restoreOverrideCursor()
-    
+
 def split_text_to_urls(urls):
     # remove dups while preserving order.
     dups=set()

calibre-plugin/jobs.py

@@ -23,6 +23,12 @@ from calibre.library.comments import sanitize_comments_html
 from calibre_plugins.fanficfare_plugin.wordcount import get_word_count
 from calibre_plugins.fanficfare_plugin.prefs import (SAVE_YES, SAVE_YES_UNLESS_SITE)
 
+# pulls in translation files for _() strings
+try:
+    load_translations()
+except NameError:
+    pass # load_translations() added in calibre 1.9
+
 # ------------------------------------------------------------------------------
 #
 #              Functions to perform downloads using worker jobs
@@ -82,7 +88,7 @@ def do_download_worker(book_list,
         book_list.append(job.result)
         book_id = job._book['calibre_id']
         count = count + 1
-        notification(float(count)/total, '%d of %d stories finished downloading'%(count,total))
+        notification(float(count)/total, _('%d of %d stories finished downloading')%(count,total))
         # Add this job's output to the current log
         logger.info('Logfile for book ID %s (%s)'%(book_id, job._book['title']))
         logger.info(job.details)
@@ -189,7 +195,7 @@ def do_download_for_worker(book,options,merge,notification=lambda x,y:x):
             ## No need to download at all.  Shouldn't ever get down here.
             if options['collision'] in (CALIBREONLY, CALIBREONLYSAVECOL):
                 logger.info("Skipping CALIBREONLY 'update' down inside worker--this shouldn't be happening...")
-                book['comment'] = 'Metadata collected.'
+                book['comment'] = _('Metadata collected.')
                 book['all_metadata'] = story.getAllMetadata(removeallentities=True)
                 if options['savemetacol'] != '':
                     book['savemetacol'] = story.dump_html_metadata()
@@ -214,14 +220,14 @@ def do_download_for_worker(book,options,merge,notification=lambda x,y:x):
                     # updated does have time, use full timestamps.
                     if (lastupdated.time() == time.min and fileupdated.date() > lastupdated.date()) or \
                             (lastupdated.time() != time.min and fileupdated > lastupdated):
-                        raise NotGoingToDownload(_("Not Overwriting, web site is not newer."),'edit-undo.png')
+                        raise NotGoingToDownload(_("Not Overwriting, web site is not newer."),'edit-undo.png',showerror=False)
 
                 
                 logger.info("write to %s"%outfile)
                 inject_cal_cols(book,story,configuration)
                 writer.writeStory(outfilename=outfile, forceOverwrite=True)
                 
-                book['comment'] = 'Download %s completed, %s chapters.'%(options['fileform'],story.getMetadata("numChapters"))
+                book['comment'] = _('Download %s completed, %s chapters.')%(options['fileform'],story.getMetadata("numChapters"))
                 book['all_metadata'] = story.getAllMetadata(removeallentities=True)
                 if options['savemetacol'] != '':
                     book['savemetacol'] = story.dump_html_metadata()
@@ -253,7 +259,7 @@ def do_download_for_worker(book,options,merge,notification=lambda x,y:x):
                             book['outfile'] = book['epub_for_update'] # for anthology merge ops.
                             return book
                         else: # not merge,
-                            raise NotGoingToDownload(_("Already contains %d chapters.")%chaptercount,'edit-undo.png')
+                            raise NotGoingToDownload(_("Already contains %d chapters.")%chaptercount,'edit-undo.png',showerror=False)
                     elif chaptercount > urlchaptercount:
                         raise NotGoingToDownload(_("Existing epub contains %d chapters, web site only has %d. Use Overwrite to force update.") % (chaptercount,urlchaptercount),'dialog_error.png')
                     elif chaptercount == 0:
@@ -304,6 +310,7 @@ def do_download_for_worker(book,options,merge,notification=lambda x,y:x):
             
         except NotGoingToDownload as d:
             book['good']=False
+            book['showerror']=d.showerror
             book['comment']=unicode(d)
             book['icon'] = d.icon
     
@@ -311,9 +318,8 @@ def do_download_for_worker(book,options,merge,notification=lambda x,y:x):
             book['good']=False
             book['comment']=unicode(e)
             book['icon']='dialog_error.png'
-            book['status'] = 'Error'
-            logger.info("Exception: %s:%s"%(book,unicode(e)))
-            traceback.print_exc()
+            book['status'] = _('Error')
+            logger.info("Exception: %s:%s"%(book,unicode(e)),exc_info=True)
             
         #time.sleep(10)
     return book

calibre-plugin/plugin-defaults.ini

@@ -474,8 +474,8 @@ add_to_include_subject_tags:,tagsfromtitle.SPLIT,forumtags
 ## base_xenforoforum reads Published and Updated datetimes from
 ## Threadmarks if used, or from the posted & updated times of the
 ## 'first' post if no threadmarks.
-datePublished_format:%%Y-%%m-%%d %%H:%%M:%%S
-dateUpdated_format:%%Y-%%m-%%d %%H:%%M:%%S
+datePublished_format:%%Y-%%m-%%d %%H:%%M
+dateUpdated_format:%%Y-%%m-%%d %%H:%%M
 
 ## Only take the first X characters of the 'first' post to use as
 ## the description.
@@ -847,18 +847,6 @@ extracategories:The Sentinel
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
-[bdsm-geschichten.net]
-## Some sites do not require a login, but do require the user to
-## confirm they are adult for adult content.  In commandline version,
-## this should go in your personal.ini, not defaults.ini.
-#is_adult:true
-
-## This site offers no index page so we can either guess the chapter URLs
-## by dec/incrementing numbers ('guess') or walk all the chapters in the metadata
-## parsing state ('parse'). Since guessing can lead to errors for non-standard
-## story URLs, the default is to parse
-#find_chapters:guess 
-
 [bloodshedverse.com]
 ## website encoding(s) In theory, each website reports the character
 ## encoding they use for each page.  In practice, some sites report it
@@ -869,6 +857,12 @@ extracategories:The Sentinel
 ## it has +90% confidence.  'auto' is not reliable.
 website_encodings:Windows-1252,ISO-8859-1,auto
 
+## dateUpdate doesn't usually have time, but it does on
+## bloodshedverse.com.  See
+## http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+## Note that ini format requires % to be escaped as %%.
+dateUpdated_format:%%Y-%%m-%%d %%H:%%M
+
 ## Extra metadata that this adapter knows about.  See [dramione.org]
 ## for examples of how to use them.
 extra_valid_entries:warnings,reviews
@@ -898,15 +892,6 @@ strip_text_links:true
 ## Site dedicated to these categories/characters/ships
 extracategories:Blood Ties
 
-[buffynfaith.net]
-## Site dedicated to these categories/characters/ships
-extracategories:Buffy: The Vampire Slayer
-
-## Some sites also require the user to confirm they are adult for
-## adult content.  In commandline version, this should go in your
-## personal.ini, not defaults.ini.
-#is_adult:true
-
 [fanfic.castletv.net]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -1262,27 +1247,6 @@ cover_exclusion_regexp:/css/bir.png
 [forums.sufficientvelocity.com]
 ## see [base_xenforoforum]
 
-[grangerenchanted.com]
-## Some sites require login (or login for some rated stories) The
-## program can prompt you, or you can save it in config.  In
-## commandline version, this should go in your personal.ini, not
-## defaults.ini.
-#username:YourName
-#password:yourpassword
-
-## Some sites also require the user to confirm they are adult for
-## adult content.  In commandline version, this should go in your
-## personal.ini, not defaults.ini.
-#is_adult:true
-
-## Site dedicated to these categories/characters/ships
-extracategories:Harry Potter
-extracharacters:Hermione Granger
-
-## Extra metadata that this adapter knows about.  See [dramione.org]
-## for examples of how to use them.
-extra_valid_entries:read,reviews
-
 [hlfiction.net]
 ## Site dedicated to these categories/characters/ships
 extracategories:Highlander
@@ -1339,6 +1303,17 @@ eroticatags_label:Erotica Tags
 averrating_label:Average Rating
 extra_titlepage_entries:eroticatags,averrating
 
+## Extract more erotica_tags from the meta tag of each chapter
+use_meta_keywords: true
+
+## For multiple chapter stories, attempt to clean up the chapter title. This will
+## remove the story title and change "Ch. 01" to "Chapter 1", "Pt. 01" to "Part 1"
+## or just use the text. If this can't be done, the full title is used.
+clean_chapter_titles: false
+
+## Add the chapter description at the start of each chapter.
+description_in_chapter: false
+
 [lotrfanfiction.com]
 extra_valid_entries: readings
 readings_label: Readings
@@ -1519,22 +1494,6 @@ extracategories:Supernatural
 extracharacters:Sam,Dean
 extraships:Sam/Dean
 
-[scarhead.net]
-## Some sites require login (or login for some rated stories) The
-## program can prompt you, or you can save it in config.  In
-## commandline version, this should go in your personal.ini, not
-## defaults.ini.
-#username:YourName
-#password:yourpassword
-
-## Some sites also require the user to confirm they are adult for
-## adult content.  In commandline version, this should go in your
-## personal.ini, not defaults.ini.
-#is_adult:true
-
-## Site dedicated to these categories/characters/ships
-extracategories:Harry Potter
-
 [sheppardweir.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -1676,15 +1635,6 @@ readings_label: Readings
 ## personal.ini, not defaults.ini.
 #is_adult:true
 
-[tokra.fandomnet.com]
-## Some sites do not require a login, but do require the user to
-## confirm they are adult for adult content.  In commandline version,
-## this should go in your personal.ini, not defaults.ini.
-#is_adult:true
-
-## Site dedicated to these categories/characters/ships
-extracategories:Stargate: SG-1
-
 [tolkienfanfiction.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:Lord of the Rings
@@ -1846,6 +1796,10 @@ check_next_chapter:false
 #password:yourpassword
 
 [www.ficbook.net]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
 
 [www.fictionalley.org]
 ## Some sites do not require a login, but do require the user to
@@ -2048,11 +2002,6 @@ cover_exclusion_regexp:/stories/999/images/.*?_trophy.png
 ## Site dedicated to these categories/characters/ships
 extracategories:NCIS
 
-[www.nickngreg.nl]
-## Site dedicated to these categories/characters/ships
-extracategories:CSI
-extraships:Nick Stokes/Greg Sanders
-
 [www.phoenixsong.net]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -2346,20 +2295,6 @@ extracategories:Andromeda
 ## Site dedicated to these categories/characters/ships
 extracategories:Artemis Fowl
 
-[www.therabidreader.com]
-## Some sites do not require a login, but do require the user to
-## confirm they are adult for adult content.  In commandline version,
-## this should go in your personal.ini, not defaults.ini.
-#is_adult:true
-
-## Some sites require login (or login for some rated stories) The
-## program can prompt you, or you can save it in config.  In
-## commandline version, this should go in your personal.ini, not
-## defaults.ini.
-#username:YourName
-#password:yourpassword
-
-
 [www.naiceanilme.net]
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
@@ -2373,8 +2308,6 @@ extracategories:Artemis Fowl
 #username:YourName
 #password:yourpassword
 
-
-
 [overrides]
 ## It may sometimes be useful to override all of the specific format,
 ## site and site:format sections in your private configuration.  For

calibre-plugin/prefs.py

@@ -158,6 +158,7 @@ default_prefs['countpagesstats'] = []
 default_prefs['wordcountmissing'] = False
 
 default_prefs['errorcol'] = ''
+default_prefs['save_all_errors'] = True
 default_prefs['savemetacol'] = ''
 default_prefs['custom_cols'] = {}
 default_prefs['custom_cols_newonly'] = {}

fanficfare/adapters/__init__.py

@@ -85,7 +85,6 @@ import adapter_hpfanficarchivecom
 import adapter_twilightarchivescom
 import adapter_nhamagicalworldsus
 import adapter_hlfictionnet
-import adapter_grangerenchantedcom
 import adapter_dracoandginnycom
 import adapter_scarvesandcoffeenet
 import adapter_thepetulantpoetesscom
@@ -102,13 +101,9 @@ import adapter_efictionestelielde
 import adapter_pommedesangcom
 import adapter_restrictedsectionorg
 import adapter_imagineeficcom
-import adapter_buffynfaithnet
 import adapter_psychficcom
-import adapter_tokrafandomnetcom
 import adapter_asr3slashzoneorg
-import adapter_nickandgregnet
 import adapter_potterheadsanonymouscom
-import adapter_scarheadnet
 import adapter_fictionpadcom
 import adapter_storiesonlinenet
 import adapter_trekiverseorg
@@ -120,7 +115,6 @@ import adapter_nocturnallightnet
 import adapter_fanfichu
 import adapter_fanfictioncsodaidokhu
 import adapter_fictionmaniatv
-import adapter_bdsmgeschichten
 import adapter_tolkienfanfiction
 import adapter_themaplebookshelf
 import adapter_fannation
@@ -139,13 +133,10 @@ import adapter_ninelivesarchivecom
 import adapter_masseffect2in
 import adapter_quotevcom
 import adapter_mcstoriescom
-
 import adapter_lucifaelff
 import adapter_buffygilescom
-#import adapter_rubyquillcom
-import adapter_andromedawebcom # Not all lables are captured
+import adapter_andromedawebcom
 import adapter_artemisfowlcom
-import adapter_rabidreadercom
 import adapter_naiceanilmenet
 
 ## This bit of complexity allows adapters to be added by just adding

fanficfare/adapters/adapter_ashwindersycophanthexcom.py

@@ -77,7 +77,8 @@ class AshwinderSycophantHexComAdapter(BaseSiteAdapter):
     ## Login seems to be reasonably standard across eFiction sites.
     def needToLoginCheck(self, data):
         if 'This story contains adult content and/or themes.' in data \
-                or "That password doesn't match the one in our database" in data:
+                or "That password doesn't match the one in our database" in data \
+                or "Member Login" in data:
             return True
         else:
             return False

fanficfare/adapters/adapter_bdsmgeschichten.py

@@ -1,347 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2014 Fanficdownloader team, 2015 FanFicFare team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-import urlparse
-import time
-
-from bs4.element import Tag, Comment
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter, makeDate
-
-def _translate_date_german_english(date):
-    fullmon = {"Januar":"01",
-               "Februar":"02",
-               u"März":"03",
-               "April":"04",
-               "Mai":"05",
-               "Juni":"06",
-               "Juli":"07",
-               "August":"08",
-               "September":"09",
-               "Oktober":"10",
-               "November":"11",
-               "Dezember":"12"}
-    for (name,num) in fullmon.items():
-        date = date.replace(name,num)
-    return date
-
-_REGEX_TRAILING_DIGIT = re.compile("(\d+)$")
-_REGEX_DASH_TO_END = re.compile("-[^-]+$")
-_REGEX_CHAPTER_TITLE = re.compile(ur"""
-    \s*
-    [\u2013-]?
-    \s*
-    ([\dIVX-]+)?
-    \.?
-    \s*
-    [\[\(]?
-    \s*
-    (Teil|Kapitel|Tag)?
-    \s*
-    ([\dIVX-]+)?
-    \s*
-    [\]\)]?
-    \s*
-    $
-""", re.VERBOSE)
-_INITIAL_STEP = 5
-
-class BdsmGeschichtenAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["utf8", "Windows-1252"]
-
-        self.story.setMetadata('siteabbrev','bdsmgesch')
-
-        # Replace possible chapter numbering
-        chapterMatch = _REGEX_TRAILING_DIGIT.search(url)
-        if chapterMatch is None:
-            self.maxChapter = 1
-        else:
-            self.maxChapter = int(chapterMatch.group(1))
-        # url = re.sub(_REGEX_TRAILING_DIGIT, "1", url)
-
-        # set storyId
-        self.story.setMetadata('storyId', re.compile(self.getSiteURLPattern()).match(url).group('storyId'))
-
-        # normalize URL
-        self._setURL('http://%s/%s' % (self.getSiteDomain(), self.story.getMetadata('storyId')))
-
-        self.dateformat = '%d. %m %Y - %H:%M'
-
-    @staticmethod
-    def getSiteDomain():
-        return 'bdsm-geschichten.net'
-
-    @classmethod
-    def getAcceptDomains(cls):
-        return ['www.bdsm-geschichten.net', 'www.bdsm-geschichten.net']
-
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://www.bdsm-geschichten.net/title-of-story-1 http://bdsm-geschichten.net/title-of-story-1"
-
-    def getSiteURLPattern(self):
-        return r"http://(www\.)?bdsm-geschichten.net/(?P<storyId>[a-zA-Z0-9_-]+)"
-
-    def extractChapterUrlsAndMetadata(self):
-
-        if not (self.is_adult or self.getConfig("is_adult")):
-            raise exceptions.AdultCheckRequired(self.url)
-
-        try:
-            data1 = self._fetchUrl(self.url)
-            soup = self.make_soup(data1)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        #strip comments from soup
-        [comment.extract() for comment in soup.findAll(text=lambda text:isinstance(text, Comment))]
-
-        # Cache the soups so we won't have to redownload in getChapterText later
-        self.soupsCache = {}
-        self.soupsCache[self.url] = soup
-
-        # author
-        authorDiv = soup.find("div", "author-pane-line author-name")
-        authorId = authorDiv.string.strip()
-        self.story.setMetadata('authorId', authorId)
-        self.story.setMetadata('author', authorId)
-        # TODO not really true need to be loggedin for this to work or fetch userid
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+authorId)
-
-        # TODO better metadata
-        date = soup.find("div", {"class": "submitted"}).string.strip()
-        # 11. April 2015 - 17:08
-        date = re.sub(r"(\d+\. \D+ \d+ - \d+:\d+).*", r"\1", date)
-        date = _translate_date_german_english(date)
-        self.story.setMetadata('datePublished', makeDate(date, self.dateformat))
-        title1 = soup.find("h1", {'class': 'title'}).string
-
-
-        for tagLink in soup.find("ul", "taxonomy").findAll("a"):
-            self.story.addToList('category', tagLink.string)
-
-        ## Retrieve chapter soups
-        if self.getConfig('find_chapters') == 'guess':
-            self.chapterUrls = []
-            self._find_chapters_by_guessing(title1)
-        else:
-            self._find_chapters_by_parsing(soup)
-
-        firstChapterUrl = self.chapterUrls[0][1]
-        if firstChapterUrl in self.soupsCache:
-            firstChapterSoup = self.soupsCache[firstChapterUrl]
-            h1 = firstChapterSoup.find("h1").text
-        else:
-            h1 = soup.find("h1").text
-
-        h1 = re.sub(_REGEX_CHAPTER_TITLE, "", h1)
-        self.story.setMetadata('title', h1)
-        self.story.setMetadata('numChapters', len(self.chapterUrls))
-        return
-
-    def _find_chapters_by_parsing(self, soup):
-
-        # store original soup
-        origSoup = soup
-
-        #
-        # find first chapter
-        #
-        firstLink = None
-        firstLinkDiv = soup.find("div", "field-field-erster-teil")
-        if firstLinkDiv is not None:
-            firstLink = "http://%s%s" % (self.getSiteDomain(), firstLinkDiv.findNext("a")['href'])
-            logger.debug("Found first chapter right away <%s>" % firstLink)
-            try:
-                soup = self.make_soup(self._fetchUrl(firstLink))
-                self.soupsCache[firstLink] = soup
-                self.chapterUrls.insert(0, (soup.find("h1").text, firstLink))
-            except urllib2.HTTPError, e:
-                if e.code == 404:
-                    raise exceptions.StoryDoesNotExist(self.url)
-                else:
-                    raise exceptions.StoryDoesNotExist(firstLink)
-        else:
-            logger.debug("DIDN'T find first chapter right away")
-            # parse previous Link until first
-            while True:
-                prevLink = None
-                prevLinkDiv = soup.find("div", "field-field-vorheriger-teil")
-                if prevLinkDiv is not None:
-                    prevLink = prevLinkDiv.find("a")
-                if prevLink is None:
-                    prevLink = soup.find("a", text=re.compile("&lt;&lt;&lt;")) # <<<
-                if prevLink is None:
-                    logger.debug("Couldn't find prev part")
-                    break
-                else:
-                    logger.debug("Previous Chapter <%s>" % prevLink)
-                    if type(prevLink) != Tag or prevLink.name != "a":
-                        prevLink = prevLink.findParent("a")
-                    if prevLink is None or '#' in prevLink['href']:
-                        logger.debug("Couldn't find prev part (false positive) <%s>" % prevLink)
-                        break
-                    prevLink = prevLink['href']
-                try:
-                    soup = self.make_soup(self._fetchUrl(prevLink))
-                    self.soupsCache[prevLink] = soup
-                    prevTtitle = soup.find("h1", {'class': 'title'}).string
-                    self.chapterUrls.insert(0, (prevTtitle, prevLink))
-                except urllib2.HTTPError, e:
-                    if e.code == 404:
-                        raise exceptions.StoryDoesNotExist(nextLink)
-                    else:
-                        raise e
-                firstLink = prevLink
-
-        # if first chapter couldn't be determined, assume the URL originally
-        # passed is the first chapter
-        if firstLink is None:
-            logger.debug("Couldn't set first chapter")
-            firstLink = self.url
-            self.chapterUrls.insert(0, (soup.find("h1").text, firstLink))
-
-        # set first URL
-        logger.debug("Set first link: %s" % firstLink)
-        self._setURL(firstLink)
-        self.story.setMetadata('storyId', re.compile(self.getSiteURLPattern()).match(firstLink).group('storyId'))
-
-        #
-        # Parse next chapters
-        #
-        while True:
-            nextLink = None
-            nextLinkDiv = soup.find("div", "field-field-naechster-teil")
-            if nextLinkDiv is not None:
-                nextLink = nextLinkDiv.find("a")
-            if nextLink is None:
-                nextLink = soup.find("a", text=re.compile("&gt;&gt;&gt;"))
-            if nextLink is None:
-                nextLink = soup.find("a", text=re.compile("Fortsetzung"))
-
-            if nextLink is None:
-                logger.debug("Couldn't find next part")
-                break
-            else:
-                if type(nextLink) != Tag or nextLink.name != "a":
-                    nextLink = nextLink.findParent("a")
-                if nextLink is None or '#' in nextLink['href']:
-                    logger.debug("Couldn't find next part (false positive) <%s>" % nextLink)
-                    break
-                nextLink = nextLink['href']
-
-            if not nextLink.startswith('http:'):
-                nextLink = 'http://' + self.getSiteDomain() + nextLink
-
-            for loadedChapter in self.chapterUrls:
-                if loadedChapter[0] == nextLink:
-                    logger.debug("ERROR: Repeating chapter <%s> Try to fix it" % nextLink)
-                    nextLinkMatch = _REGEX_TRAILING_DIGIT.match(nextLink)
-                    if nextLinkMatch is not None:
-                        curChap = nextLinkMatch.group(1)
-                        nextLink = re.sub(_REGEX_TRAILING_DIGIT, unicode(int(curChap) + 1), nextLink)
-                    else:
-                        break
-            try:
-                data = self._fetchUrl(nextLink)
-                soup = self.make_soup(data)
-            except urllib2.HTTPError, e:
-                if e.code == 404:
-                    raise exceptions.StoryDoesNotExist(nextLink)
-                else:
-                    raise e
-            title2 = soup.find("h1", {'class': 'title'}).string
-            self.chapterUrls.append((title2, nextLink))
-            logger.debug("Grabbing next chapter URL " + nextLink)
-            self.soupsCache[nextLink] = soup
-            # [comment.extract() for comment in soup.findAll(text=lambda text:isinstance(text, Comment))]
-        logger.debug("Chapters: %s" % self.chapterUrls)
-
-
-    def _find_chapters_by_guessing(self, title1):
-        step = _INITIAL_STEP
-        curMax = self.maxChapter + step
-        lastHit = True
-        while True:
-            nextChapterUrl = re.sub(_REGEX_TRAILING_DIGIT, unicode(curMax), self.url)
-            if nextChapterUrl == self.url:
-                logger.debug("Unable to guess next chapter because URL doesn't end in numbers")
-                break;
-            try:
-                logger.debug("Trying chapter URL " + nextChapterUrl)
-                data = self._fetchUrl(nextChapterUrl)
-                hit = True
-            except urllib2.HTTPError, e:
-                if e.code == 404:
-                    hit = False
-                else:
-                    raise e
-            if hit:
-                logger.debug("Found chapter URL " + nextChapterUrl)
-                self.maxChapter = curMax
-                self.soupsCache[nextChapterUrl] = self.make_soup(data)
-                if not lastHit:
-                    break
-                lastHit = curMax
-                curMax += step
-            else:
-                lastHit = False
-                curMax -= 1
-            logger.debug(curMax)
-
-        for i in xrange(1, self.maxChapter):
-            nextChapterUrl = re.sub(_REGEX_TRAILING_DIGIT, unicode(i), self.url)
-            nextChapterTitle = re.sub("1", unicode(i), title1)
-            self.chapterUrls.append((nextChapterTitle, nextChapterUrl))
-
-    def getChapterText(self, url):
-
-        if url in self.soupsCache:
-            logger.debug('Getting chapter <%s> from cache' % url)
-            soup = self.soupsCache[url]
-        else:
-            logger.debug('Downloading chapter <%s>' % url)
-            data1 = self._fetchUrl(url)
-            soup = self.make_soup(data1)
-            #strip comments from soup
-            [comment.extract() for comment in soup.findAll(text=lambda text:isinstance(text, Comment))]
-
-        # get story text
-        storyDiv1 = soup.new_tag("div")
-        for para in soup.find("div", "full-node").find('div', 'content').findAll("p"):
-            storyDiv1.append(para)
-        storyDiv1.append(soup.new_tag("br"))
-        storytext = self.utf8FromSoup(url,storyDiv1)
-
-        return storytext
-
-
-def getClass():
-    return BdsmGeschichtenAdapter

fanficfare/adapters/adapter_bloodshedversecom.py

@@ -28,7 +28,7 @@ class BloodshedverseComAdapter(BaseSiteAdapter):
     READ_URL_TEMPLATE = BASE_URL + 'stories.php?go=read&no=%s'
 
     STARTED_DATETIME_FORMAT = '%m/%d/%Y'
-    UPDATED_DATETIME_FORMAT = '%m/%d/%Y %I:%M'
+    UPDATED_DATETIME_FORMAT = '%m/%d/%Y %I:%M %p'
 
     def __init__(self, config, url):
         BaseSiteAdapter.__init__(self, config, url)
@@ -168,14 +168,8 @@ class BloodshedverseComAdapter(BaseSiteAdapter):
                     self.story.setMetadata('datePublished', makeDate(value, self.STARTED_DATETIME_FORMAT))
 
                 elif key == 'Updated':
-                    date_string, period = value.rsplit(' ', 1)
-                    date = makeDate(date_string, self.UPDATED_DATETIME_FORMAT)
-
-                    # Rather ugly hack to work around Calibre's changing of
-                    # Python's locale setting, causing am/pm to not be properly
-                    # parsed by strptime() when using a non-english locale
-                    if period == 'pm':
-                        date += timedelta(hours=12)
+                    date = makeDate(value, self.UPDATED_DATETIME_FORMAT)
+                    # ugly %p(am/pm) hack moved into makeDate so other sites can use it.
                     self.story.setMetadata('dateUpdated', date)
 
         if self.story.getMetadata('rating') == 'NC-17' and not (self.is_adult or self.getConfig('is_adult')):

fanficfare/adapters/adapter_buffynfaithnet.py

@@ -1,292 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2013 Fanficdownloader team, 2015 FanFicFare team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-import cookielib as cl
-
-
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-
-# This function is called by the downloader in all adapter_*.py files
-# in this dir to register the adapter class.  So it needs to be
-# updated to reflect the class below it.  That, plus getSiteDomain()
-# take care of 'Registering'.
-def getClass():
-    return BuffyNFaithNetAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class BuffyNFaithNetAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.setHeader()
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-
-        # get storyId from url--url validation guarantees query correct
-        m = re.match(self.getSiteURLPattern(),url)
-        if m:
-            self.story.setMetadata('storyId',m.group('id'))
-
-            # normalized story URL. gets rid of chapter if there, left with ch 1 URL on this site
-            nurl = "http://"+self.getSiteDomain()+"/fanfictions/index.php?act=vie&id="+self.story.getMetadata('storyId')
-            self._setURL(nurl)
-        else:
-            raise exceptions.InvalidStoryURL(url,
-                                             self.getSiteDomain(),
-                                             self.getSiteExampleURLs())
-
-
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','bnfnet')
-
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'buffynfaith.net'
-
-    @classmethod
-    def stripURLParameters(cls,url):
-        "Only needs to be overriden if URL contains more than one parameter"
-        ## This adapter needs at least two parameters left on the URL, act and id
-        return re.sub(r"(\?act=(vie|ovr)&id=\d+)&.*$",r"\1",url)
-
-    def setHeader(self):
-        "buffynfaith.net wants a Referer for images.  Used both above and below(after cookieproc added)"
-        self.opener.addheaders.append(('Referer', 'http://'+self.getSiteDomain()+'/'))
-
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://"+cls.getSiteDomain()+"/fanfictions/index.php?act=vie&id=1234 http://"+cls.getSiteDomain()+"/fanfictions/index.php?act=ovr&id=1234 http://"+cls.getSiteDomain()+"/fanfictions/index.php?act=vie&id=1234&ch=2"
-
-    def getSiteURLPattern(self):
-        #http://buffynfaith.net/fanfictions/index.php?act=vie&id=963
-        #http://buffynfaith.net/fanfictions/index.php?act=vie&id=949
-        #http://buffynfaith.net/fanfictions/index.php?act=vie&id=949&ch=2
-        p = re.escape("http://"+self.getSiteDomain()+"/fanfictions/index.php?act=")+\
-            r"(vie|ovr)&id=(?P<id>\d+)(&ch=(?P<ch>\d+))?$"
-        return p
-
-    def use_pagecache(self):
-        '''
-        adapters that will work with the page cache need to implement
-        this and change it to True.
-        '''
-        return True
-
-    def extractChapterUrlsAndMetadata(self):
-
-        dateformat = "%d %B %Y"
-        url = self.url
-        logger.debug("URL: "+url)
-
-        #set a cookie to get past adult check
-        if self.is_adult or self.getConfig("is_adult"):
-            cookie = cl.Cookie(version=0, name='my_age', value='yes',
-                               port=None, port_specified=False,
-                               domain=self.getSiteDomain(), domain_specified=False, domain_initial_dot=False,
-                               path='/', path_specified=True,
-                               secure=False,
-                               expires=time.time()+10000,
-                               discard=False,
-                               comment=None,
-                               comment_url=None,
-                               rest={'HttpOnly': None},
-                               rfc2109=False)
-            self.get_cookiejar().set_cookie(cookie)
-            self.setHeader()
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        #print data
-
-        if "ADULT CONTENT WARNING" in data:
-            raise exceptions.AdultCheckRequired(self.url)
-
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = self.make_soup(data)
-
-        # Now go hunting for all the meta data and the chapter list.
-
-        #stuff in <head>: description
-        svalue = soup.head.find('meta',attrs={'name':'description'})['content']
-        #self.story.setMetadata('description',svalue)
-        self.setDescription(url,svalue)
-
-        #useful stuff in rest of doc, all contained in this:
-        doc = soup.body.find('div', id='my_wrapper')
-
-        #first the site category (more of a genre to me, meh) and title, in this element:
-        mt = doc.find('div',attrs={'class':'maintitle'})
-        self.story.addToList('genre',mt.findAll('a')[1].string)
-        self.story.setMetadata('title',stripHTML(mt).split(u'»')[-1].strip())
-        del mt
-
-        #the actual category, for me, is 'Buffy: The Vampire Slayer'
-        #self.story.addToList('category','Buffy: The Vampire Slayer')
-        #No need to do it here, it is better to set it in in plugin-defaults.ini and defaults.ini
-
-        #then a block that sits in a table cell like so:
-        #(contains a lot of metadata)
-        mblock = doc.find('td', align='left', width = '70%').contents
-        while len(mblock) > 0:
-            i = mblock.pop(0)
-            if 'Author:' in i.string:
-                #drop empty space
-                mblock.pop(0)
-                #get author link
-                a = mblock.pop(0)
-                authre = re.escape('./index.php?act=bio&id=')+'(?P<authid>\d+)'
-                m = re.match(authre,a['href'])
-                self.story.setMetadata('author',a.string)
-                self.story.setMetadata('authorId',m.group('authid'))
-                authurl = u'http://%s/fanfictions/index.php?act=bio&id=%s' % ( self.getSiteDomain(),
-                                            self.story.getMetadata('authorId'))
-                self.story.setMetadata('authorUrl',authurl,condremoveentities=False)
-                #drop empty space
-                mblock.pop(0)
-            if 'Rating:' in i.string:
-                self.story.setMetadata('rating',mblock.pop(0).strip())
-            if 'Published:' in i.string:
-                date = mblock.pop(0).strip()
-                #get rid of 'st', 'nd', 'rd', 'th' after day number
-                date = date[0:2]+date[4:]
-                self.story.setMetadata('datePublished',makeDate(date, dateformat))
-            if 'Last Updated:' in i.string:
-                date = mblock.pop(0).strip()
-                #get rid of 'st', 'nd', 'rd', 'th' after day number
-                date = date[0:2]+date[4:]
-                self.story.setMetadata('dateUpdated',makeDate(date, dateformat))
-            if 'Genre:' in i.string:
-                genres = mblock.pop(0).strip()
-                genres = genres.split('/')
-                for genre in genres: self.story.addToList('genre',genre)
-            #end ifs
-        #end while
-
-        # Find the chapter selector
-        select = soup.find('select', { 'name' : 'ch' } )
-
-        if select is None:
-           # no selector found, so it's a one-chapter story.
-           #self.chapterUrls.append((self.story.getMetadata('title'),url))
-           self.chapterUrls.append((self.story.getMetadata('title'),url))
-        else:
-            allOptions = select.findAll('option')
-            for o in allOptions:
-                url = u'http://%s/fanfictions/index.php?act=vie&id=%s&ch=%s' % ( self.getSiteDomain(),
-                                            self.story.getMetadata('storyId'),
-                                            o['value'])
-                title = u"%s" % o
-                title = stripHTML(title)
-                ts = title.split(' ',1)
-                title = ts[0]+'. '+ts[1]
-                self.chapterUrls.append((title,url))
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        ## Go scrape the rest of the metadata from the author's page.
-        data = self._fetchUrl(self.story.getMetadata('authorUrl'))
-        soup = self.make_soup(data)
-        #find the story link and its parent div
-        storya = soup.find('a',{'href':self.story.getMetadata('storyUrl')})
-        storydiv = storya.parent
-        #warnings come under a <spawn> tag. Never seen that before...
-        #appears to just be a line of freeform text, not necessarily a list
-        #optional
-        spawn = storydiv.find('spawn',{'id':'warnings'})
-        if spawn is not None:
-            warns = spawn.nextSibling.strip()
-            self.story.addToList('warnings',warns)
-        #some meta in spans - this should get all, even the ones jammed in a table
-        spans = storydiv.findAll('span')
-        for s in spans:
-            if s.string == 'Ship:':
-                list = s.nextSibling.strip().split()
-                self.story.extendList('ships',list)
-            if s.string == 'Characters:':
-                list = s.nextSibling.strip().split(',')
-                self.story.extendList('characters',list)
-            if s.string == 'Status:':
-                st = s.nextSibling.strip()
-                self.story.setMetadata('status',st)
-            if s.string == 'Words:':
-                st = s.nextSibling.strip()
-                self.story.setMetadata('numWords',st)
-
-        #reviews - is this worth having?
-        #ffnet adapter gathers it, don't know if anything else does
-        #or if it's ever going to be used!
-        a = storydiv.find('a',{'id':'bold-blue'})
-        if a:
-            revs = a.nextSibling.strip()[1:-1]
-            self.story.setMetadata('reviews',st)
-        else:
-            revs = '0'
-            self.story.setMetadata('reviews',st)
-
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = self.make_soup(self._fetchUrl(url))
-
-        div = soup.find('div', {'id' : 'fanfiction'})
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-
-        #remove all the unnecessary bookmark tags
-        [s.extract() for s in div('div',{'class':"tiny_box2"})]
-
-        #is there a review link?
-        r = div.find('a',href=re.compile(re.escape("./index.php?act=irv")+".*$"))
-        if r is not None:
-        #remove the review link and its parent div
-            r.parent.extract()
-
-        #There might also be a link to the sequel on the last chapter
-        #I'm inclined to keep it in, but the URL needs to be changed from relative to absolute
-        #Shame there isn't proper series metadata available
-        #(I couldn't find it anyway)
-        s = div.find('a',href=re.compile(re.escape("./index.php?act=ovr")+".*$"))
-        if s is not None:
-            s['href'] = 'http://'+self.getSiteDomain()+'/fanfictions'+s['href'][1:]
-
-        return self.utf8FromSoup(url,div)

fanficfare/adapters/adapter_ficbooknet.py

@@ -55,7 +55,7 @@ class FicBookNetAdapter(BaseSiteAdapter):
 
 
         # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/readfic/'+self.story.getMetadata('storyId'))
+        self._setURL('https://' + self.getSiteDomain() + '/readfic/'+self.story.getMetadata('storyId'))
 
         # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','fbn')
@@ -71,10 +71,10 @@ class FicBookNetAdapter(BaseSiteAdapter):
 
     @classmethod
     def getSiteExampleURLs(cls):
-        return "http://"+cls.getSiteDomain()+"/readfic/12345 http://"+cls.getSiteDomain()+"/readfic/93626/246417#part_content"
+        return "https://"+cls.getSiteDomain()+"/readfic/12345 https://"+cls.getSiteDomain()+"/readfic/93626/246417#part_content"
 
     def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/readfic/")+r"\d+"
+        return r"https?://"+re.escape(self.getSiteDomain()+"/readfic/")+r"\d+"
 
     ## Getting the chapter list and the meta data, plus 'is adult' checking.
     def extractChapterUrlsAndMetadata(self):
@@ -92,39 +92,51 @@ class FicBookNetAdapter(BaseSiteAdapter):
         # use BeautifulSoup HTML parser to make everything easier to find.
         soup = self.make_soup(data)
 
+        adult_div = soup.find('div',id='adultCoverWarning')
+        if adult_div:
+            if self.is_adult or self.getConfig("is_adult"):
+                adult_div.extract()
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+        
         # Now go hunting for all the meta data and the chapter list.
 		
-        table = soup.find('td',{'width':'50%'})
-
         ## Title
-        a = soup.find('h1')
+        a = soup.find('section',{'class':'chapter-info'}).find('h1')
+        # kill '+' marks if present.
+        sup = a.find('sup')
+        if sup:
+            sup.extract()
         self.story.setMetadata('title',stripHTML(a))
         logger.debug("Title: (%s)"%self.story.getMetadata('title'))
 
         # Find authorid and URL from... author url.
-        a = table.find('a')
+        # assume first avatar-nickname -- there can be a second marked 'beta'.
+        a = soup.find('a',{'class':'avatar-nickname'})
         self.story.setMetadata('authorId',a.text) # Author's name is unique
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
+        self.story.setMetadata('authorUrl','https://'+self.host+'/'+a['href'])
         self.story.setMetadata('author',a.text)
         logger.debug("Author: (%s)"%self.story.getMetadata('author'))
 
         # Find the chapters:
-        chapters = soup.find('div', {'class' : 'part_list'})
+        chapters = soup.find('ul', {'class' : 'table-of-contents'})
         if chapters != None:
             chapters=chapters.findAll('a', href=re.compile(r'/readfic/'+self.story.getMetadata('storyId')+"/\d+#part_content$"))
             self.story.setMetadata('numChapters',len(chapters))
             for x in range(0,len(chapters)):
                 chapter=chapters[x]
-                churl='http://'+self.host+chapter['href']
+                churl='https://'+self.host+chapter['href']
                 self.chapterUrls.append((stripHTML(chapter),churl))
                 if x == 0:
-                    pubdate = translit.translit(stripHTML(self.make_soup(self._fetchUrl(churl)).find('div', {'class' : 'part_added'}).find('span')))
+                    pubdate = translit.translit(stripHTML(chapter.parent.find('span')))
+                    # pubdate = translit.translit(stripHTML(self.make_soup(self._fetchUrl(churl)).find('div', {'class' : 'part_added'}).find('span')))
                 if x == len(chapters)-1:
-                    update = translit.translit(stripHTML(self.make_soup(self._fetchUrl(churl)).find('div', {'class' : 'part_added'}).find('span')))
+                    update = translit.translit(stripHTML(chapter.parent.find('span')))
+                    # update = translit.translit(stripHTML(self.make_soup(self._fetchUrl(churl)).find('div', {'class' : 'part_added'}).find('span')))
         else:
             self.chapterUrls.append((self.story.getMetadata('title'),url))
             self.story.setMetadata('numChapters',1)
-            pubdate=translit.translit(stripHTML(soup.find('div', {'class' : 'part_added'}).find('span')))
+            pubdate=translit.translit(stripHTML(soup.find('div',{'class':'title-area'}).find('span')))
             update=pubdate
 
         logger.debug("numChapters: (%s)"%self.story.getMetadata('numChapters'))
@@ -158,54 +170,63 @@ class FicBookNetAdapter(BaseSiteAdapter):
         self.story.setMetadata('dateUpdated', makeDate(update, self.dateformat))
         self.story.setMetadata('datePublished', makeDate(pubdate, self.dateformat))
         self.story.setMetadata('language','Russian')
-		
-        pr=soup.find('a', href=re.compile(r'/printfic/\w+'))
-        pr='http://'+self.host+pr['href']
-        pr = self.make_soup(self._fetchUrl(pr))
-        pr=pr.findAll('div', {'class' : 'part_text'})
+
+        ## after site change, I don't see word count anywhere.
+        # pr=soup.find('a', href=re.compile(r'/printfic/\w+'))
+        # pr='https://'+self.host+pr['href']
+        # pr = self.make_soup(self._fetchUrl(pr))
+        # pr=pr.findAll('div', {'class' : 'part_text'})
+        # i=0
+        # for part in pr:
+        #     i=i+len(stripHTML(part).split(' '))
+        # self.story.setMetadata('numWords', unicode(i))
+
+
+        dlinfo = soup.find('dl',{'class':'info'})
+        
         i=0
-        for part in pr:
-            i=i+len(stripHTML(part).split(' '))
-        self.story.setMetadata('numWords', unicode(i))
-		
-        i=0
-        fandoms = table.findAll('a', href=re.compile(r'/fanfiction/\w+'))
+        fandoms = dlinfo.find('dd').findAll('a', href=re.compile(r'/fanfiction/\w+'))
         for fandom in fandoms:
             self.story.addToList('category',fandom.string)
             i=i+1
         if i > 1:
             self.story.addToList('genre', u'Кроссовер')
-		
-        meta=table.findAll('a', href=re.compile(r'/ratings/'))
-        i=0
-        for m in meta:
-            if i == 0:
-                self.story.setMetadata('rating', stripHTML(m))
-                i=1
-            elif i == 1:
-                if not "," in m.nextSibling:
-                    i=2
-                self.story.addToList('genre', m.find('b').text)
-            elif i == 2:
-                self.story.addToList('warnings', m.find('b').text)
-		
 
-        if table.find('span', {'style' : 'color: green'}):
+        for genre in dlinfo.findAll('a',href=re.compile(r'/genres/')):
+            self.story.addToList('genre',stripHTML(genre))
+
+        ratingdt = dlinfo.find('dt',text='Рейтинг:')
+        self.story.setMetadata('rating', stripHTML(ratingdt.next_sibling))
+            
+        # meta=table.findAll('a', href=re.compile(r'/ratings/'))
+        # i=0
+        # for m in meta:
+        #     if i == 0:
+        #         self.story.setMetadata('rating', stripHTML(m))
+        #         i=1
+        #     elif i == 1:
+        #         if not "," in m.nextSibling:
+        #             i=2
+        #         self.story.addToList('genre', m.find('b').text)
+        #     elif i == 2:
+        #         self.story.addToList('warnings', m.find('b').text)		
+
+        if dlinfo.find('span', {'style' : 'color: green'}):
             self.story.setMetadata('status', 'Completed')
         else:
             self.story.setMetadata('status', 'In-Progress')
 		
 
-        tags = table.findAll('b')
+        tags = dlinfo.findAll('dt')
         for tag in tags:
             label = translit.translit(tag.text)
             if 'Piersonazhi:' in label or u'Персонажи:' in label:
-                chars=tag.nextSibling.string.split(', ')
+                chars=stripHTML(tag.next_sibling).split(', ')
                 for char in chars:
                     self.story.addToList('characters',char)
                 break
 				
-        summary=soup.find('span', {'class' : 'urlize'})
+        summary=soup.find('div', {'class' : 'urlize'})
         self.setDescription(url,summary)
         #self.story.setMetadata('description', summary.text)
 

fanficfare/adapters/adapter_fictionmaniatv.py

@@ -117,7 +117,7 @@ class FictionManiaTVAdapter(BaseSiteAdapter):
                 self.story.setMetadata('rating', value)
 
             elif key == 'Complete':
-                self.story.setMetadata('status', 'Complete' if value == 'Complete' else 'In-Progress')
+                self.story.setMetadata('status', 'Completed' if value == 'Complete' else 'In-Progress')
 
             elif key == 'Categories':
                 for element in cells[1]('a'):

fanficfare/adapters/adapter_ficwadcom.py

@@ -93,7 +93,7 @@ class FicwadComSiteAdapter(BaseSiteAdapter):
         try:
             data = self._fetchUrl(url)
             # non-existent/removed story urls get thrown to the front page.
-            if "<h2>Welcome to FicWad</h2>" in data:
+            if "<h4>Featured Story</h4>" in data:
                 raise exceptions.StoryDoesNotExist(self.url)
             soup = self.make_soup(data)
         except urllib2.HTTPError, e:

fanficfare/adapters/adapter_grangerenchantedcom.py

@@ -1,309 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2012 Fanficdownloader team, 2015 FanFicFare team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Software: eFiction
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return GrangerEnchantedCom
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class GrangerEnchantedCom(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-
-        self.section=self.parsedUrl.path.split('/',)[1]
-
-        # normalized story URL.
-        if "malfoymanor" in self.parsedUrl.netloc:
-            self._setURL('http://malfoymanor.' + self.getSiteDomain() + '/themanor/viewstory.php?sid='+self.story.getMetadata('storyId'))
-            self.story.addToList("category","The Manor")
-        else:
-            self._setURL('http://' + self.getSiteDomain() + '/enchant/viewstory.php?sid='+self.story.getMetadata('storyId'))
-
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','gech')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%d/%b/%Y"
-
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'grangerenchanted.com'
-
-    @classmethod
-    def getAcceptDomains(cls):
-        return ['grangerenchanted.com','malfoymanor.grangerenchanted.com']
-
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://grangerenchanted.com/enchant/viewstory.php?sid=1234 http://malfoymanor.grangerenchanted.com/themanor/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return r"http://(malfoymanor.)?grangerenchanted.com/(enchant|themanor)?/viewstory.php\?sid=\d+$"
-
-    ## Login seems to be reasonably standard across eFiction sites.
-    def needToLoginCheck(self, data):
-        if 'Registered Users Only' in data \
-                or 'There is no such account on our website' in data \
-                or "That password doesn't match the one in our database" in data:
-            return True
-        else:
-            return False
-
-    def performLogin(self, url):
-        params = {}
-
-        if self.password:
-            params['penname'] = self.username
-            params['password'] = self.password
-        else:
-            params['penname'] = self.getConfig("username")
-            params['password'] = self.getConfig("password")
-        params['cookiecheck'] = '1'
-        params['submit'] = 'Submit'
-
-        if "enchant" in self.section:
-            loginUrl = 'http://grangerenchanted.com/enchant/user.php?action=login'
-        else:
-            loginUrl = 'http://malfoymanor.grangerenchanted.com/themanor/user.php?action=login'
-        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
-                                                              params['penname']))
-
-        d = self._fetchUrl(loginUrl, params)
-
-        if "Member Account" not in d : #Member Account
-            logger.info("Failed to login to URL %s as %s" % (loginUrl,
-                                                              params['penname']))
-            raise exceptions.FailedToLogin(url,params['penname'])
-            return False
-        else:
-            return True
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        if self.is_adult or self.getConfig("is_adult"):
-            # Weirdly, different sites use different warning numbers.
-            # If the title search below fails, there's a good chance
-            # you need a different number.  print data at that point
-            # and see what the 'click here to continue' url says.
-            addurl = "&ageconsent=ok&warning=1"
-        else:
-            addurl=""
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url+addurl
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        if self.needToLoginCheck(data):
-            # need to log in for this one.
-            self.performLogin(url)
-            data = self._fetchUrl(url)
-
-        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
-        if m != None:
-            if self.is_adult or self.getConfig("is_adult"):
-                # We tried the default and still got a warning, so
-                # let's pull the warning number from the 'continue'
-                # link and reload data.
-                addurl = m.group(1)
-                # correct stupid & error in url.
-                addurl = addurl.replace("&","&")
-                url = self.url+'&index=1'+addurl
-                logger.debug("URL 2nd try: "+url)
-
-                try:
-                    data = self._fetchUrl(url)
-                except urllib2.HTTPError, e:
-                    if e.code == 404:
-                        raise exceptions.StoryDoesNotExist(self.url)
-                    else:
-                        raise e
-            else:
-                raise exceptions.AdultCheckRequired(self.url)
-
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.AccessDenied(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = self.make_soup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-
-        ## Title
-        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',stripHTML(a))
-
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
-            # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+self.section+'/'+chapter['href']+addurl))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        # utility method
-        def defaultGetattr(d,k):
-            try:
-                return d[k]
-            except:
-                return ""
-
-        # <span class="label">Rated:</span> NC-17<br /> etc
-        labels = soup.findAll('span',{'class':'label'})
-        for labelspan in labels:
-            value = labelspan.nextSibling
-            label = labelspan.string
-
-            if 'Summary' in label:
-                ## Everything until the next span class='label'
-                svalue = ""
-                while value and 'label' not in defaultGetattr(value,'class') and '<span class="label">' not in unicode(value):
-                    svalue += unicode(value)
-                    value = value.nextSibling
-                self.setDescription(url,svalue)
-
-            if 'Rated' in label:
-                self.story.setMetadata('rating', value)
-
-            if 'Word count' in label:
-                self.story.setMetadata('numWords', value)
-
-            if 'Read' in label:
-                self.story.setMetadata('read', value)
-
-            if 'Categories' in label:
-                cats = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-                for cat in cats:
-                    self.story.addToList('category',cat.string)
-
-            if 'Characters' in label:
-                chars = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-                for char in chars:
-                    self.story.addToList('characters',char.string)
-
-            if 'Genre' in label:
-                genres = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=4'))
-                for genre in genres:
-                    self.story.addToList('genre',genre.string)
-
-            if 'Warnings' in label:
-                warnings = labelspan.parent.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
-                for warning in warnings:
-                    self.story.addToList('warnings',warning.string)
-
-            if 'Completed' in label:
-                if 'Yes' in value:
-                    self.story.setMetadata('status', 'Completed')
-                else:
-                    self.story.setMetadata('status', 'In-Progress')
-
-            if 'Published' in label:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
-
-            if 'Updated' in label:
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
-
-        try:
-            # Find Series name from series URL.
-            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
-            series_name = a.string
-            series_url = 'http://'+self.host+'/'+self.section+'/'+a['href']
-
-            # use BeautifulSoup HTML parser to make everything easier to find.
-            seriessoup = self.make_soup(self._fetchUrl(series_url))
-            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
-            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
-            i=1
-            for a in storyas:
-                # skip 'report this' and 'TOC' links
-                if 'contact.php' not in a['href'] and 'index' not in a['href']:
-                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                        self.setSeries(series_name, i)
-                        self.story.setMetadata('seriesUrl',series_url)
-                        break
-                    i+=1
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-
-        try:
-            self.story.setMetadata('reviews',
-                                   stripHTML(soup.find('div',{'id':'sort'}).
-                                             findAll('a', href=re.compile(r'^reviews.php'))[1]))
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = self.make_soup(self._fetchUrl(url))
-
-        div = soup.find('div', {'id' : 'story1'})
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-
-        return self.utf8FromSoup(url,div)

fanficfare/adapters/adapter_harrypotterfanfictioncom.py

@@ -188,9 +188,13 @@ class HarryPotterFanFictionComSiteAdapter(BaseSiteAdapter):
 
         data = self._fetchUrl(url)
 
-        # remove everything after here--the site's chapters break the
-        # BS4 parser.
-        data = data[:data.index('<script type="text/javascript" src="reviewjs.js">')]
+        try:
+            # remove everything after here--the site's chapters break
+            # the BS4 parser.
+            data = data[:data.index('<script type="text/javascript" src="reviewjs.js">')]
+        except:
+            # some older stories don't have the code at the end that breaks things.
+            pass
         
         soup = self.make_soup(data)
 

fanficfare/adapters/adapter_literotica.py

@@ -32,13 +32,15 @@ class LiteroticaSiteAdapter(BaseSiteAdapter):
 
     def __init__(self, config, url):
         BaseSiteAdapter.__init__(self, config, url)
+        logger.debug("LiteroticaComAdapter:__init__ - url='%s'" % url)
 
         self.decode = ["utf8",
                        "Windows-1252"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
+                            # Most sites that claim to be
+                            # iso-8859-1 (and some that claim to be
+                            # utf8) are really windows-1252.
 
+        # Each adapter needs to have a unique site abbreviation.
         self.story.setMetadata('siteabbrev','litero')
 
         # normalize to first chapter.  Not sure if they ever have more than 2 digits.
@@ -61,7 +63,7 @@ class LiteroticaSiteAdapter(BaseSiteAdapter):
 
         # The date format will vary from site to site.
         # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = '%m/%d/%y'
+        self.dateformat = "%m/%d/%y"
 
     @staticmethod
     def getSiteDomain():
@@ -95,6 +97,18 @@ class LiteroticaSiteAdapter(BaseSiteAdapter):
     def getSiteURLPattern(self):
         return r"https?://(www|german|spanish|french|dutch|italian|romanian|portuguese|other)(\.i)?\.literotica\.com/s/([a-zA-Z0-9_-]+)"
 
+    def getCategories(self, soup):
+        if self.getConfig("use_meta_keywords"):
+            categories = soup.find("meta", {"name":"keywords"})['content'].split(', ')
+            categories = [c for c in categories if not self.story.getMetadata('title') in c]
+            if self.story.getMetadata('author') in categories:
+                categories.remove(self.story.getMetadata('author'))
+            logger.debug("Meta = %s" % categories)
+            for category in categories:
+    #            logger.debug("\tCategory=%s" % category)
+#                 self.story.addToList('category', category.title())
+                self.story.addToList('eroticatags', category.title())
+
     def extractChapterUrlsAndMetadata(self):
         """
         NOTE: Some stories can have versions,
@@ -118,6 +132,7 @@ class LiteroticaSiteAdapter(BaseSiteAdapter):
             raise exceptions.AdultCheckRequired(self.url)
 
         logger.debug("Chapter/Story URL: <%s> " % self.url)
+
         try:
             data1 = self._fetchUrl(self.url)
             soup1 = self.make_soup(data1)
@@ -144,6 +159,7 @@ class LiteroticaSiteAdapter(BaseSiteAdapter):
             soupAuth = self.make_soup(dataAuth)
             #strip comments from soup
             [comment.extract() for comment in soupAuth.findAll(text=lambda text:isinstance(text, Comment))]
+#            logger.debug(soupAuth)
         except urllib2.HTTPError, e:
             if e.code == 404:
                 raise exceptions.StoryDoesNotExist(authorurl)
@@ -154,6 +170,15 @@ class LiteroticaSiteAdapter(BaseSiteAdapter):
         ## site has started using //domain.name/asdf urls remove https?: from front
         ## site has started putting https back on again.
         storyLink = soupAuth.find('a', href=re.compile(r'(https?:)?'+re.escape(self.url[self.url.index(':')+1:])))
+#         storyLink = soupAuth.find('a', href=self.url)#[self.url.index(':')+1:])
+
+        if storyLink is not None:
+            # pull the published date from the author page
+            # default values from single link.  Updated below if multiple chapter.
+            logger.debug("Found story on the author page.")
+            date = storyLink.parent.parent.findAll('td')[-1].text
+            self.story.setMetadata('datePublished', makeDate(date, self.dateformat))
+            self.story.setMetadata('dateUpdated',makeDate(date, self.dateformat))
 
         if storyLink is not None:
             urlTr = storyLink.parent.parent
@@ -165,9 +190,14 @@ class LiteroticaSiteAdapter(BaseSiteAdapter):
             raise exceptions.FailedToDownload("Couldn't find story <%s> on author's page <%s>" % (self.url, authorurl))
 
         if isSingleStory:
-            self.story.setMetadata('title', storyLink.text)
-            self.setDescription(authorurl,urlTr.findAll("td")[1].text)
-            self.story.addToList('eroticatags', urlTr.findAll("td")[2].text)
+#             self.chapterUrls = [(soup1.h1.string, self.url)]
+#             self.story.setMetadata('title', soup1.h1.string)
+
+            self.story.setMetadata('title', storyLink.text.strip('/'))
+            logger.debug('Title: "%s"' % storyLink.text.strip('/'))
+            self.story.setMetadata('description', urlTr.findAll("td")[1].text)
+            self.story.addToList('category', urlTr.findAll("td")[2].text)
+#             self.story.addToList('eroticatags', urlTr.findAll("td")[2].text)
             date = urlTr.findAll('td')[-1].text
             self.story.setMetadata('datePublished', makeDate(date, self.dateformat))
             self.story.setMetadata('dateUpdated',makeDate(date, self.dateformat))
@@ -175,13 +205,19 @@ class LiteroticaSiteAdapter(BaseSiteAdapter):
             averrating = stripHTML(storyLink.parent)
             ## title (0.00)
             averrating = averrating[averrating.rfind('(')+1:averrating.rfind(')')]
-            self.story.setMetadata('averrating',averrating)
+            try:
+                self.story.setMetadata('averrating', float(averrating))
+            except:
+                pass
+#             self.story.setMetadata('averrating',averrating)
+        # parse out the list of chapters
         else:
             seriesTr = urlTr.previousSibling
             while 'ser-ttl' not in seriesTr['class']:
                 seriesTr = seriesTr.previousSibling
             m = re.match("^(?P<title>.*?):\s(?P<numChapters>\d+)\sPart\sSeries$", seriesTr.find("strong").text)
             self.story.setMetadata('title', m.group('title'))
+            seriesTitle = m.group('title')
 
             ## Walk the chapters
             chapterTr = seriesTr.nextSibling
@@ -189,88 +225,149 @@ class LiteroticaSiteAdapter(BaseSiteAdapter):
             dates = []
             descriptions = []
             ratings = []
+            chapters = []
             while chapterTr is not None and 'sl' in chapterTr['class']:
-                descriptions.append("%d. %s" % (len(descriptions)+1,stripHTML(chapterTr.findAll("td")[1])) )
+                description = "%d. %s" % (len(descriptions)+1,stripHTML(chapterTr.findAll("td")[1]))
+                description = stripHTML(chapterTr.findAll("td")[1])
                 chapterLink = chapterTr.find("td", "fc").find("a")
-                if not chapterLink["href"].startswith('http'):
-                    chapterLink["href"] = "http:" + chapterLink["href"]
-                self.chapterUrls.append((chapterLink.text, chapterLink["href"]))
                 self.story.addToList('eroticatags', chapterTr.findAll("td")[2].text)
-                dates.append(makeDate(chapterTr.findAll('td')[-1].text, self.dateformat))
+                pub_date = makeDate(chapterTr.findAll('td')[-1].text, self.dateformat)
+                dates.append(pub_date)
                 chapterTr = chapterTr.nextSibling
+                
+                chapter_title = chapterLink.text
+                if self.getConfig("clean_chapter_titles"):
+                    logger.debug('\tChapter Name: "%s"' % chapterLink.string)
+                    logger.debug('\tChapter Name: "%s"' % chapterLink.text)
+                    if chapterLink.text.lower().startswith(seriesTitle.lower()):
+                        chapter = chapterLink.text[len(seriesTitle):].strip()
+                        logger.debug('\tChapter: "%s"' % chapter)
+                        if chapter == '':
+                            chapter_title = 'Chapter %d' % (len(self.chapterUrls) + 1)
+                        else:
+                            separater_char = chapter[0]
+                            logger.debug('\tseparater_char: "%s"' % separater_char)
+                            chapter = chapter[1:].strip() if separater_char in [":", "-"] else chapter
+                            logger.debug('\tChapter: "%s"' % chapter)
+                            if chapter.lower().startswith('ch.'):
+                                chapter = chapter[len('ch.'):]
+                                try:
+                                    chapter_title = 'Chapter %d' % int(chapter)
+                                except:
+                                    chapter_title = 'Chapter %s' % chapter
+                            elif chapter.lower().startswith('pt.'):
+                                chapter = chapter[len('pt.'):]
+                                try:
+                                    chapter_title = 'Part %d' % int(chapter)
+                                except:
+                                    chapter_title = 'Part %s' % chapter
+                            elif separater_char in [":", "-"]:
+                                chapter_title = chapter
+    
+    #                 if chapter_title == '':
+    #                     chapter_title = chapterLink.string
+
+                # pages include full URLs.
+                chapurl = chapterLink['href']
+                if chapurl.startswith('//'):
+                    chapurl = self.parsedUrl.scheme + ':' + chapurl
+                logger.debug("Chapter URL: " + chapurl)
+                logger.debug("Chapter Title: " + chapter_title)
+                logger.debug("Chapter description: " + description)
+                chapters.append((chapter_title, chapurl, description, pub_date))
+#                 self.chapterUrls.append((chapter_title, chapurl))
                 numrating = stripHTML(chapterLink.parent)
                 ## title (0.00)
                 numrating = numrating[numrating.rfind('(')+1:numrating.rfind(')')]
-                ratings.append(float(numrating))
-
-            ## Set description to joint chapter descriptions
-            self.setDescription(authorurl,"<p>"+"</p>\n<p>".join(descriptions)+"</p>")
+                try:
+                    ratings.append(float(numrating))
+                except:
+                    pass
 
+            chapters = sorted(chapters, key=lambda chapter: chapter[3])
+            for i, chapter in enumerate(chapters):
+                self.chapterUrls.append((chapter[0], chapter[1]))
+                descriptions.append("%d. %s" % (i + 1, chapter[2]))
             ## Set the oldest date as publication date, the newest as update date
             dates.sort()
             self.story.setMetadata('datePublished', dates[0])
             self.story.setMetadata('dateUpdated', dates[-1])
+            self.story.setMetadata('datePublished', chapters[0][3])
+            self.story.setMetadata('dateUpdated', chapters[-1][3])
+            ## Set description to joint chapter descriptions
+            self.setDescription(authorurl,"<p>"+"</p>\n<p>".join(descriptions)+"</p>")
 
-            # normalize on first chapter URL.
-            self._setURL(self.chapterUrls[0][1])
+            if len(ratings) > 0:
+                self.story.setMetadata('averrating','%4.2f' % (sum(ratings) / float(len(ratings))))
 
-            self.story.setMetadata('averrating','%4.2f' % (sum(ratings) / float(len(ratings))))
+        # normalize on first chapter URL.
+        self._setURL(self.chapterUrls[0][1])
+
+        # reset storyId to first chapter.
+        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2])
 
         self.story.setMetadata('numChapters', len(self.chapterUrls))
 
-        # set storyId to 'title-author' to avoid duplicates
-        # self.story.setMetadata('storyId',
-        #     re.sub("[^a-z0-9]", "", self.story.getMetadata('title').lower())
-        #     + "-"
-        #     + re.sub("[^a-z0-9]", "", self.story.getMetadata('author').lower()))
+        self.story.setMetadata('category', soup1.find('div', 'b-breadcrumbs').findAll('a')[1].string)
+        self.getCategories(soup1)
+#         self.story.setMetadata('description', soup1.find('meta', {'name': 'description'})['content'])
 
         return
 
+
+    def getPageText(self, raw_page, url):
+        logger.debug('Getting page text')
+#         logger.debug(soup)
+        raw_page = raw_page.replace('<div class="b-story-body-x x-r15"><div><p>','<div class="b-story-body-x x-r15"><div>')
+#         logger.debug("\tChapter text: %s" % raw_page)
+        page_soup = self.make_soup(raw_page)
+        [comment.extract() for comment in page_soup.findAll(text=lambda text:isinstance(text, Comment))]
+        story2 = page_soup.find('div', 'b-story-body-x').div
+#         logger.debug("getPageText- name div div...")
+#         logger.debug(soup)
+#         story2.append(page_soup.new_tag('br'))
+        div = self.utf8FromSoup(url, story2)
+#        logger.debug(div)
+
+        fullhtml = unicode(div)
+#         logger.debug(fullhtml)
+        fullhtml = re.sub(r'<br />\s*<br />', r'</p><p>', fullhtml)
+        fullhtml = re.sub(r'^<div>', r'', fullhtml)
+        fullhtml = re.sub(r'</div>$', r'', fullhtml)
+        fullhtml = re.sub(r'(<p><br/></p>\s+)+$', r'', fullhtml)
+#         logger.debug(fullhtml)
+        return fullhtml
+
     def getChapterText(self, url):
-        logger.debug('Getting chapter text from <%s>' % url)
-        data1 = self._fetchUrl(url)
-        # brute force approach to replace the wrapping <p> tag.  If
-        # done by changing tag name, it causes problems with nested
-        # <p> tags.
-        data1 = data1.replace('<div class="b-story-body-x x-r15"><div><p>','<div class="b-story-body-x x-r15"><div>')
-        soup1 = self.make_soup(data1)
 
-        #strip comments from soup
-        [comment.extract() for comment in soup1.findAll(text=lambda text:isinstance(text, Comment))]
+        logger.debug('Getting chapter text from: %s' % url)
 
-        # get story text
-        story1 = soup1.find('div', 'b-story-body-x').div
-        #print("story1:%s"%story1)
-        # story1.name='div'
-        story1.append(soup1.new_tag('br'))
-        storytext = self.utf8FromSoup(url,story1)
+        raw_page = self._fetchUrl(url)
+        page_soup = self.make_soup(raw_page)
+        pages = page_soup.find('select', {'name' : 'page'})
+        page_nums = [page.text for page in pages.findAll('option')] if pages else 0
 
-        # find num pages
-        pgs = int(soup1.find("span", "b-pager-caption-t r-d45").string.split(' ')[0])
-        logger.debug("pages: "+unicode(pgs))
+        fullhtml = "" 
+        self.getCategories(page_soup)
+        if self.getConfig("description_in_chapter"):
+            chapter_description = page_soup.find("meta", {"name" : "description"})['content']
+            logger.debug("\tChapter description: %s" % chapter_description)
+            fullhtml += '<p><b>Description:</b> %s</p><hr />' % chapter_description
+        fullhtml += self.getPageText(raw_page, url)
+        if pages:
+            for page_no in xrange(2, len(page_nums) + 1):
+                page_url = url +  "?page=%s" % page_no
+                logger.debug("page_url= %s" % page_url)
+                raw_page = self._fetchUrl(page_url)
+                fullhtml += self.getPageText(raw_page, url)
+        
+#        fullhtml = self.utf8FromSoup(url, bs.BeautifulSoup(fullhtml))
+#        fullhtml = re.sub(r'^<div>', r'', fullhtml)
+#        fullhtml = re.sub(r'</div>$', r'', fullhtml)
+#        if None == div:
+#            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
 
-        # get all the pages
-        for i in xrange(2, pgs+1):
-            try:
-                logger.debug("fetching page "+unicode(i))
-                time.sleep(0.5)
-                data2 = self._fetchUrl(url, {'page': i})
-                # brute force approach to replace the wrapping <p> tag.  If
-                # done by changing tag name, it causes problems with nested
-                # <p> tags.
-                data2 = data2.replace('<div class="b-story-body-x x-r15"><div><p>','<div class="b-story-body-x x-r15"><div>')
-                soup2 = self.make_soup(data2)
-                [comment.extract() for comment in soup2.findAll(text=lambda text:isinstance(text, Comment))]
-                story2 = soup2.find('div', 'b-story-body-x').div
-                # story2.name='div'
-                story2.append(soup2.new_tag('br'))
-                storytext += self.utf8FromSoup(url,story2)
-            except urllib2.HTTPError, e:
-                if e.code == 404:
-                    raise exceptions.StoryDoesNotExist(url)
-                else:
-                    raise e
-        return storytext
+        return fullhtml
 
 
 def getClass():

fanficfare/adapters/adapter_ncisfictionnet.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-# Copyright 2012 Fanficdownloader team, 2015 FanFicFare team
+# Copyright 2012 Fanficdownloader team, 2016 FanFicFare team
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,195 +15,22 @@
 # limitations under the License.
 #
 
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
+# Software: eFiction
+from base_efiction_adapter import BaseEfictionAdapter
 
+class NCISFictionNetAdapter(BaseEfictionAdapter):
 
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
+    @staticmethod
+    def getSiteDomain():
+        return 'ncisfiction.net'
 
-from base_adapter import BaseSiteAdapter,  makeDate
+    @classmethod
+    def getSiteAbbrev(self):
+        return 'ncisfn'
+
+    @classmethod
+    def getDateFormat(self):
+        return "%m/%d/%Y"
 
 def getClass():
     return NCISFictionNetAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class NCISFictionNetAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["iso-8859-1",
-                       "Windows-1252"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-
-
-        # normalized story URL.
-        self._setURL("http://"+self.getSiteDomain()\
-                         +"/chapters.php?stid="+self.story.getMetadata('storyId'))
-
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','ncisfn')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%d/%m/%Y"
-
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'www.ncisfiction.net'
-
-    ## Changed from www.ncisfiction.com to www.ncisfiction.net Oct
-    ## 2012 due to the ncisfiction.com domain expiring.  Still accept
-    ## .com domains for existing updates, etc.
-
-    @classmethod
-    def getAcceptDomains(cls):
-        return ['www.ncisfiction.net','www.ncisfiction.com']
-
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://"+cls.getSiteDomain()+"/story.php?stid=01234 http://"+cls.getSiteDomain()+"/chapters.php?stid=1234"
-
-    def getSiteURLPattern(self):
-        return r'http://www\.ncisfiction\.(net|com)/(chapters|story)?.php\?stid=\d+'
-
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.AccessDenied(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = self.make_soup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-
-        ## Title and author
-        a = soup.find('div', {'class' : 'main_title'})
-
-        aut = a.find('a')
-        self.story.setMetadata('authorId',aut['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+aut['href'])
-        self.story.setMetadata('author',aut.string)
-
-        aut.extract()
-        self.story.setMetadata('title',stripHTML(a)[:len(stripHTML(a))-2])
-
-        # Find the chapters:
-        i=0
-        chapters=soup.findAll('table', {'class' : 'story_table'})
-        for chapter in chapters:
-            ch=chapter.find('a')
-            # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(ch),'http://'+self.host+'/'+ch['href']))
-            if i == 0:
-                self.story.setMetadata('datePublished', makeDate(stripHTML(chapter.find('td')).split('Added: ')[1], self.dateformat))
-            if i == len(chapters)-1:
-                self.story.setMetadata('dateUpdated', makeDate(stripHTML(chapter.find('td')).split('Added: ')[1], self.dateformat))
-            i=i+1
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        info = soup.find('table', {'class' : 'story_info'})
-
-        # no convenient way to calculate word count as it is logged differently for stories with and without series
-
-        labels = info.findAll('tr')
-        for tr in labels:
-            value = tr.find('td')
-            label = tr.find('th').string
-
-            if 'Summary' in label:
-                self.setDescription(url,value)
-
-            if 'Rating' in label:
-                self.story.setMetadata('rating', value.string)
-
-            if 'Category' in label:
-                cats = value.findAll('a')
-                for cat in cats:
-                    self.story.addToList('category',cat.string)
-
-            if 'Characters' in label:
-                chars = value.findAll('a')
-                for char in chars:
-                    self.story.addToList('characters',char.string)
-
-            if 'Pairing' in label:
-                ships = value.findAll('a')
-                for ship in ships:
-                    self.story.addToList('ships',ship.string)
-
-            if 'Genre' in label:
-                genres = value.findAll('a')
-                for genre in genres:
-                    self.story.addToList('genre',genre.string)
-
-            if 'Warnings' in label:
-                warnings = value.findAll('a')
-                for warning in warnings:
-                    self.story.addToList('warnings',warning.string)
-
-            if 'Status' in label:
-                if 'not completed' in value.text:
-                    self.story.setMetadata('status', 'In-Progress')
-                else:
-                    self.story.setMetadata('status', 'Completed')
-
-        try:
-            # Find Series name from series URL.
-            a = soup.find('div',{'class' : 'sub_header'})
-            series_name = a.find('a').string
-            i = a.text.split('#')[1]
-            self.setSeries(series_name, i)
-            self.story.setMetadata('seriesUrl','http://'+self.host+'/'+a.find('a')['href'])
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = self.make_soup(self._fetchUrl(url))
-
-        div = soup.find('div', {'class' : 'story_text'})
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-
-        return self.utf8FromSoup(url,div)

fanficfare/adapters/adapter_nickandgregnet.py

@@ -1,179 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2013 Fanficdownloader team, 2015 FanFicFare team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Software: eFiction
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return NickAndGregNetAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class NickAndGregNetAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-
-
-        # normalized story URL.
-        # XXX Most sites don't have the /fanfic part.  Replace all to remove it usually.
-        self._setURL('http://' + self.getSiteDomain() + '/desert_archive/viewstory.php?sid='+self.story.getMetadata('storyId'))
-
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','nag')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%Y/%m/%d"
-
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'www.nickngreg.nl'
-
-    @classmethod
-    def getAcceptDomains(cls):
-        return ['www.nickngreg.nl','www.nickandgreg.net']
-
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://"+cls.getSiteDomain()+"/desert_archive/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return "http://("+self.getSiteDomain()+"|www.nickandgreg.net)"+re.escape("/desert_archive/viewstory.php?sid=")+r"\d+$"
-
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url+'&i=1'
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.AccessDenied(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = self.make_soup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-
-        ## Title
-        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',stripHTML(a))
-
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/desert_archive/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        # Find the chapters:
-        chapters = soup.find('select')
-        for chapter in chapters.findAll('option'):
-            if chapter.text != 'Story Index' and chapter.text != 'Chapters':
-                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/desert_archive/'+chapter['value']))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        asoup = self.make_soup(self._fetchUrl(self.story.getMetadata('authorUrl')))
-
-        for div in asoup.findAll('td', {'class' : 'tblborder6'}):
-            a = div.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-            if a != None:
-                break
-
-        self.setDescription(url,div.find('br').nextSibling)
-
-        a=div.text.split('Rating:')
-        if len(a) == 2: self.story.setMetadata('rating', a[1].split(' -')[0])
-
-        a=div.text.split('Characters:')
-        if len(a) == 2:
-        	for char in a[1].split(' -')[0].split(', '):
-        	    self.story.addToList('characters',char)
-        	
-        a=div.text.split('Genres:')
-        if len(a) == 2:
-        	for genre in a[1].split(' -')[0].split(', '):
-        	    self.story.addToList('genre',genre)
-        	
-        a=div.text.split('Warnings:')
-        if len(a) == 2:
-        	for warn in a[1].split(' -')[0].split(', '):
-        	    if 'none' not in warn:
-        	        self.story.addToList('warnings',warn)
-        	
-        a=div.text.split('Completed:')
-        if len(a) ==2:
-            if 'Yes' in a[1]:
-                self.story.setMetadata('status', 'Completed')
-            else:
-                self.story.setMetadata('status', 'In-Progress')
-
-        a=div.text.split('Published:')
-        if len(a) == 2: self.story.setMetadata('datePublished', makeDate(stripHTML(a[1].split(' -')[0]), self.dateformat))
-
-        a=div.text.split('Updated:')
-        if len(a) == 2: self.story.setMetadata('dateUpdated', makeDate(stripHTML(a[1].split(' -')[0]), self.dateformat))
-
-
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = self.make_soup(self._fetchUrl(url))
-
-        # wrap a div around it.
-        divsoup = self.make_soup('<div class="story"></div>')
-        div = divsoup.find('div')
-        div.append(soup.find('table', {'class' : 'tblborder6'}))
-
-        return self.utf8FromSoup(url,div)

fanficfare/adapters/adapter_quotevcom.py

@@ -63,17 +63,17 @@ class QuotevComAdapter(BaseSiteAdapter):
         title = element.find('h1')
         self.story.setMetadata('title', title.get_text())
         
-        authdiv = title.next_sibling # soup.find('div', {'style':"font-size:0.7em;color:#aaa;margin-top:-10px;text-align:center;margin-left:45px;cursor:pointer"})
+        authdiv = soup.find('div', {'class':"quizAuthorList"})
         if authdiv:
-            #print("div:%s"%authdiv.find_all('a'))
+            print("div:%s"%authdiv)
             for a in authdiv.find_all('a'):
                 self.story.addToList('author', a.get_text())
                 self.story.addToList('authorId', a['href'].split('/')[-1])
                 self.story.addToList('authorUrl', urlparse.urljoin(self.url, a['href']))
-        else:
-            self.story.setMetadata('author','Anonymous')
-            self.story.setMetadata('authorUrl','http://www.quotev.com')
-            self.story.setMetadata('authorId','0')
+        if not self.story.getList('author'):
+            self.story.addToList('author','Anonymous')
+            self.story.addToList('authorUrl','http://www.quotev.com')
+            self.story.addToList('authorId','0')
 
         self.setDescription(self.url, soup.find('div', id='qdesct'))
         imgmeta = soup.find('meta',{'property':"og:image" })

fanficfare/adapters/adapter_rabidreadercom.py

@@ -1,36 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2016 FanFicFare team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Software: eFiction
-from base_efiction_adapter import BaseEfictionAdapter
-
-class RabidReaderComAdapter(BaseEfictionAdapter):
-
-    @staticmethod
-    def getSiteDomain():
-        return 'www.therabidreader.com'
-
-    @classmethod
-    def getSiteAbbrev(self):
-        return 'rrcom'
-
-    @classmethod
-    def getDateFormat(self):
-        return "%d %b %Y"
-
-def getClass():
-    return RabidReaderComAdapter

fanficfare/adapters/adapter_scarheadnet.py

@@ -1,299 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2013 Fanficdownloader team, 2015 FanFicFare team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Software: eFiction
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return ScarHeadNetAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class ScarHeadNetAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-
-
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
-
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','shn')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%d/%m/%y"
-
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.
-        return 'scarhead.net'
-
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
-
-    ## Login seems to be reasonably standard across eFiction sites.
-    def needToLoginCheck(self, data):
-        if 'Registered Users Only' in data \
-                or 'There is no such account on our website' in data \
-                or "That password doesn't match the one in our database" in data:
-            return True
-        else:
-            return False
-
-    def performLogin(self, url):
-        params = {}
-
-        if self.password:
-            params['penname'] = self.username
-            params['password'] = self.password
-        else:
-            params['penname'] = self.getConfig("username")
-            params['password'] = self.getConfig("password")
-        params['cookiecheck'] = '1'
-        params['submit'] = 'Submit'
-
-        loginUrl = 'http://' + self.getSiteDomain() + '/user.php?action=login'
-        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
-                                                              params['penname']))
-
-        d = self._fetchUrl(loginUrl, params)
-
-        if "Member Account" not in d : #Member Account
-            logger.info("Failed to login to URL %s as %s" % (loginUrl,
-                                                              params['penname']))
-            raise exceptions.FailedToLogin(url,params['penname'])
-            return False
-        else:
-            return True
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        if self.is_adult or self.getConfig("is_adult"):
-            # Weirdly, different sites use different warning numbers.
-            # If the title search below fails, there's a good chance
-            # you need a different number.  print data at that point
-            # and see what the 'click here to continue' url says.
-            addurl = "&ageconsent=ok&warning=5"
-        else:
-            addurl=""
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url+'&index=1'+addurl
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        if self.needToLoginCheck(data):
-            # need to log in for this one.
-            self.performLogin(url)
-            data = self._fetchUrl(url)
-
-        # Since the warning text can change by warning level, let's
-        # look for the warning pass url.  ksarchive uses
-        # &warning= -- actually, so do other sites.  Must be an
-        # eFiction book.
-
-        # viewstory.php?sid=1882&warning=4
-        # viewstory.php?sid=1654&ageconsent=ok&warning=5
-        #print data
-        #m = re.search(r"'viewstory.php\?sid=1882(&warning=4)'",data)
-        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
-        if m != None:
-            if self.is_adult or self.getConfig("is_adult"):
-                # We tried the default and still got a warning, so
-                # let's pull the warning number from the 'continue'
-                # link and reload data.
-                addurl = m.group(1)
-                # correct stupid & error in url.
-                addurl = addurl.replace("&","&")
-                url = self.url+'&index=1'+addurl
-                logger.debug("URL 2nd try: "+url)
-
-                try:
-                    data = self._fetchUrl(url)
-                except urllib2.HTTPError, e:
-                    if e.code == 404:
-                        raise exceptions.StoryDoesNotExist(self.url)
-                    else:
-                        raise e
-            else:
-                raise exceptions.AdultCheckRequired(self.url)
-
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.AccessDenied(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = self.make_soup(data)
-        # print data
-
-        # Now go hunting for all the meta data and the chapter list.
-
-        pagetitle = soup.find('tr',{'valign':'top'})
-
-        ## Title
-        a = pagetitle.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',stripHTML(a))
-
-        # Find authorid and URL from... author url.
-        a = pagetitle.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
-            # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        cats = soup.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-        for cat in cats:
-            if '/' == cat.string[0]:
-                self.story.addToList('ships','Harry Potter'+cat.string.split('(')[0])
-            elif 'Harry' in cat.string:
-                self.story.addToList('ships',cat.string.split('(')[0])
-            else:
-                self.story.addToList('category',cat.string)
-            if '(' in cat.string:
-                self.story.addToList('category',cat.string.split('(')[1].split(')')[0])
-
-
-
-
-        chars = soup.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-        for char in chars:
-            self.story.addToList('characters',char.string)
-
-        genres = soup.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=2'))
-        for genre in genres:
-            self.story.addToList('genre',genre.string)
-
-        warnings = soup.findAll('a',href=re.compile(r'browse.php\?type=class&type_id=1'))
-        for warning in warnings:
-            self.story.addToList('warnings',warning.string)
-
-        textsoup = stripHTML(soup)
-
-        a = textsoup.split('Published: ')[1].split(' ')[0]
-        self.story.setMetadata('datePublished', makeDate(stripHTML(a), self.dateformat))
-        a = textsoup.split('Updated: ')[1].split(' ')[0]
-        self.story.setMetadata('dateUpdated', makeDate(stripHTML(a), self.dateformat))
-        a = textsoup.split('Rating: ')[1].split(' ')[0]
-        self.story.setMetadata('rating', a)
-        a = textsoup.split('Length: ')[1].split('(')[1].split(' ')[0]
-        self.story.setMetadata('numWords', a)
-        a = textsoup.split('Completed: ')[1].split(' ')[0]
-        if 'Yes' in a:
-            self.story.setMetadata('status', 'Completed')
-        else:
-            self.story.setMetadata('status', 'In-Progress')
-        #a = textsoup.split('Summary: ')[1].split('Add Story to Favorites')[0]
-        #self.setDescription(url,a)
-
-
-
-        a=soup.find(text=re.compile("Summary: "))
-        i=0
-        svalue = ""
-        while i == 0:
-            try:
-                b = unicode(a)
-                svalue += b.split('Summary: ')[1]
-            except:
-                svalue += unicode(a)
-            if a.nextSibling != None:
-                a = a.nextSibling
-            else:
-                a = a.parent.nextSibling
-            if 'Disclaimer: ' in stripHTML(a):
-                i=1
-        self.setDescription(url,svalue)
-
-        try:
-            # Find Series name from series URL.
-            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
-            series_name = a.string
-            series_url = 'http://'+self.host+'/'+a['href']
-
-            # use BeautifulSoup HTML parser to make everything easier to find.
-            seriessoup = self.make_soup(self._fetchUrl(series_url))
-            storyas = seriessoup.findAll('a', href=re.compile(r'^viewstory.php\?sid=\d+$'))
-            i=1
-            for a in storyas:
-                if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                    self.setSeries(series_name, i)
-                    self.story.setMetadata('seriesUrl',series_url)
-                    break
-                i+=1
-
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = self.make_soup(self._fetchUrl(url))
-
-        div = soup.find('div', {'id' : 'story'})
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-
-        return self.utf8FromSoup(url,div)

fanficfare/adapters/adapter_storiesonlinenet.py

@@ -97,7 +97,7 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
         params['page'] = 'http://'+self.getSiteDomain()+'/'
         params['submit'] = 'Login'
 
-        loginUrl = 'https://' + self.getSiteDomain() + '/login.php'
+        loginUrl = 'https://' + self.getSiteDomain() + '/sol-secure/login.php'
         logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['theusername']))
 
@@ -147,6 +147,8 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
             raise exceptions.AccessDenied(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
         elif "Error! The story you're trying to access is being filtered by your choice of contents filtering." in data:
             raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Error! The story you're trying to access is being filtered by your choice of contents filtering.")
+        elif "Error! Daily Limit Reached" in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Error! Daily Limit Reached")
 
         # use BeautifulSoup HTML parser to make everything easier to find.
         soup = self.make_soup(data)
@@ -183,7 +185,8 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
         page=0
         i=0
         while i == 0:
-            asoup = self.make_soup(self._fetchUrl(self.story.getList('authorUrl')[0]+"/"+unicode(page)))
+            data = self._fetchUrl(self.story.getList('authorUrl')[0]+"/"+unicode(page))
+            asoup = self.make_soup(data)
 
             a = asoup.findAll('td', {'class' : 'lc2'})
             for lc2 in a:
@@ -208,6 +211,7 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
 
         try:
             a = lc4.find('a', href=re.compile(r"/series/\d+/.*"))
+            # logger.debug("Looking for series - a='{0}'".format(a))
             if a:
                 # if there's a number after the series name, series_contents is a two element list:
                 # [<a href="...">Title</a>, u' (2)']
@@ -216,59 +220,62 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
                 seriesUrl = 'http://'+self.host+a['href']
                 self.story.setMetadata('seriesUrl',seriesUrl)
                 series_name = stripHTML(a)
-                logger.debug("Series name= %s" % series_name)
+                # logger.debug("Series name= %s" % series_name)
                 series_soup = self.make_soup(self._fetchUrl(seriesUrl))
                 if series_soup:
-                    logger.debug("Retrieving Series - looking for name")
-                    series_name = series_soup.find('span', {'id' : 'ptitle'}).text.partition(' — ')[0]
-                    logger.debug("Series name: '{0}'".format(series_name))
+                    # logger.debug("Retrieving Series - looking for name")
+                    series_name = stripHTML(series_soup.find('span', {'id' : 'ptitle'}))
+                    series_name = re.sub(r' . a series by.*$','',series_name)
+                    # logger.debug("Series name: '%s'" % series_name)
                 self.setSeries(series_name, i)
                 desc = lc4.contents[2]
                 # Check if series is in a universe
                 universe_url = self.story.getList('authorUrl')[0]  + "&type=uni"
                 universes_soup = self.make_soup(self._fetchUrl(universe_url) )
-                logger.debug("Universe url='{0}'".format(universe_url))
+                # logger.debug("Universe url='{0}'".format(universe_url))
                 if universes_soup:
                     universes = universes_soup.findAll('div', {'class' : 'ser-box'})
-                    logger.debug("Number of Universes: %d" % len(universes))
+                    # logger.debug("Number of Universes: %d" % len(universes))
                     for universe in universes:
-                        logger.debug("universe.find('a')={0}".format(universe.find('a')))
+                        # logger.debug("universe.find('a')={0}".format(universe.find('a')))
                         # The universe id is in an "a" tag that has an id but nothing else. It is the first tag.
                         # The id is prefixed with the letter "u".
                         universe_id = universe.find('a')['id'][1:]
-                        logger.debug("universe_id='%s'" % universe_id)
-                        universe_name = universe.find('div', {'class' : 'ser-name'}).text.partition(' ')[2]
-                        logger.debug("universe_name='%s'" % universe_name)
+                        # logger.debug("universe_id='%s'" % universe_id)
+                        universe_name = stripHTML(universe.find('div', {'class' : 'ser-name'})).partition(' ')[2]
+                        # logger.debug("universe_name='%s'" % universe_name)
                         # If there is link to the story, we have the right universe
                         story_a = universe.find('a', href=re.compile('/s/'+self.story.getMetadata('storyId')))
                         if story_a:
-                            logger.debug("Story is in a series that is in a universe! The universe is '%s'" % universe_name)
+                            # logger.debug("Story is in a series that is in a universe! The universe is '%s'" % universe_name)
                             self.story.setMetadata("universe", universe_name)
                             self.story.setMetadata('universeUrl','http://'+self.host+ '/library/universe.php?id=' + universe_id)
                             break
                 else:
                     logger.debug("No universe page")
         except:
+            raise
             pass
         try:
             a = lc4.find('a', href=re.compile(r"/universe/\d+/.*"))
-            logger.debug("Looking for universe - a='{0}'".format(a))
+            # logger.debug("Looking for universe - a='{0}'".format(a))
             if a:
                 self.story.setMetadata("universe",stripHTML(a))
                 desc = lc4.contents[2]
                 # Assumed only one universe, but it does have a URL--use universeHTML
                 universe_name = stripHTML(a)
                 universeUrl = 'http://'+self.host+a['href']
-                logger.debug("Retrieving Universe - about to get page - universeUrl='{0}".format(universeUrl))
+                # logger.debug("Retrieving Universe - about to get page - universeUrl='{0}".format(universeUrl))
                 universe_soup = self.make_soup(self._fetchUrl(universeUrl))
                 logger.debug("Retrieving Universe - have page")
                 if universe_soup:
                     logger.debug("Retrieving Universe - looking for name")
-                    universe_name = universe_soup.find('h1', {'id' : 'ptitle'}).text.partition('—')[0]
-                    logger.debug("Universes name: '{0}'".format(universe_name))
+                    universe_name = stripHTML(universe_soup.find('h1', {'id' : 'ptitle'}))
+                    universe_name = re.sub(r' . A Universe from the Mind.*$','',universe_name)
+                    # logger.debug("Universes name: '{0}'".format(universe_name))
 
                 self.story.setMetadata('universeUrl',universeUrl)
-                logger.debug("Setting universe name: '{0}'".format(universe_name))
+                # logger.debug("Setting universe name: '{0}'".format(universe_name))
                 self.story.setMetadata('universe',universe_name)
                 if self.getConfig("universe_as_series"):
                     self.setSeries(universe_name, 0)
@@ -276,6 +283,7 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
             else:
                 logger.debug("Do not have a universe")
         except:
+            raise
             pass
 
         self.setDescription('http://'+self.host+'/s/'+self.story.getMetadata('storyId'),desc)
@@ -312,12 +320,12 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
         #     http://storiesonline.net/s/11999
         #     http://storiesonline.net/s/10823
         if get_cover:
-            logger.debug("Looking for the cover image...")
+            # logger.debug("Looking for the cover image...")
             cover_url = ""
             img = soup.find('img')
             if img:
                 cover_url=img['src']
-            logger.debug("cover_url: %s"%cover_url)
+            # logger.debug("cover_url: %s"%cover_url)
             if cover_url:
                 self.setCoverImage(url,cover_url)
 
@@ -355,7 +363,7 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
 
             urls=pager.findAll('a')
             urls=urls[:len(urls)-1]
-            logger.debug("pager urls:%s"%urls)
+            # logger.debug("pager urls:%s"%urls)
             pager.extract()
             chaptertag.contents = chaptertag.contents[2:]
 
@@ -364,7 +372,7 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter):
 
                 pagetag = soup.find('div', {'id' : 'story'})
                 if not pagetag:
-                    logger.debug("div id=story not found, try article")
+                    # logger.debug("div id=story not found, try article")
                     pagetag = soup.find('article', {'id' : 'story'})
 
                 self.cleanPage(pagetag)

fanficfare/adapters/adapter_tokrafandomnetcom.py

@@ -1,238 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright 2013 Fanficdownloader team, 2015 FanFicFare team
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Software: eFiction
-import time
-import logging
-logger = logging.getLogger(__name__)
-import re
-import urllib2
-
-
-from ..htmlcleanup import stripHTML
-from .. import exceptions as exceptions
-
-from base_adapter import BaseSiteAdapter,  makeDate
-
-def getClass():
-    return TokraFandomnetComAdapter
-
-# Class name has to be unique.  Our convention is camel case the
-# sitename with Adapter at the end.  www is skipped.
-class TokraFandomnetComAdapter(BaseSiteAdapter):
-
-    def __init__(self, config, url):
-        BaseSiteAdapter.__init__(self, config, url)
-
-        self.decode = ["Windows-1252",
-                       "utf8"] # 1252 is a superset of iso-8859-1.
-                               # Most sites that claim to be
-                               # iso-8859-1 (and some that claim to be
-                               # utf8) are really windows-1252.
-        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
-        self.password = ""
-        self.is_adult=False
-
-        # get storyId from url--url validation guarantees query is only sid=1234
-        self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
-
-
-        # normalized story URL.
-        self._setURL('http://' + self.getSiteDomain() + '/viewstory.php?sid='+self.story.getMetadata('storyId'))
-
-        # Each adapter needs to have a unique site abbreviation.
-        self.story.setMetadata('siteabbrev','tokra')
-
-        # The date format will vary from site to site.
-        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
-        self.dateformat = "%m/%d/%Y"
-
-    @staticmethod # must be @staticmethod, don't remove it.
-    def getSiteDomain():
-        # The site domain.  Does have www here, if it uses it.  But it
-        # doesn't matter too much anymore.
-        return 'tokra.fandomnet.com'
-
-    @classmethod
-    def getSiteExampleURLs(cls):
-        return "http://"+cls.getSiteDomain()+"/viewstory.php?sid=1234"
-
-    def getSiteURLPattern(self):
-        return re.escape("http://"+self.getSiteDomain()+"/viewstory.php?sid=")+r"\d+$"
-
-    ## Getting the chapter list and the meta data, plus 'is adult' checking.
-    def extractChapterUrlsAndMetadata(self):
-
-        if self.is_adult or self.getConfig("is_adult"):
-            # Weirdly, different sites use different warning numbers.
-            # If the title search below fails, there's a good chance
-            # you need a different number.  print data at that point
-            # and see what the 'click here to continue' url says.
-            addurl = "&ageconsent=ok&warning=3"
-        else:
-            addurl=""
-
-        # index=1 makes sure we see the story chapter index.  Some
-        # sites skip that for one-chapter stories.
-        url = self.url+'&index=1'+addurl
-        logger.debug("URL: "+url)
-
-        try:
-            data = self._fetchUrl(url)
-        except urllib2.HTTPError, e:
-            if e.code == 404:
-                raise exceptions.StoryDoesNotExist(self.url)
-            else:
-                raise e
-
-        m = re.search(r"'viewstory.php\?sid=\d+((?:&ageconsent=ok)?&warning=\d+)'",data)
-        if m != None:
-            if self.is_adult or self.getConfig("is_adult"):
-                # We tried the default and still got a warning, so
-                # let's pull the warning number from the 'continue'
-                # link and reload data.
-                addurl = m.group(1)
-                # correct stupid & error in url.
-                addurl = addurl.replace("&","&")
-                url = self.url+'&index=1'+addurl
-                logger.debug("URL 2nd try: "+url)
-
-                try:
-                    data = self._fetchUrl(url)
-                except urllib2.HTTPError, e:
-                    if e.code == 404:
-                        raise exceptions.StoryDoesNotExist(self.url)
-                    else:
-                        raise e
-            else:
-                raise exceptions.AdultCheckRequired(self.url)
-
-        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
-            raise exceptions.AccessDenied(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
-
-        # use BeautifulSoup HTML parser to make everything easier to find.
-        soup = self.make_soup(data)
-        #print data
-
-        # Now go hunting for all the meta data and the chapter list.
-
-        ## Title
-        a = soup.find('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"$"))
-        self.story.setMetadata('title',stripHTML(a))
-
-        # Find authorid and URL from... author url.
-        a = soup.find('a', href=re.compile(r"viewuser.php\?uid=\d+"))
-        self.story.setMetadata('authorId',a['href'].split('=')[1])
-        self.story.setMetadata('authorUrl','http://'+self.host+'/'+a['href'])
-        self.story.setMetadata('author',a.string)
-
-        # Rating
-        rate = stripHTML(soup.find('div',{'id':'pagetitle'}))
-        rate = rate[rate.rindex('[')+1:rate.rindex(']')]
-        self.story.setMetadata('rating', rate)
-
-        # Find the chapters:
-        for chapter in soup.findAll('a', href=re.compile(r'viewstory.php\?sid='+self.story.getMetadata('storyId')+"&chapter=\d+$")):
-            # just in case there's tags, like <i> in chapter titles.
-            self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+'/'+chapter['href']+addurl))
-
-        self.story.setMetadata('numChapters',len(self.chapterUrls))
-
-        # eFiction sites don't help us out a lot with their meta data
-        # formating, so it's a little ugly.
-
-        metadiv = soup.find('div',{'class':'content'})
-        smalldiv = metadiv.find('div',{'class':'small'})
-
-        # tokra categories -> genre
-        # categories will be filled from ini.
-        genres = smalldiv.parent.findAll('a',href=re.compile(r'browse.php\?type=categories'))
-        for genre in genres:
-            self.story.addToList('genre',genre.string)
-
-        chars = smalldiv.parent.findAll('a',href=re.compile(r'browse.php\?type=characters'))
-        for char in chars:
-            self.story.addToList('characters',char.string)
-
-        metatext = stripHTML(smalldiv)
-
-        if 'Completed: Yes' in metatext:
-            self.story.setMetadata('status', 'Completed')
-        else:
-            self.story.setMetadata('status', 'In-Progress')
-
-        wordstart=metatext.rindex('Word count:')+12
-        words = metatext[wordstart:metatext.index(' ',wordstart)]
-        self.story.setMetadata('numWords', words)
-
-        datesdiv = soup.find('div',{'class':'bottom'})
-        dates = stripHTML(datesdiv).split()
-        # Published: 04/26/2011 Updated: 03/06/2013
-        self.story.setMetadata('datePublished', makeDate(dates[1], self.dateformat))
-        self.story.setMetadata('dateUpdated', makeDate(dates[3], self.dateformat))
-
-        try:
-            # Find Series name from series URL.
-            a = soup.find('a', href=re.compile(r"viewseries.php\?seriesid=\d+"))
-            series_name = a.string
-            series_url = 'http://'+self.host+'/'+a['href']
-
-            # use BeautifulSoup HTML parser to make everything easier to find.
-            seriessoup = self.make_soup(self._fetchUrl(series_url))
-            # can't use ^viewstory...$ in case of higher rated stories with javascript href.
-            storyas = seriessoup.findAll('a', href=re.compile(r'viewstory.php\?sid=\d+'))
-            i=1
-            for a in storyas:
-                # skip 'report this' and 'TOC' links
-                if 'contact.php' not in a['href'] and 'index' not in a['href']:
-                    if a['href'] == ('viewstory.php?sid='+self.story.getMetadata('storyId')):
-                        self.setSeries(series_name, i)
-                        self.story.setMetadata('seriesUrl',series_url)
-                        break
-                    i+=1
-
-        except:
-            # I find it hard to care if the series parsing fails
-            pass
-
-        # remove 'small' leaving only summary.
-        smalldiv.extract()
-        self.setDescription(url,metadiv)
-
-    # grab the text for an individual chapter.
-    def getChapterText(self, url):
-
-        logger.debug('Getting chapter text from: %s' % url)
-
-        soup = self.make_soup(self._fetchUrl(url))
-
-        div = soup.find('div', {'class' : 'content'})
-
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-
-        # remove some decorations while keeping notes.
-        remove = div.find('div', {'id' : 'pagetitle'})
-        remove.extract()
-
-        for remove in div.findAll('div', {'class' : 'right'}):
-            remove.extract()
-
-        for remove in div.findAll('div', {'class' : 'left'}):
-            remove.extract()
-
-        return self.utf8FromSoup(url,div)

fanficfare/adapters/base_adapter.py

@@ -16,7 +16,8 @@
 #
 
 import re
-import datetime
+from datetime import datetime, timedelta
+
 import time
 import logging
 import urllib
@@ -99,7 +100,7 @@ class BaseSiteAdapter(Configurable):
         self.metadataDone = False
         self.story = Story(configuration)
         self.story.setMetadata('site',self.getConfigSection())
-        self.story.setMetadata('dateCreated',datetime.datetime.now())
+        self.story.setMetadata('dateCreated',datetime.now())
         self.chapterUrls = [] # tuples of (chapter title,chapter url)
         self.chapterFirst = None
         self.chapterLast = None
@@ -571,11 +572,14 @@ class BaseSiteAdapter(Configurable):
         #print("include_images:"+self.getConfig('include_images'))
         if self.getConfig('include_images'):
             acceptable_attributes.extend(('src','alt','longdesc'))
-            for img in soup.findAll('img'):
-                # some pre-existing epubs have img tags that had src stripped off.
-                if img.has_attr('src'):
-                    (img['src'],img['longdesc'])=self.story.addImgUrl(url,img['src'],fetch,
-                                                                      coverexclusion=self.getConfig('cover_exclusion_regexp'))
+            try:
+                for img in soup.find_all('img'):
+                    # some pre-existing epubs have img tags that had src stripped off.
+                    if img.has_attr('src'):
+                        (img['src'],img['longdesc'])=self.story.addImgUrl(url,img['src'],fetch,
+                                                                          coverexclusion=self.getConfig('cover_exclusion_regexp'))
+            except AttributeError as ae:
+                logger.info("Parsing for img tags failed--probably poor input HTML.  Skipping images.")
 
         for attr in self.get_attr_keys(soup):
             if attr not in acceptable_attributes:
@@ -685,8 +689,23 @@ def makeDate(string,dateform):
             if name in string:
                 string = string.replace(name,num)
                 break
+
+    # Many locales don't define %p for AM/PM.  So if %p, remove from
+    # dateform, look for 'pm' in string, remove am/pm from string and
+    # add 12 hours if pm found.
+    add_hours = False
+    if u"%p" in dateform:
+        dateform = dateform.replace(u"%p",u"")
+        if 'pm' in string or 'PM' in string:
+            add_hours = True
+        string = string.replace(u"AM",u"").replace(u"PM",u"").replace(u"am",u"").replace(u"pm",u"")
+
+    date = datetime.strptime(string.encode('utf-8'),dateform.encode('utf-8'))
+    
+    if add_hours:
+        date += timedelta(hours=12)
             
-    return datetime.datetime.strptime(string.encode('utf-8'),dateform.encode('utf-8'))
+    return date
 
 # .? for AO3's ']' in param names.
 safe_url_re = re.compile(r'(?P<attr>(password|name|login).?=)[^&]*(?P<amp>&|$)',flags=re.MULTILINE)

fanficfare/adapters/base_xenforoforum_adapter.py

@@ -228,11 +228,15 @@ class BaseXenForoForumAdapter(BaseSiteAdapter):
 
                 if ( url.startswith(self.getURLPrefix()) or
                      url.startswith('http://'+self.getSiteDomain()) or
-                     url.startswith('https://'+self.getSiteDomain()) ) and ('/posts/' in url or '/threads/' in url):
+                     url.startswith('https://'+self.getSiteDomain()) ) and \
+                     ( '/posts/' in url or '/threads/' in url or 'showpost.php' in url):
 
                     # brute force way to deal with SB's http->https change when hardcoded http urls.
                     url = url.replace('http://'+self.getSiteDomain(),self.getURLPrefix())
 
+                    # http://forums.spacebattles.com/showpost.php?p=4755532&postcount=9
+                    url = re.sub(r'showpost\.php\?p=([0-9]+)(&postcount=[0-9]+)?',r'/posts/\1/',url)
+
                     url = re.sub(r'(^[\'"]+|[\'"]+$)','',url) # strip leading or trailing '" from incorrect quoting.
                     url = re.sub(r'like$','',url) # strip 'like' if incorrect 'like' link instead of proper post URL.
 
@@ -272,7 +276,7 @@ class BaseXenForoForumAdapter(BaseSiteAdapter):
             datestr = re.sub(r' (\d[^\d])',r' 0\1',datestr) # add leading 0 for single digit day & hours.
             return makeDate(datestr, self.dateformat)
         except:
-            logger.debug('No date found in %s'%parenttag)
+            logger.debug('No date found in %s'%parenttag,exc_info=True)
             return None
 
     # grab the text for an individual chapter.

fanficfare/configurable.py

@@ -166,6 +166,10 @@ def get_valid_set_options():
                'pairingcat_to_characters_ships':(['tthfanfic.org'],None,boollist),
                'romancecat_to_characters_ships':(['tthfanfic.org'],None,boollist),
 
+               'use_meta_keywords':(['literotica.com'],None,boollist),
+               'clean_chapter_titles':(['literotica.com'],None,boollist),
+               'description_in_chapter':(['literotica.com'],None,boollist),
+
                # eFiction Base adapters allow bulk_load
                # kept forgetting to add them, so now it's automatic.
                'bulk_load':(adapters.get_bulk_load_sites(),
@@ -184,9 +188,6 @@ def get_valid_set_options():
                'minimum_threadmarks':(base_xenforo_list,None,None),
                'first_post_title':(base_xenforo_list,None,None),
                'always_include_first_post':(base_xenforo_list,None,boollist),
-               '':(base_xenforo_list,None,boollist),
-               '':(base_xenforo_list,None,boollist),
-               '':(base_xenforo_list,None,boollist),
                }
 
     return dict(valdict)
@@ -326,6 +327,9 @@ def get_valid_keywords():
                  'centeredcat_to_characters',
                  'pairingcat_to_characters_ships',
                  'romancecat_to_characters_ships',
+                 'use_meta_keywords',
+                 'clean_chapter_titles',
+                 'description_in_chapter',
                  'titlepage_end',
                  'titlepage_entries',
                  'titlepage_entry',

fanficfare/defaults.ini

@@ -473,8 +473,8 @@ add_to_include_subject_tags:,tagsfromtitle.SPLIT,forumtags
 ## base_xenforoforum reads Published and Updated datetimes from
 ## Threadmarks if used, or from the posted & updated times of the
 ## 'first' post if no threadmarks.
-datePublished_format:%%Y-%%m-%%d %%H:%%M:%%S
-dateUpdated_format:%%Y-%%m-%%d %%H:%%M:%%S
+datePublished_format:%%Y-%%m-%%d %%H:%%M
+dateUpdated_format:%%Y-%%m-%%d %%H:%%M
 
 ## Only take the first X characters of the 'first' post to use as
 ## the description.
@@ -853,18 +853,6 @@ extracategories:The Sentinel
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
-[bdsm-geschichten.net]
-## Some sites do not require a login, but do require the user to
-## confirm they are adult for adult content.  In commandline version,
-## this should go in your personal.ini, not defaults.ini.
-#is_adult:true
-
-## This site offers no index page so we can either guess the chapter URLs
-## by dec/incrementing numbers ('guess') or walk all the chapters in the metadata
-## parsing state ('parse'). Since guessing can lead to errors for non-standard
-## story URLs, the default is to parse
-#find_chapters:guess 
-
 [bloodshedverse.com]
 ## website encoding(s) In theory, each website reports the character
 ## encoding they use for each page.  In practice, some sites report it
@@ -875,6 +863,12 @@ extracategories:The Sentinel
 ## it has +90% confidence.  'auto' is not reliable.
 website_encodings:Windows-1252,ISO-8859-1,auto
 
+## dateUpdate doesn't usually have time, but it does on
+## bloodshedverse.com.  See
+## http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+## Note that ini format requires % to be escaped as %%.
+dateUpdated_format:%%Y-%%m-%%d %%H:%%M
+
 ## Extra metadata that this adapter knows about.  See [dramione.org]
 ## for examples of how to use them.
 extra_valid_entries:warnings,reviews
@@ -904,15 +898,6 @@ strip_text_links:true
 ## Site dedicated to these categories/characters/ships
 extracategories:Blood Ties
 
-[buffynfaith.net]
-## Site dedicated to these categories/characters/ships
-extracategories:Buffy: The Vampire Slayer
-
-## Some sites also require the user to confirm they are adult for
-## adult content.  In commandline version, this should go in your
-## personal.ini, not defaults.ini.
-#is_adult:true
-
 [fanfic.castletv.net]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -1250,27 +1235,6 @@ cover_exclusion_regexp:/css/bir.png
 [forums.sufficientvelocity.com]
 ## see [base_xenforoforum]
 
-[grangerenchanted.com]
-## Some sites require login (or login for some rated stories) The
-## program can prompt you, or you can save it in config.  In
-## commandline version, this should go in your personal.ini, not
-## defaults.ini.
-#username:YourName
-#password:yourpassword
-
-## Some sites also require the user to confirm they are adult for
-## adult content.  In commandline version, this should go in your
-## personal.ini, not defaults.ini.
-#is_adult:true
-
-## Site dedicated to these categories/characters/ships
-extracategories:Harry Potter
-extracharacters:Hermione Granger
-
-## Extra metadata that this adapter knows about.  See [dramione.org]
-## for examples of how to use them.
-extra_valid_entries:read,reviews
-
 [hlfiction.net]
 ## Site dedicated to these categories/characters/ships
 extracategories:Highlander
@@ -1327,6 +1291,17 @@ eroticatags_label:Erotica Tags
 averrating_label:Average Rating
 extra_titlepage_entries:eroticatags,averrating
 
+## Extract more erotica_tags from the meta tag of each chapter
+use_meta_keywords: true
+
+## For multiple chapter stories, attempt to clean up the chapter title. This will
+## remove the story title and change "Ch. 01" to "Chapter 1", "Pt. 01" to "Part 1"
+## or just use the text. If this can't be done, the full title is used.
+clean_chapter_titles: false
+
+## Add the chapter description at the start of each chapter.
+description_in_chapter: false
+
 [lotrfanfiction.com]
 extra_valid_entries: readings
 readings_label: Readings
@@ -1507,22 +1482,6 @@ extracategories:Supernatural
 extracharacters:Sam,Dean
 extraships:Sam/Dean
 
-[scarhead.net]
-## Some sites require login (or login for some rated stories) The
-## program can prompt you, or you can save it in config.  In
-## commandline version, this should go in your personal.ini, not
-## defaults.ini.
-#username:YourName
-#password:yourpassword
-
-## Some sites also require the user to confirm they are adult for
-## adult content.  In commandline version, this should go in your
-## personal.ini, not defaults.ini.
-#is_adult:true
-
-## Site dedicated to these categories/characters/ships
-extracategories:Harry Potter
-
 [sheppardweir.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -1664,15 +1623,6 @@ readings_label: Readings
 ## personal.ini, not defaults.ini.
 #is_adult:true
 
-[tokra.fandomnet.com]
-## Some sites do not require a login, but do require the user to
-## confirm they are adult for adult content.  In commandline version,
-## this should go in your personal.ini, not defaults.ini.
-#is_adult:true
-
-## Site dedicated to these categories/characters/ships
-extracategories:Stargate: SG-1
-
 [tolkienfanfiction.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:Lord of the Rings
@@ -1825,6 +1775,10 @@ check_next_chapter:false
 #password:yourpassword
 
 [www.ficbook.net]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
 
 [www.fictionalley.org]
 ## Some sites do not require a login, but do require the user to
@@ -2030,11 +1984,6 @@ cover_exclusion_regexp:/stories/999/images/.*?_trophy.png
 ## Site dedicated to these categories/characters/ships
 extracategories:NCIS
 
-[www.nickngreg.nl]
-## Site dedicated to these categories/characters/ships
-extracategories:CSI
-extraships:Nick Stokes/Greg Sanders
-
 [www.phoenixsong.net]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -2328,20 +2277,6 @@ extracategories:Andromeda
 ## Site dedicated to these categories/characters/ships
 extracategories:Artemis Fowl
 
-[www.therabidreader.com]
-## Some sites do not require a login, but do require the user to
-## confirm they are adult for adult content.  In commandline version,
-## this should go in your personal.ini, not defaults.ini.
-#is_adult:true
-
-## Some sites require login (or login for some rated stories) The
-## program can prompt you, or you can save it in config.  In
-## commandline version, this should go in your personal.ini, not
-## defaults.ini.
-#username:YourName
-#password:yourpassword
-
-
 [www.naiceanilme.net]
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
@@ -2355,8 +2290,6 @@ extracategories:Artemis Fowl
 #username:YourName
 #password:yourpassword
 
-
-
 [overrides]
 ## It may sometimes be useful to override all of the specific format,
 ## site and site:format sections in your private configuration.  For

fanficfare/epubutils.py

@@ -84,8 +84,7 @@ def get_update_data(inputio,
                 oldcover = (oldcoverhtmlhref,oldcoverhtmltype,oldcoverhtmldata,oldcoverimghref,oldcoverimgtype,oldcoverimgdata)
             except Exception as e:
                 logger.warn("Cover Image %s not found"%src)
-                logger.warn("Exception: %s"%(unicode(e)))
-                traceback.print_exc()
+                logger.warn("Exception: %s"%(unicode(e)),exc_info=True)
 
     filecount = 0
     soups = [] # list of xhmtl blocks
@@ -127,8 +126,7 @@ def get_update_data(inputio,
                                 # originally.
                                 if newsrc != u'OEBPS/failedtoload':
                                     logger.warn("Image %s not found!\n(originally:%s)"%(newsrc,longdesc))
-                                    logger.warn("Exception: %s"%(unicode(e)))
-                                    traceback.print_exc()
+                                    logger.warn("Exception: %s"%(unicode(e)),exc_info=True)
                         bodysoup = soup.find('body')
                         # ffdl epubs have chapter title h3
                         h3 = bodysoup.find('h3')

fanficfare/html2text.py

@@ -1,453 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-"""html2text: Turn HTML into equivalent Markdown-structured text."""
-__version__ = "2.37"
-__author__ = "Aaron Swartz (me@aaronsw.com)"
-__copyright__ = "(C) 2004-2008 Aaron Swartz. GNU GPL 3."
-__contributors__ = ["Martin 'Joey' Schulze", "Ricardo Reyes", "Kevin Jay North"]
-
-# TODO:
-#   Support decoded entities with unifiable.
-
-if not hasattr(__builtins__, 'True'): True, False = 1, 0
-import re, sys, urllib, htmlentitydefs, codecs, StringIO, types
-import sgmllib
-import urlparse
-sgmllib.charref = re.compile('&#([xX]?[0-9a-fA-F]+)[^0-9a-fA-F]')
-
-try: from textwrap import wrap
-except: pass
-
-# Use Unicode characters instead of their ascii psuedo-replacements
-UNICODE_SNOB = 0
-
-# Put the links after each paragraph instead of at the end.
-LINKS_EACH_PARAGRAPH = 0
-
-# Wrap long lines at position. 0 for no wrapping. (Requires Python 2.3.)
-BODY_WIDTH = 78
-
-# Don't show internal links (href="#local-anchor") -- corresponding link targets
-# won't be visible in the plain text file anyway.
-SKIP_INTERNAL_LINKS = False
-
-### Entity Nonsense ###
-
-def name2cp(k):
-    if k == 'apos': return ord("'")
-    if hasattr(htmlentitydefs, "name2codepoint"): # requires Python 2.3
-        return htmlentitydefs.name2codepoint[k]
-    else:
-        k = htmlentitydefs.entitydefs[k]
-        if k.startswith("&#") and k.endswith(";"): return int(k[2:-1]) # not in latin-1
-        return ord(codecs.latin_1_decode(k)[0])
-
-unifiable = {'rsquo':"'", 'lsquo':"'", 'rdquo':'"', 'ldquo':'"', 
-'copy':'(C)', 'mdash':'--', 'nbsp':' ', 'rarr':'->', 'larr':'<-', 'middot':'*',
-'ndash':'-', 'oelig':'oe', 'aelig':'ae',
-'agrave':'a', 'aacute':'a', 'acirc':'a', 'atilde':'a', 'auml':'a', 'aring':'a', 
-'egrave':'e', 'eacute':'e', 'ecirc':'e', 'euml':'e', 
-'igrave':'i', 'iacute':'i', 'icirc':'i', 'iuml':'i',
-'ograve':'o', 'oacute':'o', 'ocirc':'o', 'otilde':'o', 'ouml':'o', 
-'ugrave':'u', 'uacute':'u', 'ucirc':'u', 'uuml':'u'}
-
-unifiable_n = {}
-
-for k in unifiable.keys():
-    unifiable_n[name2cp(k)] = unifiable[k]
-
-def charref(name):
-    if name[0] in ['x','X']:
-        c = int(name[1:], 16)
-    else:
-        c = int(name)
-    
-    if not UNICODE_SNOB and c in unifiable_n.keys():
-        return unifiable_n[c]
-    else:
-        return unichr(c)
-
-def entityref(c):
-    if not UNICODE_SNOB and c in unifiable.keys():
-        return unifiable[c]
-    else:
-        try: name2cp(c)
-        except KeyError: return "&" + c
-        else: return unichr(name2cp(c))
-
-def replaceEntities(s):
-    s = s.group(1)
-    if s[0] == "#": 
-        return charref(s[1:])
-    else: return entityref(s)
-
-r_unescape = re.compile(r"&(#?[xX]?(?:[0-9a-fA-F]+|\w{1,8}));")
-def unescape(s):
-    return r_unescape.sub(replaceEntities, s)
-    
-def fixattrs(attrs):
-    # Fix bug in sgmllib.py
-    if not attrs: return attrs
-    newattrs = []
-    for attr in attrs:
-        newattrs.append((attr[0], unescape(attr[1])))
-    return newattrs
-
-### End Entity Nonsense ###
-
-def onlywhite(line):
-    """Return true if the line does only consist of whitespace characters."""
-    for c in line:
-        if c is not ' ' and c is not '  ':
-            return c is ' '
-    return line
-
-def optwrap(text,wrap_width=BODY_WIDTH):
-    """Wrap all paragraphs in the provided text."""
-    
-    if not wrap_width:
-        return text
-    
-    assert wrap, "Requires Python 2.3."
-    result = ''
-    newlines = 0
-    for para in text.split("\n"):
-        if len(para) > 0:
-            if para[0] is not ' ' and para[0] is not '-' and para[0] is not '*':
-                for line in wrap(para, wrap_width):
-                    result += line + "\n"
-                result += "\n"
-                newlines = 2
-            else:
-                if not onlywhite(para):
-                    result += para + "\n"
-                    newlines = 1
-        else:
-            if newlines < 2:
-                result += "\n"
-                newlines += 1
-    return result
-
-def hn(tag):
-    if tag[0] == 'h' and len(tag) == 2:
-        try:
-            n = int(tag[1])
-            if n in range(1, 10): return n
-        except ValueError: return 0
-
-class _html2text(sgmllib.SGMLParser):
-    def __init__(self, out=None, baseurl=''):
-        sgmllib.SGMLParser.__init__(self)
-        
-        if out is None: self.out = self.outtextf
-        else: self.out = out
-        self.outtext = u''
-        self.quiet = 0
-        self.p_p = 0
-        self.outcount = 0
-        self.start = 1
-        self.space = 0
-        self.a = []
-        self.astack = []
-        self.acount = 0
-        self.list = []
-        self.blockquote = 0
-        self.pre = 0
-        self.startpre = 0
-        self.lastWasNL = 0
-        self.abbr_title = None # current abbreviation definition
-        self.abbr_data = None # last inner HTML (for abbr being defined)
-        self.abbr_list = {} # stack of abbreviations to write later
-        self.baseurl = baseurl
-    
-    def outtextf(self, s): 
-        self.outtext += s
-    
-    def close(self):
-        sgmllib.SGMLParser.close(self)
-        
-        self.pbr()
-        self.o('', 0, 'end')
-        
-        return self.outtext
-        
-    def handle_charref(self, c):
-        self.o(charref(c))
-
-    def handle_entityref(self, c):
-        self.o(entityref(c))
-            
-    def unknown_starttag(self, tag, attrs):
-        self.handle_tag(tag, attrs, 1)
-    
-    def unknown_endtag(self, tag):
-        self.handle_tag(tag, None, 0)
-        
-    def previousIndex(self, attrs):
-        """ returns the index of certain set of attributes (of a link) in the
-            self.a list
- 
-            If the set of attributes is not found, returns None
-        """
-        if not attrs.has_attr('href'): return None
-        
-        i = -1
-        for a in self.a:
-            i += 1
-            match = 0
-            
-            if a.has_attr('href') and a['href'] == attrs['href']:
-                if a.has_attr('title') or attrs.has_attr('title'):
-                        if (a.has_attr('title') and attrs.has_attr('title') and
-                            a['title'] == attrs['title']):
-                            match = True
-                else:
-                    match = True
-
-            if match: return i
-
-    def handle_tag(self, tag, attrs, start):
-        attrs = fixattrs(attrs)
-    
-        if hn(tag):
-            self.p()
-            if start: self.o(hn(tag)*"#" + ' ')
-
-        if tag in ['p', 'div']: self.p()
-        
-        if tag == "br" and start: self.o("  \n")
-
-        if tag == "hr" and start:
-            self.p()
-            self.o("* * *")
-            self.p()
-
-        if tag in ["head", "style", 'script']: 
-            if start: self.quiet += 1
-            else: self.quiet -= 1
-
-        if tag in ["body"]:
-            self.quiet = 0 # sites like 9rules.com never close <head>
-        
-        if tag == "blockquote":
-            if start: 
-                self.p(); self.o('> ', 0, 1); self.start = 1
-                self.blockquote += 1
-            else:
-                self.blockquote -= 1
-                self.p()
-        
-        if tag in ['em', 'i', 'u']: self.o("_")
-        if tag in ['strong', 'b']: self.o("**")
-        if tag == "code" and not self.pre: self.o('`') #TODO: `` `this` ``
-        if tag == "abbr":
-            if start:
-                attrsD = {}
-                for (x, y) in attrs: attrsD[x] = y
-                attrs = attrsD
-                
-                self.abbr_title = None
-                self.abbr_data = ''
-                if attrs.has_attr('title'):
-                    self.abbr_title = attrs['title']
-            else:
-                if self.abbr_title != None:
-                    self.abbr_list[self.abbr_data] = self.abbr_title
-                    self.abbr_title = None
-                self.abbr_data = ''
-        
-        if tag == "a":
-            if start:
-                attrsD = {}
-                for (x, y) in attrs: attrsD[x] = y
-                attrs = attrsD
-                if attrs.has_attr('href') and not (SKIP_INTERNAL_LINKS and attrs['href'].startswith('#')): 
-                    self.astack.append(attrs)
-                    self.o("[")
-                else:
-                    self.astack.append(None)
-            else:
-                if self.astack:
-                    a = self.astack.pop()
-                    if a:
-                        i = self.previousIndex(a)
-                        if i is not None:
-                            a = self.a[i]
-                        else:
-                            self.acount += 1
-                            a['count'] = self.acount
-                            a['outcount'] = self.outcount
-                            self.a.append(a)
-                        self.o("][" + `a['count']` + "]")
-        
-        if tag == "img" and start:
-            attrsD = {}
-            for (x, y) in attrs: attrsD[x] = y
-            attrs = attrsD
-            if attrs.has_attr('src'):
-                attrs['href'] = attrs['src']
-                alt = attrs.get('alt', '')
-                i = self.previousIndex(attrs)
-                if i is not None:
-                    attrs = self.a[i]
-                else:
-                    self.acount += 1
-                    attrs['count'] = self.acount
-                    attrs['outcount'] = self.outcount
-                    self.a.append(attrs)
-                self.o("![")
-                self.o(alt)
-                self.o("]["+`attrs['count']`+"]")
-        
-        if tag == 'dl' and start: self.p()
-        if tag == 'dt' and not start: self.pbr()
-        if tag == 'dd' and start: self.o('    ')
-        if tag == 'dd' and not start: self.pbr()
-        
-        if tag in ["ol", "ul"]:
-            if start:
-                self.list.append({'name':tag, 'num':0})
-            else:
-                if self.list: self.list.pop()
-            
-            self.p()
-        
-        if tag == 'li':
-            if start:
-                self.pbr()
-                if self.list: li = self.list[-1]
-                else: li = {'name':'ul', 'num':0}
-                self.o("  "*len(self.list)) #TODO: line up <ol><li>s > 9 correctly.
-                if li['name'] == "ul": self.o("* ")
-                elif li['name'] == "ol":
-                    li['num'] += 1
-                    self.o(`li['num']`+". ")
-                self.start = 1
-            else:
-                self.pbr()
-        
-        if tag in ["table", "tr"] and start: self.p()
-        if tag == 'td': self.pbr()
-        
-        if tag == "pre":
-            if start:
-                self.startpre = 1
-                self.pre = 1
-            else:
-                self.pre = 0
-            self.p()
-            
-    def pbr(self):
-        if self.p_p == 0: self.p_p = 1
-
-    def p(self): self.p_p = 2
-    
-    def o(self, data, puredata=0, force=0):
-        if self.abbr_data is not None: self.abbr_data += data
-        
-        if not self.quiet: 
-            if puredata and not self.pre:
-                data = re.sub('\s+', ' ', data)
-                if data and data[0] == ' ':
-                    self.space = 1
-                    data = data[1:]
-            if not data and not force: return
-            
-            if self.startpre:
-                #self.out(" :") #TODO: not output when already one there
-                self.startpre = 0
-            
-            bq = (">" * self.blockquote)
-            if not (force and data and data[0] == ">") and self.blockquote: bq += " "
-            
-            if self.pre:
-                bq += "    "
-                data = data.replace("\n", "\n"+bq)
-            
-            if self.start:
-                self.space = 0
-                self.p_p = 0
-                self.start = 0
-
-            if force == 'end':
-                # It's the end.
-                self.p_p = 0
-                self.out("\n")
-                self.space = 0
-
-
-            if self.p_p:
-                self.out(('\n'+bq)*self.p_p)
-                self.space = 0
-                
-            if self.space:
-                if not self.lastWasNL: self.out(' ')
-                self.space = 0
-
-            if self.a and ((self.p_p == 2 and LINKS_EACH_PARAGRAPH) or force == "end"):
-                if force == "end": self.out("\n")
-
-                newa = []
-                for link in self.a:
-                    if self.outcount > link['outcount']:
-                        self.out("   ["+`link['count']`+"]: " + urlparse.urljoin(self.baseurl, link['href'])) 
-                        if link.has_attr('title'): self.out(" ("+link['title']+")")
-                        self.out("\n")
-                    else:
-                        newa.append(link)
-
-                if self.a != newa: self.out("\n") # Don't need an extra line when nothing was done.
-
-                self.a = newa
-            
-            if self.abbr_list and force == "end":
-                for abbr, definition in self.abbr_list.items():
-                    self.out("  *[" + abbr + "]: " + definition + "\n")
-
-            self.p_p = 0
-            self.out(data)
-            self.lastWasNL = data and data[-1] == '\n'
-            self.outcount += 1
-
-    def handle_data(self, data):
-        if r'\/script>' in data: self.quiet -= 1
-        self.o(data, 1)
-    
-    def unknown_decl(self, data): pass
-
-def wrapwrite(text): sys.stdout.write(text.encode('utf8'))
-
-def html2text_file(html, out=wrapwrite, baseurl=''):
-    h = _html2text(out, baseurl)
-    h.feed(html)
-    h.feed("")
-    return h.close()
-
-def html2text(html, baseurl='', wrap_width=BODY_WIDTH):
-    return optwrap(html2text_file(html, None, baseurl),wrap_width)
-
-if __name__ == "__main__":
-    baseurl = ''
-    if sys.argv[1:]:
-        arg = sys.argv[1]
-        if arg.startswith('http://'):
-            baseurl = arg
-            j = urllib.urlopen(baseurl)
-            try:
-                from feedparser import _getCharacterEncoding as enc
-            except ImportError:
-                   enc = lambda x, y: ('utf-8', 1)
-            text = j.read()
-            encoding = enc(j.headers, text)[0]
-            if encoding == 'us-ascii': encoding = 'utf-8'
-            data = text.decode(encoding)
-
-        else:
-            encoding = 'utf8'
-            if len(sys.argv) > 2:
-                encoding = sys.argv[2]
-            data = open(arg, 'r').read().decode(encoding)
-    else:
-        data = sys.stdin.read().decode('utf8')
-    wrapwrite(html2text(data, baseurl))

fanficfare/story.py

@@ -439,8 +439,8 @@ class Story(Configurable):
 
     def prepare_replacements(self):
         if not self.replacements_prepped and not self.is_lightweight():
-            logger.debug("prepare_replacements")
-            logger.debug("sections:%s"%self.configuration.sectionslist)
+            # logger.debug("prepare_replacements")
+            # logger.debug("sections:%s"%self.configuration.sectionslist)
             
             ## Look for config parameter, split and add each to metadata field.
             for (config,metadata) in [("extracategories","category"),
@@ -1067,9 +1067,12 @@ class Story(Configurable):
 
         prefix='ffdl'
         if imgurl not in self.imgurls:
-            parsedUrl = urlparse.urlparse(imgurl)
 
             try:
+                if imgurl == 'failedtoload':
+                    raise Exception("Previously failed to load")
+
+                parsedUrl = urlparse.urlparse(imgurl)
                 if self.getConfig('no_image_processing'):
                     (data,ext,mime) = no_convert_image(imgurl,
                                                        fetch(imgurl))
@@ -1161,5 +1164,5 @@ def unique_list(seq):
     try:
         return [x for x in seq if not (x in seen or seen_add(x))]
     except:
-        print("unique_list exception seq:%s"%seq)
+        logger.debug("unique_list exception seq:%s"%seq)
         raise

fanficfare/writers/writer_txt.py

@@ -21,7 +21,7 @@ from textwrap import wrap
 
 from base_writer import *
 
-from ..html2text import html2text
+from html2text import html2text
 
 ## In BaseStoryWriter, we define _write to encode <unicode> objects
 ## back into <string> for true output.  But txt needs to write the
@@ -109,7 +109,7 @@ End file.
 
         self.wrap_width = self.getConfig('wrap_width')
         if self.wrap_width == '' or self.wrap_width == '0':
-            self.wrap_width = None
+            self.wrap_width = 0
         else:
             self.wrap_width = int(self.wrap_width)
         
@@ -159,7 +159,7 @@ End file.
                 logging.debug('Writing chapter text for: %s' % chap.title)
                 vals={'url':chap.url, 'chapter':chap.title, 'index':"%04d"%(index+1), 'number':index+1}
                 self._write(out,self.lineends(self.wraplines(removeAllEntities(CHAPTER_START.substitute(vals)))))
-                self._write(out,self.lineends(html2text(chap.html,wrap_width=self.wrap_width)))
+                self._write(out,self.lineends(html2text(chap.html,bodywidth=self.wrap_width)))
                 self._write(out,self.lineends(self.wraplines(removeAllEntities(CHAPTER_END.substitute(vals)))))
 
         self._write(out,self.lineends(self.wraplines(FILE_END.substitute(self.story.getAllMetadata()))))

included_dependencies/html2text/__init__.py

@@ -0,0 +1,857 @@
+#!/usr/bin/env python
+# coding: utf-8
+"""html2text: Turn HTML into equivalent Markdown-structured text."""
+from __future__ import division
+import re
+import sys
+import cgi
+
+try:
+    from textwrap import wrap
+except ImportError:  # pragma: no cover
+    pass
+
+from html2text.compat import urlparse, HTMLParser
+from html2text import config
+
+from html2text.utils import (
+    name2cp,
+    unifiable_n,
+    google_text_emphasis,
+    google_fixed_width_font,
+    element_style,
+    hn,
+    google_has_height,
+    escape_md,
+    google_list_style,
+    list_numbering_start,
+    dumb_css_parser,
+    escape_md_section,
+    skipwrap
+)
+
+__version__ = (2016, 4, 2)
+
+
+# TODO:
+# Support decoded entities with UNIFIABLE.
+
+
+class HTML2Text(HTMLParser.HTMLParser):
+    def __init__(self, out=None, baseurl='', bodywidth=config.BODY_WIDTH):
+        """
+        Input parameters:
+            out: possible custom replacement for self.outtextf (which
+                 appends lines of text).
+            baseurl: base URL of the document we process
+        """
+        kwargs = {}
+        if sys.version_info >= (3, 4):
+            kwargs['convert_charrefs'] = False
+        HTMLParser.HTMLParser.__init__(self, **kwargs)
+
+        # Config options
+        self.split_next_td = False
+        self.td_count = 0
+        self.table_start = False
+        self.unicode_snob = config.UNICODE_SNOB  # covered in cli
+        self.escape_snob = config.ESCAPE_SNOB  # covered in cli
+        self.links_each_paragraph = config.LINKS_EACH_PARAGRAPH
+        self.body_width = bodywidth  # covered in cli
+        self.skip_internal_links = config.SKIP_INTERNAL_LINKS  # covered in cli
+        self.inline_links = config.INLINE_LINKS  # covered in cli
+        self.protect_links = config.PROTECT_LINKS  # covered in cli
+        self.google_list_indent = config.GOOGLE_LIST_INDENT  # covered in cli
+        self.ignore_links = config.IGNORE_ANCHORS  # covered in cli
+        self.ignore_images = config.IGNORE_IMAGES  # covered in cli
+        self.images_to_alt = config.IMAGES_TO_ALT  # covered in cli
+        self.images_with_size = config.IMAGES_WITH_SIZE  # covered in cli
+        self.ignore_emphasis = config.IGNORE_EMPHASIS  # covered in cli
+        self.bypass_tables = config.BYPASS_TABLES  # covered in cli
+        self.google_doc = False  # covered in cli
+        self.ul_item_mark = '*'  # covered in cli
+        self.emphasis_mark = '_'  # covered in cli
+        self.strong_mark = '**'
+        self.single_line_break = config.SINGLE_LINE_BREAK  # covered in cli
+        self.use_automatic_links = config.USE_AUTOMATIC_LINKS  # covered in cli
+        self.hide_strikethrough = False  # covered in cli
+        self.mark_code = config.MARK_CODE
+        self.wrap_links = config.WRAP_LINKS  # covered in cli
+        self.tag_callback = None
+
+        if out is None:  # pragma: no cover
+            self.out = self.outtextf
+        else:  # pragma: no cover
+            self.out = out
+
+        # empty list to store output characters before they are "joined"
+        self.outtextlist = []
+
+        self.quiet = 0
+        self.p_p = 0  # number of newline character to print before next output
+        self.outcount = 0
+        self.start = 1
+        self.space = 0
+        self.a = []
+        self.astack = []
+        self.maybe_automatic_link = None
+        self.empty_link = False
+        self.absolute_url_matcher = re.compile(r'^[a-zA-Z+]+://')
+        self.acount = 0
+        self.list = []
+        self.blockquote = 0
+        self.pre = 0
+        self.startpre = 0
+        self.code = False
+        self.br_toggle = ''
+        self.lastWasNL = 0
+        self.lastWasList = False
+        self.style = 0
+        self.style_def = {}
+        self.tag_stack = []
+        self.emphasis = 0
+        self.drop_white_space = 0
+        self.inheader = False
+        self.abbr_title = None  # current abbreviation definition
+        self.abbr_data = None  # last inner HTML (for abbr being defined)
+        self.abbr_list = {}  # stack of abbreviations to write later
+        self.baseurl = baseurl
+
+        try:
+            del unifiable_n[name2cp('nbsp')]
+        except KeyError:
+            pass
+        config.UNIFIABLE['nbsp'] = '&nbsp_place_holder;'
+
+    def feed(self, data):
+        data = data.replace("</' + 'script>", "</ignore>")
+        HTMLParser.HTMLParser.feed(self, data)
+
+    def handle(self, data):
+        self.feed(data)
+        self.feed("")
+        return self.optwrap(self.close())
+
+    def outtextf(self, s):
+        self.outtextlist.append(s)
+        if s:
+            self.lastWasNL = s[-1] == '\n'
+
+    def close(self):
+        HTMLParser.HTMLParser.close(self)
+
+        try:
+            nochr = unicode('')
+        except NameError:
+            nochr = str('')
+
+        self.pbr()
+        self.o('', 0, 'end')
+
+        outtext = nochr.join(self.outtextlist)
+        if self.unicode_snob:
+            try:
+                nbsp = unichr(name2cp('nbsp'))
+            except NameError:
+                nbsp = chr(name2cp('nbsp'))
+        else:
+            try:
+                nbsp = unichr(32)
+            except NameError:
+                nbsp = chr(32)
+        try:
+            outtext = outtext.replace(unicode('&nbsp_place_holder;'), nbsp)
+        except NameError:
+            outtext = outtext.replace('&nbsp_place_holder;', nbsp)
+
+        # Clear self.outtextlist to avoid memory leak of its content to
+        # the next handling.
+        self.outtextlist = []
+
+        return outtext
+
+    def handle_charref(self, c):
+        charref = self.charref(c)
+        if not self.code and not self.pre:
+            charref = cgi.escape(charref)
+        self.handle_data(charref, True)
+
+    def handle_entityref(self, c):
+        entityref = self.entityref(c)
+        if (not self.code and not self.pre
+                and entityref != '&nbsp_place_holder;'):
+            entityref = cgi.escape(entityref)
+        self.handle_data(entityref, True)
+
+    def handle_starttag(self, tag, attrs):
+        self.handle_tag(tag, attrs, 1)
+
+    def handle_endtag(self, tag):
+        self.handle_tag(tag, None, 0)
+
+    def previousIndex(self, attrs):
+        """
+        :type attrs: dict
+
+        :returns: The index of certain set of attributes (of a link) in the
+        self.a list. If the set of attributes is not found, returns None
+        :rtype: int
+        """
+        if 'href' not in attrs:  # pragma: no cover
+            return None
+        i = -1
+        for a in self.a:
+            i += 1
+            match = 0
+
+            if ('href' in a) and a['href'] == attrs['href']:
+                if ('title' in a) or ('title' in attrs):
+                    if (('title' in a) and ('title' in attrs) and
+                                a['title'] == attrs['title']):
+                        match = True
+                else:
+                    match = True
+
+            if match:
+                return i
+
+    def handle_emphasis(self, start, tag_style, parent_style):
+        """
+        Handles various text emphases
+        """
+        tag_emphasis = google_text_emphasis(tag_style)
+        parent_emphasis = google_text_emphasis(parent_style)
+
+        # handle Google's text emphasis
+        strikethrough = 'line-through' in \
+                        tag_emphasis and self.hide_strikethrough
+        bold = 'bold' in tag_emphasis and not 'bold' in parent_emphasis
+        italic = 'italic' in tag_emphasis and not 'italic' in parent_emphasis
+        fixed = google_fixed_width_font(tag_style) and not \
+            google_fixed_width_font(parent_style) and not self.pre
+
+        if start:
+            # crossed-out text must be handled before other attributes
+            # in order not to output qualifiers unnecessarily
+            if bold or italic or fixed:
+                self.emphasis += 1
+            if strikethrough:
+                self.quiet += 1
+            if italic:
+                self.o(self.emphasis_mark)
+                self.drop_white_space += 1
+            if bold:
+                self.o(self.strong_mark)
+                self.drop_white_space += 1
+            if fixed:
+                self.o('`')
+                self.drop_white_space += 1
+                self.code = True
+        else:
+            if bold or italic or fixed:
+                # there must not be whitespace before closing emphasis mark
+                self.emphasis -= 1
+                self.space = 0
+            if fixed:
+                if self.drop_white_space:
+                    # empty emphasis, drop it
+                    self.drop_white_space -= 1
+                else:
+                    self.o('`')
+                self.code = False
+            if bold:
+                if self.drop_white_space:
+                    # empty emphasis, drop it
+                    self.drop_white_space -= 1
+                else:
+                    self.o(self.strong_mark)
+            if italic:
+                if self.drop_white_space:
+                    # empty emphasis, drop it
+                    self.drop_white_space -= 1
+                else:
+                    self.o(self.emphasis_mark)
+            # space is only allowed after *all* emphasis marks
+            if (bold or italic) and not self.emphasis:
+                self.o(" ")
+            if strikethrough:
+                self.quiet -= 1
+
+    def handle_tag(self, tag, attrs, start):
+        # attrs is None for endtags
+        if attrs is None:
+            attrs = {}
+        else:
+            attrs = dict(attrs)
+
+        if self.tag_callback is not None:
+            if self.tag_callback(self, tag, attrs, start) is True:
+                return
+
+        # first thing inside the anchor tag is another tag that produces some output
+        if (start and not self.maybe_automatic_link is None
+                and tag not in ['p', 'div', 'style', 'dl', 'dt']
+                and (tag != "img" or self.ignore_images)):
+            self.o("[")
+            self.maybe_automatic_link = None
+            self.empty_link = False
+
+        if self.google_doc:
+            # the attrs parameter is empty for a closing tag. in addition, we
+            # need the attributes of the parent nodes in order to get a
+            # complete style description for the current element. we assume
+            # that google docs export well formed html.
+            parent_style = {}
+            if start:
+                if self.tag_stack:
+                    parent_style = self.tag_stack[-1][2]
+                tag_style = element_style(attrs, self.style_def, parent_style)
+                self.tag_stack.append((tag, attrs, tag_style))
+            else:
+                dummy, attrs, tag_style = self.tag_stack.pop() if self.tag_stack else (None, {}, {})
+                if self.tag_stack:
+                    parent_style = self.tag_stack[-1][2]
+
+        if hn(tag):
+            self.p()
+            if start:
+                self.inheader = True
+                self.o(hn(tag) * "#" + ' ')
+            else:
+                self.inheader = False
+                return  # prevent redundant emphasis marks on headers
+
+        if tag in ['p', 'div']:
+            if self.google_doc:
+                if start and google_has_height(tag_style):
+                    self.p()
+                else:
+                    self.soft_br()
+            else:
+                self.p()
+
+        if tag == "br" and start:
+            self.o("  \n")
+
+        if tag == "hr" and start:
+            self.p()
+            self.o("* * *")
+            self.p()
+
+        if tag in ["head", "style", 'script']:
+            if start:
+                self.quiet += 1
+            else:
+                self.quiet -= 1
+
+        if tag == "style":
+            if start:
+                self.style += 1
+            else:
+                self.style -= 1
+
+        if tag in ["body"]:
+            self.quiet = 0  # sites like 9rules.com never close <head>
+
+        if tag == "blockquote":
+            if start:
+                self.p()
+                self.o('> ', 0, 1)
+                self.start = 1
+                self.blockquote += 1
+            else:
+                self.blockquote -= 1
+                self.p()
+
+        if tag in ['em', 'i', 'u'] and not self.ignore_emphasis:
+            self.o(self.emphasis_mark)
+        if tag in ['strong', 'b'] and not self.ignore_emphasis:
+            self.o(self.strong_mark)
+        if tag in ['del', 'strike', 's']:
+            if start:
+                self.o('~~')
+            else:
+                self.o('~~')
+
+        if self.google_doc:
+            if not self.inheader:
+                # handle some font attributes, but leave headers clean
+                self.handle_emphasis(start, tag_style, parent_style)
+
+        if tag in ["code", "tt"] and not self.pre:
+            self.o('`')  # TODO: `` `this` ``
+            self.code = not self.code
+        if tag == "abbr":
+            if start:
+                self.abbr_title = None
+                self.abbr_data = ''
+                if ('title' in attrs):
+                    self.abbr_title = attrs['title']
+            else:
+                if self.abbr_title is not None:
+                    self.abbr_list[self.abbr_data] = self.abbr_title
+                    self.abbr_title = None
+                self.abbr_data = ''
+
+        if tag == "a" and not self.ignore_links:
+            if start:
+                if ('href' in attrs) and \
+                        (attrs['href'] is not None) and \
+                        not (self.skip_internal_links and
+                                 attrs['href'].startswith('#')):
+                    self.astack.append(attrs)
+                    self.maybe_automatic_link = attrs['href']
+                    self.empty_link = True
+                    if self.protect_links:
+                        attrs['href'] = '<'+attrs['href']+'>'
+                else:
+                    self.astack.append(None)
+            else:
+                if self.astack:
+                    a = self.astack.pop()
+                    if self.maybe_automatic_link and not self.empty_link:
+                        self.maybe_automatic_link = None
+                    elif a:
+                        if self.empty_link:
+                            self.o("[")
+                            self.empty_link = False
+                            self.maybe_automatic_link = None
+                        if self.inline_links:
+                            try:
+                                title = escape_md(a['title'])
+                            except KeyError:
+                                self.o("](" + escape_md(urlparse.urljoin(self.baseurl, a['href'])) + ")")
+                            else:
+                                self.o("](" + escape_md(urlparse.urljoin(self.baseurl, a['href']))
+                                       + ' "' + title + '" )')
+                        else:
+                            i = self.previousIndex(a)
+                            if i is not None:
+                                a = self.a[i]
+                            else:
+                                self.acount += 1
+                                a['count'] = self.acount
+                                a['outcount'] = self.outcount
+                                self.a.append(a)
+                            self.o("][" + str(a['count']) + "]")
+
+        if tag == "img" and start and not self.ignore_images:
+            if 'src' in attrs:
+                if not self.images_to_alt:
+                    attrs['href'] = attrs['src']
+                alt = attrs.get('alt') or ''
+
+                # If we have images_with_size, write raw html including width,
+                # height, and alt attributes
+                if self.images_with_size and \
+                        ("width" in attrs or "height" in attrs):
+                    self.o("<img src='" + attrs["src"] + "' ")
+                    if "width" in attrs:
+                        self.o("width='" + attrs["width"] + "' ")
+                    if "height" in attrs:
+                        self.o("height='" + attrs["height"] + "' ")
+                    if alt:
+                        self.o("alt='" + alt + "' ")
+                    self.o("/>")
+                    return
+
+                # If we have a link to create, output the start
+                if not self.maybe_automatic_link is None:
+                    href = self.maybe_automatic_link
+                    if self.images_to_alt and escape_md(alt) == href and \
+                            self.absolute_url_matcher.match(href):
+                        self.o("<" + escape_md(alt) + ">")
+                        self.empty_link = False
+                        return
+                    else:
+                        self.o("[")
+                        self.maybe_automatic_link = None
+                        self.empty_link = False
+
+                # If we have images_to_alt, we discard the image itself,
+                # considering only the alt text.
+                if self.images_to_alt:
+                    self.o(escape_md(alt))
+                else:
+                    self.o("![" + escape_md(alt) + "]")
+                    if self.inline_links:
+                        href = attrs.get('href') or ''
+                        self.o("(" + escape_md(urlparse.urljoin(self.baseurl, href)) + ")")
+                    else:
+                        i = self.previousIndex(attrs)
+                        if i is not None:
+                            attrs = self.a[i]
+                        else:
+                            self.acount += 1
+                            attrs['count'] = self.acount
+                            attrs['outcount'] = self.outcount
+                            self.a.append(attrs)
+                        self.o("[" + str(attrs['count']) + "]")
+
+        if tag == 'dl' and start:
+            self.p()
+        if tag == 'dt' and not start:
+            self.pbr()
+        if tag == 'dd' and start:
+            self.o('    ')
+        if tag == 'dd' and not start:
+            self.pbr()
+
+        if tag in ["ol", "ul"]:
+            # Google Docs create sub lists as top level lists
+            if (not self.list) and (not self.lastWasList):
+                self.p()
+            if start:
+                if self.google_doc:
+                    list_style = google_list_style(tag_style)
+                else:
+                    list_style = tag
+                numbering_start = list_numbering_start(attrs)
+                self.list.append({
+                    'name': list_style,
+                    'num': numbering_start
+                })
+            else:
+                if self.list:
+                    self.list.pop()
+                    if (not self.google_doc) and (not self.list):
+                        self.o('\n')
+            self.lastWasList = True
+        else:
+            self.lastWasList = False
+
+        if tag == 'li':
+            self.pbr()
+            if start:
+                if self.list:
+                    li = self.list[-1]
+                else:
+                    li = {'name': 'ul', 'num': 0}
+                if self.google_doc:
+                    nest_count = self.google_nest_count(tag_style)
+                else:
+                    nest_count = len(self.list)
+                # TODO: line up <ol><li>s > 9 correctly.
+                self.o("  " * nest_count)
+                if li['name'] == "ul":
+                    self.o(self.ul_item_mark + " ")
+                elif li['name'] == "ol":
+                    li['num'] += 1
+                    self.o(str(li['num']) + ". ")
+                self.start = 1
+
+        if tag in ["table", "tr", "td", "th"]:
+            if self.bypass_tables:
+                if start:
+                    self.soft_br()
+                if tag in ["td", "th"]:
+                    if start:
+                        self.o('<{0}>\n\n'.format(tag))
+                    else:
+                        self.o('\n</{0}>'.format(tag))
+                else:
+                    if start:
+                        self.o('<{0}>'.format(tag))
+                    else:
+                        self.o('</{0}>'.format(tag))
+
+            else:
+                if tag == "table" and start:
+                    self.table_start = True
+                if tag in ["td", "th"] and start:
+                    if self.split_next_td:
+                        self.o("| ")
+                    self.split_next_td = True
+
+                if tag == "tr" and start:
+                    self.td_count = 0
+                if tag == "tr" and not start:
+                    self.split_next_td = False
+                    self.soft_br()
+                if tag == "tr" and not start and self.table_start:
+                    # Underline table header
+                    self.o("|".join(["---"] * self.td_count))
+                    self.soft_br()
+                    self.table_start = False
+                if tag in ["td", "th"] and start:
+                    self.td_count += 1
+
+        if tag == "pre":
+            if start:
+                self.startpre = 1
+                self.pre = 1
+            else:
+                self.pre = 0
+                if self.mark_code:
+                    self.out("\n[/code]")
+            self.p()
+
+    # TODO: Add docstring for these one letter functions
+    def pbr(self):
+        "Pretty print has a line break"
+        if self.p_p == 0:
+            self.p_p = 1
+
+    def p(self):
+        "Set pretty print to 1 or 2 lines"
+        self.p_p = 1 if self.single_line_break else 2
+
+    def soft_br(self):
+        "Soft breaks"
+        self.pbr()
+        self.br_toggle = '  '
+
+    def o(self, data, puredata=0, force=0):
+        """
+        Deal with indentation and whitespace
+        """
+        if self.abbr_data is not None:
+            self.abbr_data += data
+
+        if not self.quiet:
+            if self.google_doc:
+                # prevent white space immediately after 'begin emphasis'
+                # marks ('**' and '_')
+                lstripped_data = data.lstrip()
+                if self.drop_white_space and not (self.pre or self.code):
+                    data = lstripped_data
+                if lstripped_data != '':
+                    self.drop_white_space = 0
+
+            if puredata and not self.pre:
+                # This is a very dangerous call ... it could mess up
+                # all handling of &nbsp; when not handled properly
+                # (see entityref)
+                data = re.sub(r'\s+', r' ', data)
+                if data and data[0] == ' ':
+                    self.space = 1
+                    data = data[1:]
+            if not data and not force:
+                return
+
+            if self.startpre:
+                #self.out(" :") #TODO: not output when already one there
+                if not data.startswith("\n"):  # <pre>stuff...
+                    data = "\n" + data
+                if self.mark_code:
+                    self.out("\n[code]")
+                    self.p_p = 0
+
+            bq = (">" * self.blockquote)
+            if not (force and data and data[0] == ">") and self.blockquote:
+                bq += " "
+
+            if self.pre:
+                if not self.list:
+                    bq += "    "
+                #else: list content is already partially indented
+                for i in range(len(self.list)):
+                    bq += "    "
+                data = data.replace("\n", "\n" + bq)
+
+            if self.startpre:
+                self.startpre = 0
+                if self.list:
+                    # use existing initial indentation
+                    data = data.lstrip("\n")
+
+            if self.start:
+                self.space = 0
+                self.p_p = 0
+                self.start = 0
+
+            if force == 'end':
+                # It's the end.
+                self.p_p = 0
+                self.out("\n")
+                self.space = 0
+
+            if self.p_p:
+                self.out((self.br_toggle + '\n' + bq) * self.p_p)
+                self.space = 0
+                self.br_toggle = ''
+
+            if self.space:
+                if not self.lastWasNL:
+                    self.out(' ')
+                self.space = 0
+
+            if self.a and ((self.p_p == 2 and self.links_each_paragraph)
+                           or force == "end"):
+                if force == "end":
+                    self.out("\n")
+
+                newa = []
+                for link in self.a:
+                    if self.outcount > link['outcount']:
+                        self.out("   [" + str(link['count']) + "]: " +
+                                 urlparse.urljoin(self.baseurl, link['href']))
+                        if 'title' in link:
+                            self.out(" (" + link['title'] + ")")
+                        self.out("\n")
+                    else:
+                        newa.append(link)
+
+                # Don't need an extra line when nothing was done.
+                if self.a != newa:
+                    self.out("\n")
+
+                self.a = newa
+
+            if self.abbr_list and force == "end":
+                for abbr, definition in self.abbr_list.items():
+                    self.out("  *[" + abbr + "]: " + definition + "\n")
+
+            self.p_p = 0
+            self.out(data)
+            self.outcount += 1
+
+    def handle_data(self, data, entity_char=False):
+        if r'\/script>' in data:
+            self.quiet -= 1
+
+        if self.style:
+            self.style_def.update(dumb_css_parser(data))
+
+        if not self.maybe_automatic_link is None:
+            href = self.maybe_automatic_link
+            if (href == data and self.absolute_url_matcher.match(href)
+                    and self.use_automatic_links):
+                self.o("<" + data + ">")
+                self.empty_link = False
+                return
+            else:
+                self.o("[")
+                self.maybe_automatic_link = None
+                self.empty_link = False
+
+        if not self.code and not self.pre and not entity_char:
+            data = escape_md_section(data, snob=self.escape_snob)
+        self.o(data, 1)
+
+    def unknown_decl(self, data):  # pragma: no cover
+        # TODO: what is this doing here?
+        pass
+
+    def charref(self, name):
+        if name[0] in ['x', 'X']:
+            c = int(name[1:], 16)
+        else:
+            c = int(name)
+
+        if not self.unicode_snob and c in unifiable_n.keys():
+            return unifiable_n[c]
+        else:
+            try:
+                try:
+                    return unichr(c)
+                except NameError:  # Python3
+                    return chr(c)
+            except ValueError:  # invalid unicode
+                return ''
+
+    def entityref(self, c):
+        if not self.unicode_snob and c in config.UNIFIABLE.keys():
+            return config.UNIFIABLE[c]
+        else:
+            try:
+                name2cp(c)
+            except KeyError:
+                return "&" + c + ';'
+            else:
+                if c == 'nbsp':
+                    return config.UNIFIABLE[c]
+                else:
+                    try:
+                        return unichr(name2cp(c))
+                    except NameError:  # Python3
+                        return chr(name2cp(c))
+
+    def replaceEntities(self, s):
+        s = s.group(1)
+        if s[0] == "#":
+            return self.charref(s[1:])
+        else:
+            return self.entityref(s)
+
+    def unescape(self, s):
+        return config.RE_UNESCAPE.sub(self.replaceEntities, s)
+
+    def google_nest_count(self, style):
+        """
+        Calculate the nesting count of google doc lists
+
+        :type style: dict
+
+        :rtype: int
+        """
+        nest_count = 0
+        if 'margin-left' in style:
+            nest_count = int(style['margin-left'][:-2]) \
+                         // self.google_list_indent
+
+        return nest_count
+
+    def optwrap(self, text):
+        """
+        Wrap all paragraphs in the provided text.
+
+        :type text: str
+
+        :rtype: str
+        """
+        if not self.body_width:
+            return text
+
+        assert wrap, "Requires Python 2.3."
+        result = ''
+        newlines = 0
+        # I cannot think of a better solution for now.
+        # To avoid the non-wrap behaviour for entire paras
+        # because of the presence of a link in it
+        if not self.wrap_links:
+            self.inline_links = False
+        for para in text.split("\n"):
+            if len(para) > 0:
+                if not skipwrap(para, self.wrap_links):
+                    result += "\n".join(wrap(para, self.body_width))
+                    if para.endswith('  '):
+                        result += "  \n"
+                        newlines = 1
+                    else:
+                        result += "\n\n"
+                        newlines = 2
+                else:
+                    # Warning for the tempted!!!
+                    # Be aware that obvious replacement of this with
+                    # line.isspace()
+                    # DOES NOT work! Explanations are welcome.
+                    if not config.RE_SPACE.match(para):
+                        result += para + "\n"
+                        newlines = 1
+            else:
+                if newlines < 2:
+                    result += "\n"
+                    newlines += 1
+        return result
+
+
+def html2text(html, baseurl='', bodywidth=None):
+    if bodywidth is None:
+        bodywidth = config.BODY_WIDTH
+    h = HTML2Text(baseurl=baseurl, bodywidth=bodywidth)
+
+    return h.handle(html)
+
+
+def unescape(s, unicode_snob=False):
+    h = HTML2Text()
+    h.unicode_snob = unicode_snob
+
+    return h.unescape(s)
+
+
+if __name__ == "__main__":
+    from html2text.cli import main
+
+    main()

included_dependencies/html2text/compat.py

@@ -0,0 +1,13 @@
+import sys
+
+
+if sys.version_info[0] == 2:
+    import htmlentitydefs
+    import urlparse
+    import HTMLParser
+    import urllib
+else:
+    import urllib.parse as urlparse
+    import html.entities as htmlentitydefs
+    import html.parser as HTMLParser
+    import urllib.request as urllib

included_dependencies/html2text/config.py

@@ -0,0 +1,123 @@
+import re
+
+# Use Unicode characters instead of their ascii psuedo-replacements
+UNICODE_SNOB = 0
+
+# Escape all special characters.  Output is less readable, but avoids
+# corner case formatting issues.
+ESCAPE_SNOB = 0
+
+# Put the links after each paragraph instead of at the end.
+LINKS_EACH_PARAGRAPH = 0
+
+# Wrap long lines at position. 0 for no wrapping. (Requires Python 2.3.)
+BODY_WIDTH = 78
+
+# Don't show internal links (href="#local-anchor") -- corresponding link
+# targets won't be visible in the plain text file anyway.
+SKIP_INTERNAL_LINKS = True
+
+# Use inline, rather than reference, formatting for images and links
+INLINE_LINKS = True
+
+# Protect links from line breaks surrounding them with angle brackets (in
+# addition to their square brackets)
+PROTECT_LINKS = False
+# WRAP_LINKS = True
+WRAP_LINKS = True
+
+# Number of pixels Google indents nested lists
+GOOGLE_LIST_INDENT = 36
+
+IGNORE_ANCHORS = False
+IGNORE_IMAGES = False
+IMAGES_TO_ALT = False
+IMAGES_WITH_SIZE = False
+IGNORE_EMPHASIS = False
+MARK_CODE = False
+DECODE_ERRORS = 'strict'
+
+# Convert links with same href and text to <href> format if they are absolute links
+USE_AUTOMATIC_LINKS = True
+
+# For checking space-only lines on line 771
+RE_SPACE = re.compile(r'\s\+')
+
+RE_UNESCAPE = re.compile(r"&(#?[xX]?(?:[0-9a-fA-F]+|\w{1,8}));")
+RE_ORDERED_LIST_MATCHER = re.compile(r'\d+\.\s')
+RE_UNORDERED_LIST_MATCHER = re.compile(r'[-\*\+]\s')
+RE_MD_CHARS_MATCHER = re.compile(r"([\\\[\]\(\)])")
+RE_MD_CHARS_MATCHER_ALL = re.compile(r"([`\*_{}\[\]\(\)#!])")
+RE_LINK = re.compile(r"(\[.*?\] ?\(.*?\))|(\[.*?\]:.*?)")  # to find links in the text
+RE_MD_DOT_MATCHER = re.compile(r"""
+    ^             # start of line
+    (\s*\d+)      # optional whitespace and a number
+    (\.)          # dot
+    (?=\s)        # lookahead assert whitespace
+    """, re.MULTILINE | re.VERBOSE)
+RE_MD_PLUS_MATCHER = re.compile(r"""
+    ^
+    (\s*)
+    (\+)
+    (?=\s)
+    """, flags=re.MULTILINE | re.VERBOSE)
+RE_MD_DASH_MATCHER = re.compile(r"""
+    ^
+    (\s*)
+    (-)
+    (?=\s|\-)     # followed by whitespace (bullet list, or spaced out hr)
+                  # or another dash (header or hr)
+    """, flags=re.MULTILINE | re.VERBOSE)
+RE_SLASH_CHARS = r'\`*_{}[]()#+-.!'
+RE_MD_BACKSLASH_MATCHER = re.compile(r'''
+    (\\)          # match one slash
+    (?=[%s])      # followed by a char that requires escaping
+    ''' % re.escape(RE_SLASH_CHARS),
+    flags=re.VERBOSE)
+
+UNIFIABLE = {
+    'rsquo': "'",
+    'lsquo': "'",
+    'rdquo': '"',
+    'ldquo': '"',
+    'copy': '(C)',
+    'mdash': '--',
+    'nbsp': ' ',
+    'rarr': '->',
+    'larr': '<-',
+    'middot': '*',
+    'ndash': '-',
+    'oelig': 'oe',
+    'aelig': 'ae',
+    'agrave': 'a',
+    'aacute': 'a',
+    'acirc': 'a',
+    'atilde': 'a',
+    'auml': 'a',
+    'aring': 'a',
+    'egrave': 'e',
+    'eacute': 'e',
+    'ecirc': 'e',
+    'euml': 'e',
+    'igrave': 'i',
+    'iacute': 'i',
+    'icirc': 'i',
+    'iuml': 'i',
+    'ograve': 'o',
+    'oacute': 'o',
+    'ocirc': 'o',
+    'otilde': 'o',
+    'ouml': 'o',
+    'ugrave': 'u',
+    'uacute': 'u',
+    'ucirc': 'u',
+    'uuml': 'u',
+    'lrm': '',
+    'rlm': ''
+}
+
+BYPASS_TABLES = False
+
+# Use a single line break after a block element rather an two line breaks.
+# NOTE: Requires body width setting to be 0.
+SINGLE_LINE_BREAK = False

included_dependencies/html2text/utils.py

@@ -0,0 +1,246 @@
+import sys
+
+from html2text import config
+from html2text.compat import htmlentitydefs
+
+
+def name2cp(k):
+    """Return sname to codepoint"""
+    if k == 'apos':
+        return ord("'")
+    return htmlentitydefs.name2codepoint[k]
+
+
+unifiable_n = {}
+
+for k in config.UNIFIABLE.keys():
+    unifiable_n[name2cp(k)] = config.UNIFIABLE[k]
+
+
+def hn(tag):
+    if tag[0] == 'h' and len(tag) == 2:
+        try:
+            n = int(tag[1])
+            if n in range(1, 10):  # pragma: no branch
+                return n
+        except ValueError:
+            return 0
+
+
+def dumb_property_dict(style):
+    """
+    :returns: A hash of css attributes
+    """
+    out = dict([(x.strip(), y.strip()) for x, y in
+                [z.split(':', 1) for z in
+                 style.split(';') if ':' in z
+                 ]
+                ]
+               )
+
+    return out
+
+
+def dumb_css_parser(data):
+    """
+    :type data: str
+
+    :returns: A hash of css selectors, each of which contains a hash of
+    css attributes.
+    :rtype: dict
+    """
+    # remove @import sentences
+    data += ';'
+    importIndex = data.find('@import')
+    while importIndex != -1:
+        data = data[0:importIndex] + data[data.find(';', importIndex) + 1:]
+        importIndex = data.find('@import')
+
+    # parse the css. reverted from dictionary comprehension in order to
+    # support older pythons
+    elements = [x.split('{') for x in data.split('}') if '{' in x.strip()]
+    try:
+        elements = dict([(a.strip(), dumb_property_dict(b))
+                         for a, b in elements])
+    except ValueError:  # pragma: no cover
+        elements = {}  # not that important
+
+    return elements
+
+
+def element_style(attrs, style_def, parent_style):
+    """
+    :type attrs: dict
+    :type style_def: dict
+    :type style_def: dict
+
+    :returns: A hash of the 'final' style attributes of the element
+    :rtype: dict
+    """
+    style = parent_style.copy()
+    if 'class' in attrs:
+        for css_class in attrs['class'].split():
+            css_style = style_def.get('.' + css_class, {})
+            style.update(css_style)
+    if 'style' in attrs:
+        immediate_style = dumb_property_dict(attrs['style'])
+        style.update(immediate_style)
+
+    return style
+
+
+def google_list_style(style):
+    """
+    Finds out whether this is an ordered or unordered list
+
+    :type style: dict
+
+    :rtype: str
+    """
+    if 'list-style-type' in style:
+        list_style = style['list-style-type']
+        if list_style in ['disc', 'circle', 'square', 'none']:
+            return 'ul'
+
+    return 'ol'
+
+
+def google_has_height(style):
+    """
+    Check if the style of the element has the 'height' attribute
+    explicitly defined
+
+    :type style: dict
+
+    :rtype: bool
+    """
+    if 'height' in style:
+        return True
+
+    return False
+
+
+def google_text_emphasis(style):
+    """
+    :type style: dict
+
+    :returns: A list of all emphasis modifiers of the element
+    :rtype: list
+    """
+    emphasis = []
+    if 'text-decoration' in style:
+        emphasis.append(style['text-decoration'])
+    if 'font-style' in style:
+        emphasis.append(style['font-style'])
+    if 'font-weight' in style:
+        emphasis.append(style['font-weight'])
+
+    return emphasis
+
+
+def google_fixed_width_font(style):
+    """
+    Check if the css of the current element defines a fixed width font
+
+    :type style: dict
+
+    :rtype: bool
+    """
+    font_family = ''
+    if 'font-family' in style:
+        font_family = style['font-family']
+    if 'Courier New' == font_family or 'Consolas' == font_family:
+        return True
+
+    return False
+
+
+def list_numbering_start(attrs):
+    """
+    Extract numbering from list element attributes
+
+    :type attrs: dict
+
+    :rtype: int or None
+    """
+    if 'start' in attrs:
+        try:
+            return int(attrs['start']) - 1
+        except ValueError:
+            pass
+
+    return 0
+
+
+def skipwrap(para, wrap_links):
+    # If it appears to contain a link
+    # don't wrap
+    if (len(config.RE_LINK.findall(para)) > 0) and not wrap_links:
+        return True
+    # If the text begins with four spaces or one tab, it's a code block;
+    # don't wrap
+    if para[0:4] == '    ' or para[0] == '\t':
+        return True
+
+    # If the text begins with only two "--", possibly preceded by
+    # whitespace, that's an emdash; so wrap.
+    stripped = para.lstrip()
+    if stripped[0:2] == "--" and len(stripped) > 2 and stripped[2] != "-":
+        return False
+
+    # I'm not sure what this is for; I thought it was to detect lists,
+    # but there's a <br>-inside-<span> case in one of the tests that
+    # also depends upon it.
+    if stripped[0:1] == '-' or stripped[0:1] == '*':
+        return True
+
+    # If the text begins with a single -, *, or +, followed by a space,
+    # or an integer, followed by a ., followed by a space (in either
+    # case optionally proceeded by whitespace), it's a list; don't wrap.
+    if config.RE_ORDERED_LIST_MATCHER.match(stripped) or \
+            config.RE_UNORDERED_LIST_MATCHER.match(stripped):
+        return True
+
+    return False
+
+
+def wrapwrite(text):
+    text = text.encode('utf-8')
+    try:  # Python3
+        sys.stdout.buffer.write(text)
+    except AttributeError:
+        sys.stdout.write(text)
+
+
+def wrap_read():  # pragma: no cover
+    """
+    :rtype: str
+    """
+    try:
+        return sys.stdin.read()
+    except AttributeError:
+        return sys.stdin.buffer.read()
+
+
+def escape_md(text):
+    """
+    Escapes markdown-sensitive characters within other markdown
+    constructs.
+    """
+    return config.RE_MD_CHARS_MATCHER.sub(r"\\\1", text)
+
+
+def escape_md_section(text, snob=False):
+    """
+    Escapes markdown-sensitive characters across whole document sections.
+    """
+    text = config.RE_MD_BACKSLASH_MATCHER.sub(r"\\\1", text)
+
+    if snob:
+        text = config.RE_MD_CHARS_MATCHER_ALL.sub(r"\\\1", text)
+
+    text = config.RE_MD_DOT_MATCHER.sub(r"\1\\\2", text)
+    text = config.RE_MD_PLUS_MATCHER.sub(r"\1\\\2", text)
+    text = config.RE_MD_DASH_MATCHER.sub(r"\1\\\2", text)
+
+    return text

makeplugin.py

@@ -36,7 +36,7 @@ if __name__=="__main__":
 
     os.chdir('../included_dependencies')
     # 'a' for append
-    files=['gif.py','six.py','bs4','html5lib','chardet']
+    files=['gif.py','six.py','bs4','html5lib','chardet','html2text']
     createZipFile("../"+filename,"a",
                   files,
                   exclude=exclude)

setup.py

@@ -23,7 +23,7 @@ setup(
     # Versions should comply with PEP440.  For a discussion on single-sourcing
     # the version across setup.py and the project code, see
     # https://packaging.python.org/en/latest/single_source_version.html
-    version="2.3.0",
+    version="2.3.4",
 
     description='A tool for downloading fanfiction to eBook formats',
     long_description=long_description,
@@ -81,7 +81,7 @@ setup(
     # your project is installed. For an analysis of "install_requires" vs pip's
     # requirements files see:
     # https://packaging.python.org/en/latest/requirements.html
-    install_requires=['beautifulsoup4','chardet','html5lib'], # html5lib requires 'six'.
+    install_requires=['beautifulsoup4','chardet','html5lib','html2text'], # html5lib requires 'six'.
 
     # List additional groups of dependencies here (e.g. development
     # dependencies). You can install these using the following syntax,

webservice/app.yaml

@@ -1,6 +1,6 @@
 # ffd-retief-hrd fanficfare
 application: fanficfare
-version: 2-3-00
+version: 2-3-04
 runtime: python27
 api_version: 1
 threadsafe: true

webservice/editconfig.html

@@ -39,8 +39,8 @@
 	<img src="http://code.google.com/appengine/images/appengine-silver-120x30.gif"
 	     alt="Powered by Google App Engine" />
 	<br/><br/>
-	This is a web front-end to <A href="http://code.google.com/p/fanficdownloader/">FanFicFare</a><br/>
-	Copyright &copy; Fanficdownloader team
+	This is a web front-end to <a href="https://github.com/JimmXinu/FanFicFare/">FanFicFare</a><br/>
+	Copyright &copy; FanFicFare team
       </div>
 
     </div>

webservice/index.html

@@ -35,7 +35,7 @@
 	    If you have any problems with this application, please
 	    report them in
 	    the <a href="http://groups.google.com/group/fanfic-downloader">FanFicFare Google Group</a>.  The
-	    <a href="http://2-2-18.fanficfare.appspot.com">previous version
+	    <a href="http://2-3-03.fanficfare.appspot.com">previous version
 	    </a> is also available for you to use if necessary.
 	  </p>
 	  <div id='error'>

webservice/login.html

@@ -68,7 +68,7 @@
 	<img src="http://code.google.com/appengine/images/appengine-silver-120x30.gif"
 	     alt="Powered by Google App Engine" />
 	<br/><br/>
-	This is a web front-end to <A href="http://code.google.com/p/fanficdownloader/">FanFicFare</a><br/>
+	This is a web front-end to <a href="https://github.com/JimmXinu/FanFicFare/">FanFicFare</a><br/>
 	Copyright &copy; FanFicFare team
       </div>
 

webservice/status.html

@@ -45,7 +45,7 @@
 	<img src="http://code.google.com/appengine/images/appengine-silver-120x30.gif"
 	     alt="Powered by Google App Engine" />
 	<br/><br/>
-	This is a web front-end to <A href="http://code.google.com/p/fanficdownloader/">FanFicFare</a><br/>
+	This is a web front-end to <a href="https://github.com/JimmXinu/FanFicFare/">FanFicFare</a><br/>
 	Copyright &copy; FanFicFare team
       </div>