Fix for yet another ffnet change, bump versions.

the soup processing.

app.yaml

@@ -1,6 +1,6 @@
 # ffd-retief-hrd fanfictiondownloader
 application: fanfictiondownloader
-version: 4-4-71
+version: 4-4-82
 runtime: python27
 api_version: 1
 threadsafe: true

calibre-plugin/__init__.py

@@ -7,6 +7,16 @@ __license__   = 'GPL v3'
 __copyright__ = '2013, Jim Miller'
 __docformat__ = 'restructuredtext en'
 
+import sys
+if sys.version_info >= (2, 7):
+    import logging
+    logger = logging.getLogger(__name__)
+    loghandler=logging.StreamHandler()
+    loghandler.setFormatter(logging.Formatter("FFDL:%(levelname)s:%(filename)s(%(lineno)d):%(message)s"))
+    logger.addHandler(loghandler)
+    loghandler.setLevel(logging.DEBUG)
+    logger.setLevel(logging.DEBUG)
+
 # The class that all Interface Action plugin wrappers must inherit from
 from calibre.customize import InterfaceActionBase
 
@@ -26,7 +36,7 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
     description         = 'UI plugin to download FanFiction stories from various sites.'
     supported_platforms = ['windows', 'osx', 'linux']
     author              = 'Jim Miller'
-    version             = (1, 7, 42)
+    version             = (1, 7, 53)
     minimum_calibre_version = (0, 8, 57)
 
     #: This field defines the GUI plugin class that contains all the code

calibre-plugin/config.py

@@ -7,6 +7,9 @@ __license__   = 'GPL v3'
 __copyright__ = '2012, Jim Miller'
 __docformat__ = 'restructuredtext en'
 
+import logging
+logger = logging.getLogger(__name__)
+
 import traceback, copy, threading
 from collections import OrderedDict
 
@@ -126,9 +129,15 @@ class ConfigWidget(QWidget):
         label.setOpenExternalLinks(True)
         self.l.addWidget(label)
 
-        tab_widget = QTabWidget(self)
-        self.l.addWidget(tab_widget)
 
+        self.scroll_area = QScrollArea(self)
+        self.scroll_area.setFrameShape(QScrollArea.NoFrame)
+        self.scroll_area.setWidgetResizable(True)
+        self.l.addWidget(self.scroll_area)
+
+        tab_widget = QTabWidget(self)
+        self.scroll_area.setWidget(tab_widget)
+        
         self.basic_tab = BasicTab(self, plugin_action)
         tab_widget.addTab(self.basic_tab, 'Basic')
 
@@ -171,6 +180,7 @@ class ConfigWidget(QWidget):
         prefs['keeptags'] = self.basic_tab.keeptags.isChecked()
         prefs['suppressauthorsort'] = self.basic_tab.suppressauthorsort.isChecked()
         prefs['suppresstitlesort'] = self.basic_tab.suppresstitlesort.isChecked()
+        prefs['mark'] = self.basic_tab.mark.isChecked()
         prefs['showmarked'] = self.basic_tab.showmarked.isChecked()
         prefs['urlsfromclip'] = self.basic_tab.urlsfromclip.isChecked()
         prefs['updatedefault'] = self.basic_tab.updatedefault.isChecked()
@@ -375,8 +385,13 @@ class BasicTab(QWidget):
         self.lookforurlinhtml.setChecked(prefs['lookforurlinhtml'])
         self.l.addWidget(self.lookforurlinhtml)
 
-        self.showmarked = QCheckBox("Show added/updated books when finished?",self)
-        self.showmarked.setToolTip("Show added/updated books only when finished.\nYou can also manually search for 'marked:ffdl_success'.\n'marked:ffdl_failed' is also available, or search 'marked:ffdl' for both.")
+        self.mark = QCheckBox("Mark added/updated books when finished?",self)
+        self.mark.setToolTip("Mark added/updated books when finished.  Use with option below.\nYou can also manually search for 'marked:ffdl_success'.\n'marked:ffdl_failed' is also available, or search 'marked:ffdl' for both.")
+        self.mark.setChecked(prefs['mark'])
+        self.l.addWidget(self.mark)
+
+        self.showmarked = QCheckBox("Show Marked books when finished?",self)
+        self.showmarked.setToolTip("Show Marked added/updated books only when finished.\nYou can also manually search for 'marked:ffdl_success'.\n'marked:ffdl_failed' is also available, or search 'marked:ffdl' for both.")
         self.showmarked.setChecked(prefs['showmarked'])
         self.l.addWidget(self.showmarked)
 
@@ -437,15 +452,17 @@ class BasicTab(QWidget):
         topl.addWidget(defs_gb)
 
         horz = QHBoxLayout()
-        topl.addLayout(horz)
-        horz.addWidget(cali_gb)
-        horz.addWidget(rej_gb)
-        
-        horz = QHBoxLayout()
-        topl.addLayout(horz)
-        horz.addWidget(gui_gb)
-        horz.addWidget(misc_gb)
 
+        horz.addWidget(cali_gb)
+
+        vert = QVBoxLayout()
+        vert.addWidget(gui_gb)
+        vert.addWidget(misc_gb)
+        vert.addWidget(rej_gb)
+        
+        horz.addLayout(vert)
+        
+        topl.addLayout(horz)
         topl.insertStretch(-1)
         
     def set_collisions(self):
@@ -523,7 +540,7 @@ class PersonalIniTab(QWidget):
             self.ini.setFont(QFont("Courier",
                                    self.plugin_action.gui.font().pointSize()+1))
         except Exception as e:
-            print("Couldn't get font: %s"%e)
+            logger.error("Couldn't get font: %s"%e)
         self.ini.setLineWrapMode(QTextEdit.NoWrap)
         self.ini.setText(prefs['personal.ini'])
         self.l.addWidget(self.ini)
@@ -559,7 +576,7 @@ class ShowDefaultsIniDialog(QDialog):
             self.ini.setFont(QFont("Courier",
                                    get_gui().font().pointSize()+1))
         except Exception as e:
-            print("Couldn't get font: %s"%e)
+            logger.error("Couldn't get font: %s"%e)
         self.ini.setLineWrapMode(QTextEdit.NoWrap)
         self.ini.setText(text)
         self.ini.setReadOnly(True)

calibre-plugin/dialogs.py

@@ -7,9 +7,15 @@ __license__   = 'GPL v3'
 __copyright__ = '2011, Jim Miller'
 __docformat__ = 'restructuredtext en'
 
+import logging
+logger = logging.getLogger(__name__)
+
 import traceback, re
 from functools import partial
 
+import logging
+logger = logging.getLogger(__name__)
+
 import urllib
 import email
 
@@ -207,7 +213,7 @@ class AddNewDialog(SizePersistedDialog):
         self.groupbox.setCheckable(True)
         self.groupbox.setChecked(False)
         self.groupbox.setFlat(True)
-        print("style:%s"%self.groupbox.styleSheet())
+        #print("style:%s"%self.groupbox.styleSheet())
         self.groupbox.setStyleSheet(gpstyle)
 
         self.gbf = QFrame()
@@ -537,7 +543,7 @@ class LoopProgressDialog(QProgressDialog):
     def updateStatus(self):
         self.setLabelText("%s %d of %d"%(self.status_prefix,self.i+1,len(self.book_list)))
         self.setValue(self.i+1)
-        print(self.labelText())
+        #print(self.labelText())
 
     def do_loop(self):
 
@@ -558,7 +564,7 @@ class LoopProgressDialog(QProgressDialog):
         except Exception as e:
             book['good']=False
             book['comment']=unicode(e)
-            print("Exception: %s:%s"%(book,unicode(e)))
+            logger.error("Exception: %s:%s"%(book,unicode(e)))
             traceback.print_exc()
             
         self.updateStatus()

calibre-plugin/ffdl_plugin.py

@@ -7,6 +7,9 @@ __license__   = 'GPL v3'
 __copyright__ = '2012, Jim Miller'
 __docformat__ = 'restructuredtext en'
 
+import logging
+logger = logging.getLogger(__name__)
+
 import time, os, copy, threading, re, platform, sys
 from StringIO import StringIO
 from functools import partial
@@ -75,7 +78,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
     # (text, icon_path, tooltip, keyboard shortcut)
     # icon_path isn't in the zip--icon loaded below.
     action_spec = (name, None,
-                   'Download FanFiction stories from various web sites', ())
+                   _('Download FanFiction stories from various web sites'), ())
     # None for keyboard shortcut doesn't allow shortcut.  () does, there just isn't one yet
 
     action_type = 'global'
@@ -105,7 +108,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         # are not found in the zip file will result in null QIcons.
         icon = get_icon('images/icon.png')
 
-        self.qaction.setText('FanFictionDL')
+        self.qaction.setText(_('FanFictionDL'))
         
         # The qaction is automatically created from the action_spec defined
         # above
@@ -143,7 +146,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
             file_path = os.path.join(calibre_config_dir,
                                      *("plugins/fanfictiondownloader_macmenuhack.txt".split('/')))
             file_path = os.path.abspath(file_path)
-            print("Plugin %s macmenuhack file_path:%s"%(self.name,file_path))
+            logger.debug("Plugin %s macmenuhack file_path:%s"%(self.name,file_path))
             self.macmenuhack = os.access(file_path, os.F_OK)
             return self.macmenuhack
 
@@ -387,7 +390,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                         show_copy_button=False)
 
     def get_urls_from_page(self,url):
-        print("get_urls_from_page URL:%s"%url)
+        logger.debug("get_urls_from_page URL:%s"%url)
         if 'archiveofourown.org' in url:
             configuration = get_ffdl_config(url)
         else:
@@ -541,7 +544,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
             return
             
         tdir = PersistentTemporaryDirectory(prefix='ffdl_anthology_')
-        print("tdir:\n%s"%tdir)
+        logger.debug("tdir:\n%s"%tdir)
 
         bookepubio = StringIO(db.format(book_id,'EPUB',index_is_id=True))
         
@@ -617,7 +620,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
             '''%(len(urlmapfile),"</li><li>".join(urlmapfile.keys()))
             if not question_dialog(self.gui, 'Stories Removed',
                                text, show_copy_button=False):
-                print("Canceling anthology update due to removed stories.")
+                logger.debug("Canceling anthology update due to removed stories.")
                 return
 
         # Now that we've 
@@ -696,7 +699,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
             books = self.convert_urls_to_books(url_list)
 
         options['version'] = self.version
-        print(self.version)
+        logger.debug(self.version)
         
         #print("prep_downloads:%s"%books)
 
@@ -706,7 +709,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
             options['tdir']=tdir
 
         if 0 < len(filter(lambda x : x['good'], books)):
-            self.gui.status_bar.show_message(_('Started fetching metadata for %s stories.'%len(books)), 3000)
+            self.gui.status_bar.show_message(_('Started fetching metadata for %s stories.')%len(books), 3000)
             LoopProgressDialog(self.gui,
                                books,
                                partial(self.prep_download_loop, options = options, merge=merge),
@@ -731,7 +734,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         '''
 
         url = book['url']
-        print("url:%s"%url)
+        logger.debug("url:%s"%url)
         mi = None
 
         if not merge: # skip reject list when merging.
@@ -787,7 +790,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
             try:
                 adapter.getStoryMetadataOnly()
             except exceptions.FailedToLogin, f:
-                print("Login Failed, Need Username/Password.")
+                logger.warn("Login Failed, Need Username/Password.")
                 userpass = UserPassDialog(self.gui,url,f)
                 userpass.exec_() # exec_ will make it act modal
                 if userpass.status:
@@ -869,14 +872,14 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
             
             if book['calibre_id'] != None:
                 # updating an existing book.  Update mode applies.
-                print("update existing id:%s"%book['calibre_id'])
+                logger.debug("update existing id:%s"%book['calibre_id'])
                 book_id = book['calibre_id']
                 # No handling needed: OVERWRITEALWAYS,CALIBREONLY
                 
             # only care about collisions when not ADDNEW
             elif collision != ADDNEW:
                 # 'new' book from URL.  collision handling applies.
-                print("from URL(%s)"%url)
+                logger.debug("from URL(%s)"%url)
     
                 # try to find by identifier url or uri first.
                 searchstr = 'identifiers:"~ur(i|l):=%s"'%url.replace(":","|")
@@ -889,16 +892,16 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                         ## meantime, if it matches the title *and* first
                         ## 100 authors, I'm prepared to assume it's a
                         ## match.
-                        print("reduce author list to 100 only when calibre < 0.8.61")
+                        logger.debug("reduce author list to 100 only when calibre < 0.8.61")
                         authlist = authlist[:100]
                     mi = MetaInformation(story.getMetadata("title", removeallentities=True),
                                          authlist)
                     identicalbooks = db.find_identical_books(mi)
                     if len(identicalbooks) > 0:
-                        print("existing found by title/author(s)")
+                        logger.debug("existing found by title/author(s)")
     
                 else:
-                    print("existing found by identifier URL")
+                    logger.debug("existing found by identifier URL")
                     
                 if collision == SKIP and identicalbooks:
                     raise NotGoingToDownload("Skipping duplicate story.","list_remove.png")
@@ -997,7 +1000,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                                                   suffix='.epub',
                                                   dir=options['tdir'])
                     db.copy_format_to(book_id,fileform,tmp,index_is_id=True)
-                    print("existing epub tmp:"+tmp.name)
+                    logger.debug("existing epub tmp:"+tmp.name)
                     book['epub_for_update'] = tmp.name
 
             if book_id != None and prefs['injectseries']:
@@ -1014,8 +1017,8 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
             tmp = PersistentTemporaryFile(prefix=story.formatFileName("${title}-${author}-",allowunsafefilename=False)[:100],
                                           suffix='.'+options['fileform'],
                                           dir=options['tdir'])
-            print("title:"+book['title'])
-            print("outfile:"+tmp.name)
+            logger.debug("title:"+book['title'])
+            logger.debug("outfile:"+tmp.name)
             book['outfile'] = tmp.name            
                 
         return
@@ -1093,7 +1096,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         if book['calibre_id'] and prefs['errorcol'] != '' and prefs['errorcol'] in custom_columns:
             label = custom_columns[prefs['errorcol']]['label']
             if not book['good']:
-                print("record/update error message column %s %s"%(book['title'],book['url']))
+                logger.debug("record/update error message column %s %s"%(book['title'],book['url']))
                 db.set_custom(book['calibre_id'], book['comment'], label=label, commit=True) # book['comment']
             else:
                 db.set_custom(book['calibre_id'], '', label=label, commit=True) # book['comment']
@@ -1101,7 +1104,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         if not book['good']:
             return # only update errorcol on error.
         
-        print("add/update %s %s"%(book['title'],book['url']))
+        logger.debug("add/update %s %s"%(book['title'],book['url']))
         mi = self.make_mi_from_book(book)
                 
         if options['collision'] != CALIBREONLY:
@@ -1113,7 +1116,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                 self.update_metadata(db, book['calibre_id'], book, mi, options)
             except:
                 det_msg = "".join(traceback.format_exception(*sys.exc_info()))+"\nStory Details:\n%s"%pretty_book(book)
-                print("Error Updating Metadata:\n%s"%det_msg)
+                logger.error("Error Updating Metadata:\n%s"%det_msg)
                 error_dialog(self.gui,
                              "Error Updating Metadata",
                              "<p>An error has occurred while FFDL was updating calibre's metadata for <a href='%s'>%s</a>.</p>"%(book['url'],book['title'])+
@@ -1124,7 +1127,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
     def update_books_finish(self, book_list, options={}, showlist=True):
         '''Notify calibre about updated rows, update external plugins
         (Reading Lists & Count Pages) as configured'''
-        
+
         add_list = filter(lambda x : x['good'] and x['added'], book_list)
         add_ids = [ x['calibre_id'] for x in add_list ]
         update_list = filter(lambda x : x['good'] and not x['added'], book_list)
@@ -1152,7 +1155,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         if self.gui.cover_flow:
             self.gui.cover_flow.dataChanged()
 
-        if showlist: # don't use with anthology
+        if showlist and prefs['mark']: # don't use with anthology
             db = self.gui.current_db
             marked_ids = dict()
             marked_text = "ffdl_success"
@@ -1169,20 +1172,8 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                 # Sort by our marked column to display the books in order
                 self.gui.library_view.sort_by_named_field('marked', True)
                 
-        self.gui.status_bar.show_message(_('Finished Adding/Updating %d books.'%(len(update_list) + len(add_list))), 3000)
-
-        1+1
-        
-        print("all done, remove temp dir.")
-
-        1+1
-        
+        self.gui.status_bar.show_message(_('Finished Adding/Updating %d books.')%(len(update_list) + len(add_list)), 3000)
         remove_dir(options['tdir'])
-
-        1+1
-        
-        print("removed temp dir.")
-
         
         if 'Count Pages' in self.gui.iactions and len(prefs['countpagesstats']) and len(all_ids):
             cp_plugin = self.gui.iactions['Count Pages']
@@ -1277,12 +1268,12 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         (good_list,bad_list,options) = payload
         total_good = len(good_list)
 
-        print("merge titles:\n%s"%"\n".join([ "%s %s"%(x['title'],x['listorder']) for x in good_list ]))
+        logger.debug("merge titles:\n%s"%"\n".join([ "%s %s"%(x['title'],x['listorder']) for x in good_list ]))
 
         good_list = sorted(good_list,key=lambda x : x['listorder'])
         bad_list = sorted(bad_list,key=lambda x : x['listorder'])
         
-        self.gui.status_bar.show_message(_('Merging %s books.'%total_good))
+        self.gui.status_bar.show_message(_('Merging %s books.')%total_good)
 
         
         existingbook = None
@@ -1303,8 +1294,8 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
             # if still 'good', make a temp file to write the output to.
             tmp = PersistentTemporaryFile(suffix='.'+options['fileform'],
                                           dir=options['tdir'])
-            print("title:"+mergebook['title'])
-            print("outfile:"+tmp.name)
+            logger.debug("title:"+mergebook['title'])
+            logger.debug("outfile:"+tmp.name)
             mergebook['outfile'] = tmp.name
 
         self.get_epubmerge_plugin().do_merge(tmp.name,
@@ -1325,7 +1316,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
 
         self.gui.status_bar.show_message(_('FFDL Adding/Updating books.'))
 
-        if good_list or (bad_list and prefs['errorcol'] != '' and prefs['errorcol'] in self.gui.library_view.model().custom_columns):
+        if good_list or prefs['mark'] or (bad_list and prefs['errorcol'] != '' and prefs['errorcol'] in self.gui.library_view.model().custom_columns):
             LoopProgressDialog(self.gui,
                                good_list+bad_list,
                                partial(self.update_books_loop, options=options, db=self.gui.current_db),
@@ -1338,22 +1329,24 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         '''Update custom error column if configured.'''
         (empty_list,book_list,options)=payload
         custom_columns = self.gui.library_view.model().custom_columns
-        if prefs['errorcol'] != '' and prefs['errorcol'] in custom_columns:
+        if prefs['mark'] or (prefs['errorcol'] != '' and prefs['errorcol'] in custom_columns):
             self.previous = self.gui.library_view.currentIndex() # used by update_books_finish.
-            self.gui.status_bar.show_message(_('Adding/Updating %s BAD books.'%len(book_list)))
+            self.gui.status_bar.show_message(_('Adding/Updating %s BAD books.')%len(book_list))
             label = custom_columns[prefs['errorcol']]['label']
             LoopProgressDialog(self.gui,
                                book_list,
                                partial(self.update_error_column_loop, db=self.gui.current_db, label=label),
-                               partial(self.update_books_finish, options=options, showlist=False),
+                               partial(self.update_books_finish, options=options),
                                init_label="Updating calibre for BAD FanFiction stories...",
                                win_title="Update calibre for BAD FanFiction stories",
                                status_prefix="Updated")
 
     def update_error_column_loop(self,book,db=None,label='errorcol'):
         if book['calibre_id']:
-            print("add/update bad %s %s %s"%(book['title'],book['url'],book['comment']))
-            db.set_custom(book['calibre_id'], book['comment'], label=label, commit=True)
+            custom_columns = self.gui.library_view.model().custom_columns
+            if (prefs['errorcol'] != '' and prefs['errorcol'] in custom_columns):
+                logger.debug("add/update bad %s %s %s"%(book['title'],book['url'],book['comment']))
+                db.set_custom(book['calibre_id'], book['comment'], label=label, commit=True)
 
     def add_book_or_update_format(self,book,options,prefs,mi=None):
         db = self.gui.current_db
@@ -1382,7 +1375,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
             fmts = db.formats(book['calibre_id'], index_is_id=True).split(',')
             for fmt in fmts:
                 if fmt != formmapping[options['fileform']]:
-                    print("remove f:"+fmt)
+                    logger.debug("remove f:"+fmt)
                     db.remove_format(book['calibre_id'], fmt, index_is_id=True)#, notify=False
 
         return book_id
@@ -1423,7 +1416,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                 try:
                     db.set_cover(book_id, epubmi.cover_data[1])
                 except:
-                    print("Failed to set_cover, skipping")
+                    logger.info("Failed to set_cover, skipping")
 
         # implement 'newonly' flags here by setting to the current
         # value again.
@@ -1436,7 +1429,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                         try:
                             mi.__setattr__(col,oldmi.__getattribute__(col))
                         except AttributeError:
-                            print("AttributeError? %s"%col)
+                            logger.warn("AttributeError? %s"%col)
                             pass
 
         db.set_metadata(book_id,mi)
@@ -1451,25 +1444,26 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
         for col, meta in prefs['custom_cols'].iteritems():
             #print("setting %s to %s"%(col,meta))
             if col not in custom_columns:
-                print("%s not an existing column, skipping."%col)
+                logger.debug("%s not an existing column, skipping."%col)
                 continue
             coldef = custom_columns[col]
             if col in prefs['custom_cols_newonly'] and prefs['custom_cols_newonly'][col] and not book['added']:
-                print("Skipping custom column(%s) update, set to New Books Only"%coldef['name'])
+                logger.debug("Skipping custom column(%s) update, set to New Books Only"%coldef['name'])
                 continue
             if not meta.startswith('status-') and meta not in book['all_metadata'] or \
                     meta.startswith('status-') and 'status' not in book['all_metadata']:
-                print("No value for %s, skipping custom column(%s) update."%(meta,coldef['name']))
+                logger.debug("No value for %s, skipping custom column(%s) update."%(meta,coldef['name']))
                 continue
             if meta not in permitted_values[coldef['datatype']]:
-                print("%s not a valid column type for %s, skipping."%(col,meta))
+                logger.debug("%s not a valid column type for %s, skipping."%(col,meta))
                 continue
             label = coldef['label']
             if coldef['datatype'] in ('enumeration','text','comments','datetime','series'):
                 db.set_custom(book_id, book['all_metadata'][meta], label, commit=False)
             elif coldef['datatype'] in ('int','float'):
                 num = unicode(book['all_metadata'][meta]).replace(",","")
-                db.set_custom(book_id, num, label=label, commit=False)
+                if num != '':
+                    db.set_custom(book_id, num, label=label, commit=False)
             elif coldef['datatype'] == 'bool' and meta.startswith('status-'):
                 if meta == 'status-C':
                     val = book['all_metadata']['status'] == 'Completed'
@@ -1490,7 +1484,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                         (custcol,flag) = map( lambda x: x.strip(), custcol.split(",") )
 
                     if meta not in book['all_metadata']:
-                        print("No value for %s, skipping custom column(%s) update."%(meta,custcol))
+                        logger.debug("No value for %s, skipping custom column(%s) update."%(meta,custcol))
                         continue
                     
                     if custcol not in custom_columns:
@@ -1508,7 +1502,8 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                                 val = unicode(book['all_metadata'][meta]).replace(",","")
                         else:
                             val = book['all_metadata'][meta]
-                        db.set_custom(book_id, val, label=label, commit=False)
+                        if val != '':
+                            db.set_custom(book_id, val, label=label, commit=False)
 
                     if flag == 'a':
                         vallist = []
@@ -1577,9 +1572,9 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                             break
 
                 if setting_name:
-                    print("Generate Cover Setting from generate_cover_settings(%s)"%line)
+                    logger.debug("Generate Cover Setting from generate_cover_settings(%s)"%line)
                     if setting_name not in gc_plugin.get_saved_setting_names():
-                        print("GC Name %s not found, discarding! (check personal.ini for typos)"%setting_name)
+                        logger.info("GC Name %s not found, discarding! (check personal.ini for typos)"%setting_name)
                         setting_name = None
                 
             if not setting_name and book['all_metadata']['site'] in prefs['gc_site_settings']:
@@ -1589,7 +1584,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                 setting_name =  prefs['gc_site_settings']['Default']
                 
             if setting_name:
-                print("Running Generate Cover with settings %s."%setting_name)
+                logger.debug("Running Generate Cover with settings %s."%setting_name)
                 realmi = db.get_metadata(book_id, index_is_id=True)
                 gc_plugin.generate_cover_for_book(realmi,saved_setting_name=setting_name)
 
@@ -1907,7 +1902,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                         break
             
             configuration = get_ffdl_config(book['url'],fileform)
-            print("anthology_title_pattern:%s"%configuration.getConfig('anthology_title_pattern'))
+            logger.debug("anthology_title_pattern:%s"%configuration.getConfig('anthology_title_pattern'))
             if configuration.getConfig('anthology_title_pattern'):
                 tmplt = Template(configuration.getConfig('anthology_title_pattern'))
                 book['title'] = tmplt.safe_substitute({'title':book['title']}).encode('utf8')

calibre-plugin/jobs.py

@@ -8,6 +8,9 @@ __copyright__ = '2012, Jim Miller'
 __copyright__ = '2011, Grant Drake <grant.drake@gmail.com>'
 __docformat__ = 'restructuredtext en'
 
+import logging
+logger = logging.getLogger(__name__)
+
 import time, os, traceback
 
 from StringIO import StringIO
@@ -38,13 +41,13 @@ def do_download_worker(book_list, options,
     '''
     server = Server(pool_size=cpus)
 
-    print(options['version'])
+    logger.info(options['version'])
     total = 0
     alreadybad = []
     # Queue all the jobs
-    print("Adding jobs for URLs:")
+    logger.info("Adding jobs for URLs:")
     for book in book_list:
-        print("%s"%book['url'])
+        logger.info("%s"%book['url'])
         if book['good']:
             total += 1
             args = ['calibre_plugins.fanfictiondownloader_plugin.jobs',
@@ -87,19 +90,19 @@ def do_download_worker(book_list, options,
         count = count + 1
         notification(float(count)/total, '%d of %d stories finished downloading'%(count,total))
         # Add this job's output to the current log
-        print('Logfile for book ID %s (%s)'%(book_id, job._book['title']))
-        print(job.details)
+        logger.info('Logfile for book ID %s (%s)'%(book_id, job._book['title']))
+        logger.info(job.details)
 
         if count >= total:
             # All done!  Output some lists for convenience of some users.
-            print("Successfully downloaded:")
+            logger.info("Successfully downloaded:")
             for book in book_list:
                 if book['good']:
-                    print("%s %s"%(book['title'],book['url']))
-            print("\nUnsuccessful:")
+                    logger.info("%s %s"%(book['title'],book['url']))
+            logger.info("\nUnsuccessful:")
             for book in book_list:
                 if not book['good']:
-                    print("%s %s"%(book['title'],book['url']))
+                    logger.info("%s %s"%(book['title'],book['url']))
             break
 
     server.close()
@@ -147,7 +150,7 @@ def do_download_for_worker(book,options,notification=lambda x,y:x):
 
         ## No need to download at all.  Shouldn't ever get down here.
         if options['collision'] in (CALIBREONLY):
-            print("Skipping CALIBREONLY 'update' down inside worker--this shouldn't be happening...")
+            logger.info("Skipping CALIBREONLY 'update' down inside worker--this shouldn't be happening...")
             book['comment'] = 'Metadata collected.'
             
         ## checks were done earlier, it's new or not dup or newer--just write it.
@@ -170,7 +173,7 @@ def do_download_for_worker(book,options,notification=lambda x,y:x):
                 if adapter.logfile:
                     adapter.logfile = adapter.logfile.replace("span id","span notid")
             
-            print("write to %s"%outfile)
+            logger.info("write to %s"%outfile)
             writer.writeStory(outfilename=outfile, forceOverwrite=True)
             book['comment'] = 'Download %s completed, %s chapters.'%(options['fileform'],story.getMetadata("numChapters"))
             
@@ -198,9 +201,13 @@ def do_download_for_worker(book,options,notification=lambda x,y:x):
             # dup handling from ffdl_plugin needed for anthology updates.
             if chaptercount > urlchaptercount:
                 raise NotGoingToDownload("Existing epub contains %d chapters, web site only has %d. Use Overwrite to force update." % (chaptercount,urlchaptercount),'dialog_error.png')
-                            
-            print("Do update - epub(%d) vs url(%d)" % (chaptercount, urlchaptercount))
-            print("write to %s"%outfile)
+
+            if not (options['collision'] == UPDATEALWAYS and chaptercount == urlchaptercount) \
+                    and adapter.getConfig("do_update_hook"):
+                chaptercount = adapter.hookForUpdates(chaptercount)
+
+            logger.info("Do update - epub(%d) vs url(%d)" % (chaptercount, urlchaptercount))
+            logger.info("write to %s"%outfile)
 
             writer.writeStory(outfilename=outfile, forceOverwrite=True)
             
@@ -222,7 +229,7 @@ def do_download_for_worker(book,options,notification=lambda x,y:x):
             
             log = Log(level=Log.DEBUG)
             # report = []
-            polish({outfile:outfile}, opts, log, print) # report.append
+            polish({outfile:outfile}, opts, log, logger.info) # report.append
         
     except NotGoingToDownload as d:
         book['good']=False
@@ -234,7 +241,7 @@ def do_download_for_worker(book,options,notification=lambda x,y:x):
         book['comment']=unicode(e)
         book['icon']='dialog_error.png'
         book['status'] = 'Error'
-        print("Exception: %s:%s"%(book,unicode(e)))
+        logger.info("Exception: %s:%s"%(book,unicode(e)))
         traceback.print_exc()
         
     #time.sleep(10)

calibre-plugin/prefs.py

@@ -32,6 +32,7 @@ default_prefs['updateepubcover'] = False
 default_prefs['keeptags'] = False
 default_prefs['suppressauthorsort'] = False
 default_prefs['suppresstitlesort'] = False
+default_prefs['mark'] = False
 default_prefs['showmarked'] = False
 default_prefs['urlsfromclip'] = True
 default_prefs['updatedefault'] = True

defaults.ini

@@ -170,6 +170,14 @@ extratags: FanFiction
 ## Add this to genre if there's more than one category.
 #add_genre_when_multi_category: Crossover
 
+## default_value_(entry) can be used to set the value for a metadata
+## entry when no value has been found on the site.  For example, some
+## sites doesn't have a status metadatum.  If uncommented, this will
+## use 'Unknown' for status when no status is found.
+#default_value_status:Unknown
+## Can also be used for other metadata values
+#default_value_category:FanFiction
+
 ## number of seconds to sleep between calls to the story site.  May by
 ## useful if pulling large numbers of stories or if the site is slow.
 #slow_down_sleep_time:0.5
@@ -212,6 +220,11 @@ extratags: FanFiction
 ## doesn't work on some devices either.)
 #replace_hr: false
 
+## Some sites/authors/stories use br tags instead of p tags for
+## paragraphs.  This feature uses some heuristics to find and replace
+## br paragraphs with p tags while preserving scene breaks.
+#replace_br_with_p: false
+
 ## If set false, the summary will have all html stripped.
 ## Both this and include_images must be true to get images in the
 ## summary.
@@ -278,7 +291,7 @@ sort_ships:false
 #keep_in_order_author:true
 
 ## User-agent
-#user_agent:FFDL/1.7
+user_agent:FFDL/1.7
 
 ## Each output format has a section that overrides [defaults]
 [html]
@@ -760,6 +773,23 @@ extraships:Harry Potter/Hermione Granger
 #username:YourName
 #password:yourpassword
 
+[fictionpad.com]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+extra_valid_entries:followers,comments,views,likes,dislikes
+#extra_titlepage_entries:followers,comments,views,likes,dislikes
+
+followers_label:Followers
+comments_label:Comments
+views_label:Views
+likes_label:Likes
+dislikes_label:Dislikes
+
 [finestories.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -768,6 +798,24 @@ extraships:Harry Potter/Hermione Granger
 #username:YourName
 #password:yourpassword
 
+[storiesonline.net]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Clear FanFiction from defaults, site is original fiction.
+extratags:
+
+extra_valid_entries:size,universe,codes
+#extra_titlepage_entries:size,universe,codes
+
+size_label:Size
+universe_label:Universe
+codes_label:Codes
+
 [grangerenchanted.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter
@@ -810,6 +858,10 @@ extracategories:Highlander
 ## Site dedicated to these categories/characters/ships
 extracategories:In Death
 
+## some sites include images that we don't ever want becoming the
+## cover image.  This lets you exclude them.
+cover_exclusion_regexp:/public/style_emoticons/.*
+
 [ksarchive.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:Star Trek
@@ -1117,6 +1169,7 @@ context_label:Context
 type_label:Type of Couple
 
 [www.fanfiction.net]
+user_agent:
 ## fanfiction.net's 'cover' images are really just tiny thumbnails.
 ## Change this to false to use them anyway.
 never_make_cover: true
@@ -1125,6 +1178,10 @@ never_make_cover: true
 ## for examples of how to use them.
 extra_valid_entries:reviews,favs,follows
 
+## ffnet uses 'Pairings', not 'Relationship', stating they don't have
+## to be romantic pairings.
+ships_label:Pairings
+
 [www.fanfiktion.de]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -1147,7 +1204,12 @@ extracategories:Harry Potter
 ## fictionally.org storyIds are not unique.  Combine with authorId.
 output_filename: ${title}-${siteabbrev}_${authorId}_${storyId}${formatext}
 
+## fictionalley.org doesn't have a status metadatum.  If uncommented,
+## this will be used for status.
+#default_value_status:Unknown
+
 [www.fictionpress.com]
+user_agent:
 ## Clear FanFiction from defaults, fictionpress.com is original fiction.
 extratags:
 
@@ -1169,12 +1231,13 @@ extracategories:My Little Pony: Friendship is Magic
 
 ## Extra metadata that this adapter knows about.  See [dramione.org]
 ## for examples of how to use them.
-extra_valid_entries:likes,dislikes,views,total_views,short_description
+extra_valid_entries:likes,dislikes,views,total_views,short_description,groups
 likes_label:Likes
 dislikes_label:Dislikes
 views_label:Highest Single Chapter Views
 total_views_label:Total Views
 short_description_label:Short Summary
+groups_label:Groups
 
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
@@ -1186,6 +1249,13 @@ short_description_label:Short Summary
 ## when a password is required rather than prompting every time.
 #fail_on_password: false
 
+## fimfiction.net stories allow chapters to be added out of order.  So
+## the newest chapter may not be the last one.  FFDL update doesn't
+## like that.  If do_update_hook is uncommented and set true, the
+## adapter will discard all existing chapters from the newest one on
+## when updating to enforce accurate chapters.
+#do_update_hook:false
+
 [www.harrypotterfanfiction.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter

downloader.py

@@ -270,7 +270,6 @@ def main(argv,
            elif chaptercount == 0:
                print "%s doesn't contain any recognizable chapters, probably from a different source.  Not updating." % (output_filename)
            else:
-               print "Do update - epub(%d) vs url(%d)" % (chaptercount, urlchaptercount)
                if not options.metaonly:
 
                    # update now handled by pre-populating the old
@@ -284,6 +283,12 @@ def main(argv,
                     adapter.calibrebookmark,
                     adapter.logfile) = get_update_data(output_filename)
 
+                   print "Do update - epub(%d) vs url(%d)" % (chaptercount, urlchaptercount)
+
+                   if not (options.update and chaptercount == urlchaptercount) \
+                           and adapter.getConfig("do_update_hook"):
+                       chaptercount = adapter.hookForUpdates(chaptercount)
+
                    writeStory(configuration,adapter,"epub")
                    
        else:

fanficdownloader/__init__.py

@@ -4,13 +4,16 @@ try:
     # just a way to switch between web service and CLI/PI
     import google.appengine.api 
 except:
-    import sys
-    if sys.version_info >= (2, 7):
-        import logging
-        logger = logging.getLogger(__name__)
-        loghandler=logging.StreamHandler()
-        loghandler.setFormatter(logging.Formatter("FFDL:%(levelname)s:%(filename)s(%(lineno)d):%(message)s"))
-        logger.addHandler(loghandler)
-        loghandler.setLevel(logging.DEBUG)
-        logger.setLevel(logging.DEBUG)
+    try: # just a way to switch between CLI and PI
+        import calibre.constants
+    except:
+        import sys
+        if sys.version_info >= (2, 7):
+            import logging
+            logger = logging.getLogger(__name__)
+            loghandler=logging.StreamHandler()
+            loghandler.setFormatter(logging.Formatter("FFDL:%(levelname)s:%(filename)s(%(lineno)d):%(message)s"))
+            logger.addHandler(loghandler)
+            loghandler.setLevel(logging.DEBUG)
+            logger.setLevel(logging.DEBUG)
 

fanficdownloader/adapters/__init__.py

@@ -119,6 +119,8 @@ import adapter_nickandgregnet
 import adapter_potterheadsanonymouscom
 import adapter_simplyundeniablecom
 import adapter_scarheadnet
+import adapter_fictionpadcom
+import adapter_storiesonlinenet
 
 ## This bit of complexity allows adapters to be added by just adding
 ## importing.  It eliminates the long if/else clauses we used to need

fanficdownloader/adapters/adapter_archiveofourownorg.py

@@ -303,7 +303,7 @@ class ArchiveOfOurOwnOrgAdapter(BaseSiteAdapter):
     def getChapterText(self, url):
         logger.debug('Getting chapter text from: %s' % url)
 		
-        chapter=bs.BeautifulSoup('<div class="story"></div>')
+        chapter=bs.BeautifulSoup('<div class="story"></div>').find('div')
         data = self._fetchUrl(url)
         soup = bs.BeautifulSoup(data,selfClosingTags=('br','hr'))
 		

fanficdownloader/adapters/adapter_darksolaceorg.py

@@ -94,11 +94,10 @@ class DarkSolaceOrgAdapter(BaseSiteAdapter):
         else:
             params['penname'] = self.getConfig("username")
             params['password'] = self.getConfig("password")
-        params['rememberme'] = '1'
         params['action'] = 'login'
         params['submit'] = 'Submit'
     
-        loginUrl = 'http://' + self.getSiteDomain() + '/elysian/user.php'
+        loginUrl = 'http://www.' + self.getSiteDomain() + '/elysian/user.php'
         logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
                                                               params['penname']))
     

fanficdownloader/adapters/adapter_fanfictionnet.py

@@ -165,7 +165,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
 
         # after Rating, the same bit of text containing id:123456 contains
         # Complete--if completed.
-        gui_table1i = soup.find('table',{'cellpadding':'5'})
+        gui_table1i = soup.find('div',{'id':'content_wrapper_inner'})
 
         self.story.setMetadata('title', stripHTML(gui_table1i.find('b'))) # title appears to be only(or at least first) bold tag in gui_table1i
 
@@ -222,14 +222,18 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
             elif  metalist[0].startswith('Words'):
                 self.story.setMetadata('numWords',metalist[0].split(':')[1].strip())
             elif not donechars:
-                self.story.extendList('characters',metalist[0].split('&'))
+                # with 'pairing' support, pairings are bracketed w/o comma after
+                # [Caspian X, Lucy Pevensie] Edmund Pevensie, Peter Pevensie
+                self.story.extendList('characters',metalist[0].replace('[','').replace(']',',').split(','))
+
+                l = metalist[0]
+                while '[' in l:
+                    self.story.addToList('ships',l[l.index('[')+1:l.index(']')].replace(', ','/'))
+                    l = l[l.index(']')+1:]
+
                 donechars = True
             metalist=metalist[1:]
 
-        # next might be characters, otherwise Reviews, Updated, Published, Words
-        # if not ( metalist[0].startswith('Reviews') or metalist[0].startswith('Updated') or metalist[0].startswith('Published') or metalist[0].startswith('Words') or metalist[0].startswith('Chapters') ):
-        #     self.story.extendList('characters',metalist[0].split('&'))
-        
         if 'Status: Complete' in metatext:
             self.story.setMetadata('status', 'Completed')
         else:
@@ -281,10 +285,11 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
         # don't care about anything before "<div class='storytextp"
         # (there's a space after storytextp, so no close quote(')) and
         # this kills any body tags.
-        if "<div class='storytextp" not in data:
+        divstr = "<div role='main'"
+        if divstr not in data:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
         else:
-            data = data[data.index("<div class='storytextp"):]
+            data = data[data.index(divstr):]
         data.replace("<body","<notbody").replace("<BODY","<NOTBODY")
         
         soup = bs.BeautifulSoup(data)

fanficdownloader/adapters/adapter_fictionpadcom.py

@@ -0,0 +1,192 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+import time
+import json
+
+from .. import BeautifulSoup as bs
+#from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+class FictionPadSiteAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+        self.story.setMetadata('siteabbrev','fpad')
+        self.dateformat = "%Y-%m-%dT%H:%M:%SZ"
+        self.is_adult=False
+        self.username = None
+        self.password = None
+        # get storyId from url--url validation guarantees query correct
+        m = re.match(self.getSiteURLPattern(),url)
+        if m:
+            self.story.setMetadata('storyId',m.group('id'))
+            
+            # normalized story URL.
+            self._setURL("https://"+self.getSiteDomain()
+                         +"/author/"+m.group('author')
+                         +"/stories/"+self.story.getMetadata('storyId'))
+        else:
+            raise exceptions.InvalidStoryURL(url,
+                                             self.getSiteDomain(),
+                                             self.getSiteExampleURLs())
+
+    @staticmethod
+    def getSiteDomain():
+        return 'fictionpad.com'
+
+    @classmethod
+    def getSiteExampleURLs(self):
+        return "https://fictionpad.com/author/Author/stories/1234/Some-Title"
+    
+    def getSiteURLPattern(self):
+        # http://fictionpad.com/author/Serdd/stories/4275
+        return r"http(s)?://(www\.)?fictionpad\.com/author/(?P<author>[^/]+)/stories/(?P<id>\d+)"
+
+# <form method="post" action="/signin">
+#     <input name="authenticity_token" type="hidden" value="u+cfdXh46dRnwVnSlmE2B2BFmHgu760paqgBG6KQeos=" />
+#     <input type="hidden" name="remember" value="1">
+#     <strong class="help-start text-center">or with FictionPad</strong>
+#     <label class="control-label hidden-placeholder">Pseudonym or Email Address</label>
+#     <input name="login" class="input-block-level" type="text" placeholder="Pseudonym or Email Address" maxlength="50" required autofocus>
+#     <label class="control-label hidden-placeholder">Password</label>
+#     <input name="password" class="input-block-level" type="password" placeholder="Password" minlength="6" required>
+#     <button type="submit" class="btn btn-primary btn-block">Sign In</button>
+#     <p class="help-end">
+#         <a href="/passwordreset">Forgot your password?</a>
+#     </p>
+# </form>
+    def performLogin(self):
+        params = {}
+
+        if self.password:
+            params['login'] = self.username
+            params['password'] = self.password
+        else:
+            params['login'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['remember'] = '1'
+
+        loginUrl = 'http://' + self.getSiteDomain() + '/signin'
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['login']))
+
+        ## need to pull empty login page first to get authenticity_token
+        soup = bs.BeautifulSoup(self._fetchUrl(loginUrl))
+        params['authenticity_token']=soup.find('input', {'name':'authenticity_token'})['value']
+        
+        data = self._postUrl(loginUrl, params)
+
+        if "Invalid email/pseudonym and password combination." in data:
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['login']))
+            raise exceptions.FailedToLogin(loginUrl,params['login'])
+            
+
+    def extractChapterUrlsAndMetadata(self):
+        # fetch the chapter.  From that we will get almost all the
+        # metadata and chapter list
+
+        url=self.url
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url)
+            if "This is a mature story.  Please sign in to read it." in data:
+                self.performLogin()
+                data = self._fetchUrl(url)
+            
+            find = "wordyarn.config.page = "
+            data = data[data.index(find)+len(find):]
+            data = data[:data.index("</script>")]
+            data = data[:data.rindex(";")]
+            data = data.replace('tables:','"tables":')
+            tables = json.loads(data)['tables']
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(url)
+            else:
+                raise e
+
+        # looks like only one author per story allowed.
+        author = tables['users'][0]
+        story = tables['stories'][0]
+        story_ver = tables['story_versions'][0]
+        
+        self.story.setMetadata('authorId',author['id'])
+        self.story.setMetadata('author',author['display_name'])
+        self.story.setMetadata('authorUrl','https://'+self.host+'/author/'+author['display_name']+'/stories')
+
+        self.story.setMetadata('title',story_ver['title'])
+        self.setDescription(url,story_ver['description'])
+        
+        if not ('assets/story_versions/covers' in story_ver['profile_image_url@2x']):
+            self.setCoverImage(url,story_ver['profile_image_url@2x'])
+
+        self.story.setMetadata('datePublished',makeDate(story['published_at'], self.dateformat))
+        self.story.setMetadata('dateUpdated',makeDate(story['published_at'], self.dateformat))
+
+        self.story.setMetadata('followers',story['followers_count'])
+        self.story.setMetadata('comments',story['comments_count'])
+        self.story.setMetadata('views',story['views_count'])
+        self.story.setMetadata('likes',int(story['likes'])) # no idea why they floated these.
+        self.story.setMetadata('dislikes',int(story['dislikes']))
+
+        if story_ver['is_complete']:
+            self.story.setMetadata('status', 'Completed')
+        else:
+            self.story.setMetadata('status', 'In-Progress')
+
+        self.story.setMetadata('rating', story_ver['maturity_level'])
+        self.story.setMetadata('numWords', unicode(story_ver['word_count']))
+
+        for i in tables['fandoms']:
+            self.story.addToList('category',i['name'])
+        
+        for i in tables['genres']:
+            self.story.addToList('genre',i['name'])
+        
+        for i in tables['characters']:
+            self.story.addToList('characters',i['name'])
+
+        for c in tables['chapters']:
+            chtitle = "Chapter %d"%c['number']
+            if c['title']:
+                chtitle += " - %s"%c['title']
+            self.chapterUrls.append((chtitle,c['body_url']))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+            
+    def getChapterText(self, url):
+        logger.debug('Getting chapter text from: %s' % url)
+        if not url:
+            data = u"<em>This chapter has no text.</em>"
+        else:
+            data = self._fetchUrl(url)
+        soup = bs.BeautifulSoup(u"<div id='story'>"+data+u"</div>")
+        return self.utf8FromSoup(url,soup)
+    
+def getClass():
+    return FictionPadSiteAdapter
+

fanficdownloader/adapters/adapter_fimfictionnet.py

@@ -21,7 +21,6 @@ logger = logging.getLogger(__name__)
 import re
 import urllib2
 import cookielib as cl
-from datetime import datetime
 import json
 
 from .. import BeautifulSoup as bs
@@ -42,6 +41,10 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
         self._setURL("http://"+self.getSiteDomain()+"/story/"+self.story.getMetadata('storyId')+"/")
         self.is_adult = False
         
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%d %b %Y"
+            
     @staticmethod
     def getSiteDomain():
         return 'www.fimfiction.net'
@@ -93,8 +96,10 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
         if "Warning: mysql_fetch_array(): supplied argument is not a valid MySQL result resource" in data:
             raise exceptions.StoryDoesNotExist(self.url)
 
-        if "/images/missing_story.png" in data:
-            raise exceptions.StoryDoesNotExist(self.url)
+        # Can cause problems if a missing story is referenced in a comment.
+        # Shouldn't be needed anyway.
+        # if "/images/missing_story.png" in data:
+        #     raise exceptions.StoryDoesNotExist(self.url)
         
         if "This story has been marked as having adult content." in data:
             raise exceptions.AdultCheckRequired(self.url)
@@ -185,14 +190,42 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
         hrstr="<hr />"
         descdivstr = '<div class="description">'+descdivstr[descdivstr.index(hrstr)+len(hrstr):]
         self.setDescription(self.url,descdivstr)
-        
+
+        # Can't trust dates from API anymore I'm told.
         # Dates are in Unix time
         # Take the publish date from the first chapter posted
-        rawDatePublished = storyMetadata["chapters"][0]["date_modified"]
-        self.story.setMetadata("datePublished", datetime.fromtimestamp(rawDatePublished))
-        rawDateUpdated = storyMetadata["date_modified"]
-        self.story.setMetadata("dateUpdated", datetime.fromtimestamp(rawDateUpdated))
+        # rawDatePublished = storyMetadata["chapters"][0]["date_modified"]
+        # self.story.setMetadata("datePublished", datetime.fromtimestamp(rawDatePublished))
+        # rawDateUpdated = storyMetadata["date_modified"]
+        # self.story.setMetadata("dateUpdated", datetime.fromtimestamp(rawDateUpdated))
+        
+        oldestChapter = None
+        newestChapter = None
+        self.newestChapterNum = None # save for comparing during update.
+        # Scan all chapters to find the oldest and newest, on
+        # FiMFiction it's possible for authors to insert new chapters
+        # out-of-order or change the dates of earlier ones by editing
+        # them--That WILL break epub update.
+        for index, chapterDate in enumerate(soup.findAll('span', {'class':'date'})):
+            date=re.sub(r"(\d+)(st|nd|rd|th)",r"\1",chapterDate.contents[1].strip())
+            chapterDate = makeDate(date,self.dateformat)
+            if oldestChapter == None or chapterDate < oldestChapter:
+                oldestChapter = chapterDate
+            if newestChapter == None or chapterDate > newestChapter:
+                newestChapter = chapterDate
+                self.newestChapterNum = index
 
+        self.story.setMetadata("dateUpdated", newestChapter)
+        
+        pubdatetag = soup.find('span', {'class':'date_approved'})
+        if pubdatetag is None:
+            self.story.setMetadata("datePublished", oldestChapter)            
+        else:
+            pubdateraw = pubdatetag('span')[1].text
+            datestripped=re.sub(r"(\d+)(st|nd|rd|th)",r"\1",pubdateraw.strip())
+            pubDate = makeDate(datestripped,self.dateformat)
+            self.story.setMetadata("datePublished", pubDate)
+            
         chars = soup.find("div", {"class":"inner_data"})
         # fimfic stopped putting the char name on or around the char
         # icon now for some reason.  Pull it from the image name with
@@ -215,10 +248,22 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
                 if not isinstance(value,basestring):
                     value = unicode(value)
                 self.story.setMetadata(metakey, value)
-        
+
+        rawGroupList = soup.find('ul', {'id':'story_group_list'})
+        if rawGroupList is not None:
+            for groupName in rawGroupList.findAll('a', {'href':re.compile('^/group/')}):
+                self.story.addToList("groups",stripHTML(groupName).replace(',', ';'))
             
+    def hookForUpdates(self,chaptercount):
+        if self.oldchapters and len(self.oldchapters) > self.newestChapterNum:
+            print("Existing epub has %s chapters\nNewest chapter is %s.  Discarding old chapters from there on."%(len(self.oldchapters), self.newestChapterNum+1))
+            self.oldchapters = self.oldchapters[:self.newestChapterNum]
+        return len(self.oldchapters)
+            
+        
     def getChapterText(self, url):
         logger.debug('Getting chapter text from: %s' % url)
+        
         soup = bs.BeautifulSoup(self._fetchUrl(url),selfClosingTags=('br','hr')).find('div', {'class' : 'chapter_content'})
         if soup == None:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)

fanficdownloader/adapters/adapter_finestoriescom.py

@@ -93,7 +93,7 @@ class FineStoriesComAdapter(BaseSiteAdapter):
             params['theusername'] = self.getConfig("username")
             params['thepassword'] = self.getConfig("password")
         params['rememberMe'] = '1'
-        params['page'] = 'http://finestories.com/'
+        params['page'] = 'http://'+self.getSiteDomain()+'/'
         params['submit'] = 'Login'
     
         loginUrl = 'http://' + self.getSiteDomain() + '/login.php'
@@ -262,8 +262,6 @@ class FineStoriesComAdapter(BaseSiteAdapter):
                 last[len(last)-1]=last[len(last)-1].append(next)
                 div.append(div1)
             
-            
-            
         # removing all the left-over stuff    
         for a in div.findAll('span'):
             a.extract()

fanficdownloader/adapters/adapter_indeathnet.py

@@ -46,8 +46,8 @@ class InDeathNetAdapter(BaseSiteAdapter):
         self.password = ""
         self.is_adult=False
         
-		
-		  # get storyId from url--url validation guarantees query correct
+        
+        # get storyId from url--url validation guarantees query correct
         m = re.match(self.getSiteURLPattern(),url)
         if m:
             self.story.setMetadata('storyId',m.group('id'))
@@ -87,6 +87,34 @@ class InDeathNetAdapter(BaseSiteAdapter):
         postdate = makeDate(d.group('day')+' '+ym.group('mon')+' '+ym.group('year'),self.dateformat)
         return postdate
 
+    def getAuthorData(self):
+        
+        mainUrl = self.url.replace("/archive","")
+        
+        try:
+            maindata = self._fetchUrl(mainUrl)
+            
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.meta)
+            else:
+                raise e
+
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        mainsoup = bs.BeautifulSoup(maindata)
+
+        # find first entry
+        e = mainsoup.find('div',{'class':"entry"})
+        
+        # get post author as author
+        d = e.find('div',{'class':"desc"})
+        a = d.find('strong')
+        self.story.setMetadata('author',a.contents[0].string.strip())
+        
+        # Don't seem to be able to get author pages anymore
+        self.story.setMetadata('authorUrl','http://www.indeath.net/')
+        self.story.setMetadata('authorId','0')
+        
     ## Getting the chapter list and the meta data, plus 'is adult' checking.
     def extractChapterUrlsAndMetadata(self):
 
@@ -110,17 +138,13 @@ class InDeathNetAdapter(BaseSiteAdapter):
         h = soup.find('a', id="blog_title")
         t = h.find('span')
         self.story.setMetadata('title',stripHTML(t.contents[0]).strip())
-		
+        
         s = t.find('div')        
         if s != None:
             self.setDescription(url,s)
-		
-        # Find authorid and URL from first link in Recent Entries (don't yet reference 'recent entries' - let's see if that is required)
-        a = soup.find('a', href=re.compile(r"http://www.indeath.net/user/\d+\-[a-z0-9]+/$"))		#http://www.indeath.net/user/9083-cyrex/
-        m = re.search('http://www.indeath.net/user/(?P<id>\d+)\-(?P<name>[a-z0-9]*)/$',a['href'])
-        self.story.setMetadata('authorId',m.group('id'))
-        self.story.setMetadata('authorUrl',a['href'])
-        self.story.setMetadata('author',m.group('name'))
+
+        # Get Author from main blog page since it's not reliably on the archive page
+        self.getAuthorData()
 
         # Find the chapters:
         chapters=soup.findAll('a', title="View entry", href=re.compile(r'http://www.indeath.net/blog/'+self.story.getMetadata('storyId')+"/entry\-(\d+)\-([^/]*)/$"))
@@ -149,21 +173,20 @@ class InDeathNetAdapter(BaseSiteAdapter):
             if len(chapters)==1:
                 self.chapterUrls.append((self.story.getMetadata('title'),chapter['href']))
             else:
-            	ct = stripHTML(chapter)
-            	tnew = re.match("(?i)"+self.story.getMetadata('title')+r" - (?P<newtitle>.*)$",ct)
-            	if tnew:
-            		chaptertitle = tnew.group('newtitle')
-            	else:
-            		chaptertitle = ct
+                ct = stripHTML(chapter)
+                tnew = re.match("(?i)"+self.story.getMetadata('title')+r" - (?P<newtitle>.*)$",ct)
+                if tnew:
+                    chaptertitle = tnew.group('newtitle')
+                else:
+                    chaptertitle = ct
                 self.chapterUrls.append((chaptertitle,chapter['href']))
-	
-		
+
         
         
     # grab the text for an individual chapter.
     def getChapterText(self, url):
         logger.debug('Getting chapter text from: %s' % url)
-		
+        
         #chapter=bs.BeautifulSoup('<div class="story"></div>')
         data = self._fetchUrl(url)
         soup = bs.BeautifulSoup(data,selfClosingTags=('br','hr','span','center'))

fanficdownloader/adapters/adapter_nickandgregnet.py

@@ -166,10 +166,11 @@ class NickAndGregNetAdapter(BaseSiteAdapter):
 
         soup = bs.BeautifulStoneSoup(self._fetchUrl(url),
                                      selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
-        
-        div = soup.find('table', {'class' : 'tblborder6'})
 
-        if None == div:
-            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
+        # wrap a div around it.
+        divsoup = bs.BeautifulStoneSoup('<div class="story"></div>',
+                                        selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        div = divsoup.find('div')
+        div.append(soup.find('table', {'class' : 'tblborder6'}))
+
         return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_portkeyorg.py

@@ -251,8 +251,10 @@ class PortkeyOrgAdapter(BaseSiteAdapter): # XXX
 
         logger.debug('Getting chapter text from: %s' % url)
         data = self._fetchUrl(url)
-        soup = bs.BeautifulStoneSoup(data,
-                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+
+        data = data.replace("HTML>","div>")
+        
+        soup = bs.BeautifulSoup(data)
 
         #print("soup:%s"%soup)
         tag = soup.find('td', {'class' : 'story'})

fanficdownloader/adapters/adapter_restrictedsectionorg.py

@@ -22,7 +22,6 @@ import re
 import urllib2
 import cookielib as cl
 from datetime import datetime
-import json
 
 from .. import BeautifulSoup as bs
 from ..htmlcleanup import stripHTML

fanficdownloader/adapters/adapter_simplyundeniablecom.py

@@ -214,5 +214,6 @@ class SimplyUndeniableComAdapter(BaseSiteAdapter):
 
         if None == div:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+        div.name='div'
     
         return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_siyecouk.py

@@ -241,5 +241,6 @@ class SiyeCoUkAdapter(BaseSiteAdapter): # XXX
         
         if None == story:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+        story.name='div'
     
         return self.utf8FromSoup(url,story)

fanficdownloader/adapters/adapter_storiesonlinenet.py

@@ -0,0 +1,303 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+logger = logging.getLogger(__name__)
+import re
+import urllib2
+
+from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
+from .. import exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter,  makeDate
+
+def getClass():
+    return StoriesOnlineNetAdapter
+
+# Class name has to be unique.  Our convention is camel case the
+# sitename with Adapter at the end.  www is skipped.
+class StoriesOnlineNetAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+
+        self.decode = ["Windows-1252",
+                       "utf8"] # 1252 is a superset of iso-8859-1.
+                               # Most sites that claim to be
+                               # iso-8859-1 (and some that claim to be
+                               # utf8) are really windows-1252.
+        self.username = "NoneGiven" # if left empty, site doesn't return any message at all.
+        self.password = ""
+        self.is_adult=False
+        
+        # get storyId from url
+        self.story.setMetadata('storyId',self.parsedUrl.path.split('/',)[2].split(':')[0])
+        if 'storyInfo' in self.story.getMetadata('storyId'):
+            self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1])
+        
+        # normalized story URL.
+        self._setURL('http://' + self.getSiteDomain() + '/s/'+self.story.getMetadata('storyId'))
+        
+        # Each adapter needs to have a unique site abbreviation.
+        self.story.setMetadata('siteabbrev','strol')
+
+        # The date format will vary from site to site.
+        # http://docs.python.org/library/datetime.html#strftime-strptime-behavior
+        self.dateformat = "%Y-%m-%d"
+            
+    @staticmethod # must be @staticmethod, don't remove it.
+    def getSiteDomain():
+        # The site domain.  Does have www here, if it uses it.
+        return 'storiesonline.net'
+
+    @classmethod
+    def getSiteExampleURLs(self):
+        return "http://"+self.getSiteDomain()+"/s/1234 http://"+self.getSiteDomain()+"/s/1234:4010"
+
+    def getSiteURLPattern(self):
+        return re.escape("http://"+self.getSiteDomain())+r"/s/\d+((:\d+)?(;\d+)?$|(:i)?$)"
+
+    ## Login seems to be reasonably standard across eFiction sites.
+    def needToLoginCheck(self, data):
+        if 'Free Registration' in data \
+                or "Invalid Password!" in data \
+                or "Invalid User Name!" in data \
+                or "Log In" in data \
+                or "Access to unlinked chapters requires" in data:
+            return True
+        else:
+            return False
+        
+    def performLogin(self, url):
+        params = {}
+
+        if self.password:
+            params['theusername'] = self.username
+            params['thepassword'] = self.password
+        else:
+            params['theusername'] = self.getConfig("username")
+            params['thepassword'] = self.getConfig("password")
+        params['rememberMe'] = '1'
+        params['page'] = 'http://'+self.getSiteDomain()+'/'
+        params['submit'] = 'Login'
+    
+        loginUrl = 'http://' + self.getSiteDomain() + '/login.php'
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['theusername']))
+    
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "My Account" not in d : #Member Account
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                              params['theusername']))
+            raise exceptions.FailedToLogin(url,params['theusername'])
+            return False
+        else:
+            return True
+
+    ## Getting the chapter list and the meta data, plus 'is adult' checking.
+    def extractChapterUrlsAndMetadata(self):
+
+        # index=1 makes sure we see the story chapter index.  Some
+        # sites skip that for one-chapter stories.
+        url = self.url
+        logger.debug("URL: "+url)
+
+        try:
+            data = self._fetchUrl(url+":i")
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(self.url)
+            else:
+                raise e
+
+        if self.needToLoginCheck(data):
+            # need to log in for this one.
+            self.performLogin(url)
+            data = self._fetchUrl(url+":i")
+        
+        if "Access denied. This story has not been validated by the adminstrators of this site." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.")
+        elif "Error! The story you're trying to access is being filtered by your choice of contents filtering." in data:
+            raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Error! The story you're trying to access is being filtered by your choice of contents filtering.")
+            
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        soup = bs.BeautifulSoup(data)
+        #print data
+
+        # Now go hunting for all the meta data and the chapter list.
+        
+        ## Title
+        a = soup.find('h1')
+        self.story.setMetadata('title',stripHTML(a))
+        
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"/a/\w+"))
+        self.story.setMetadata('authorId',a['href'].split('/')[2])
+        self.story.setMetadata('authorUrl','http://'+self.host+a['href'])
+        self.story.setMetadata('author',stripHTML(a).replace("'s Page",""))
+
+        # Find the chapters:
+        chapters = soup.findAll('a', href=re.compile(r'^/s/'+self.story.getMetadata('storyId')+":\d+$"))
+        if len(chapters) != 0:
+            for chapter in chapters:
+                # just in case there's tags, like <i> in chapter titles.
+                self.chapterUrls.append((stripHTML(chapter),'http://'+self.host+chapter['href']))
+        else:
+            self.chapterUrls.append((self.story.getMetadata('title'),'http://'+self.host+'/s/'+self.story.getMetadata('storyId')))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+
+        # surprisingly, the detailed page does not give enough details, so go to author's page
+        skip=0
+        i=0
+        while i == 0:
+            asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getMetadata('authorUrl')+"&skip="+str(skip)))
+
+            a = asoup.findAll('td', {'class' : 'lc2'})
+            for lc2 in a:
+                if lc2.find('a')['href'] == '/s/'+self.story.getMetadata('storyId'):
+                    i=1
+                    break
+                if a[len(a)-1] == lc2:
+                    skip=skip+10
+
+        for cat in lc2.findAll('div', {'class' : 'typediv'}):
+            self.story.addToList('genre',cat.text)
+
+        # in lieu of word count.
+        self.story.setMetadata('size', lc2.findNext('td', {'class' : 'num'}).text)
+        
+        lc4 = lc2.findNext('td', {'class' : 'lc4'})        
+        
+        try:
+            a = lc4.find('a', href=re.compile(r"/library/show_series.php\?id=\d+"))
+            i = a.parent.text.split('(')[1].split(')')[0]
+            self.setSeries(stripHTML(a), i)
+            self.story.setMetadata('seriesUrl','http://'+self.host+a['href'])
+        except:
+            pass
+        try:
+            a = lc4.find('a', href=re.compile(r"/library/universe.php\?id=\d+"))
+            if a:
+                self.story.setMetadata("universe",stripHTML(a))
+        except:
+            pass
+            
+
+        desc = lc4.contents[0]
+        self.setDescription('http://'+self.host+'/s/'+self.story.getMetadata('storyId'),desc)
+            
+        for b in lc4.findAll('b'):
+            #logger.debug('Getting metadata: "%s"' % b)
+            label = b.text
+            if label in ['Posted:', 'Concluded:', 'Updated:']:
+                value = b.findNext('noscript').text
+                #logger.debug('Have a date field label: "%s", value: "%s"' % (label, value))
+            else:
+                value = b.nextSibling
+            #logger.debug('label: "%s", value: "%s"' % (label, value))
+            
+            if 'Sex' in label:
+                self.story.setMetadata('rating', value)
+                
+            if 'Codes' in label:
+                for code in value.split(' '):
+                    self.story.addToList('codes',code)
+                    
+            if 'Posted' in label:
+                self.story.setMetadata('datePublished', makeDate(stripHTML(value), self.dateformat))
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+                
+            if 'Concluded' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+                
+            if 'Updated' in label:
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+#                
+        status = lc4.find('span', {'class' : 'ab'})
+        if  status != None:
+            self.story.setMetadata('status', 'In-Progress')
+            if "Last Activity" in status.text:
+                # date is passed as a timestamp and converted in JS.
+                value = status.findNext('noscript').text
+                self.story.setMetadata('dateUpdated', makeDate(stripHTML(value), self.dateformat))
+        else:
+            self.story.setMetadata('status', 'Completed')
+
+            
+    # grab the text for an individual chapter.
+    def getChapterText(self, url):
+
+        logger.debug('Getting chapter text from: %s' % url)
+
+        soup = bs.BeautifulSoup(self._fetchUrl(url),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+        div = soup.find('div', {'id' : 'story'})
+        
+        # some big chapters are split over several pages
+        pager = div.find('span', {'class' : 'pager'})
+        if pager != None:
+            urls=pager.findAll('a')
+            urls=urls[:len(urls)-1]
+                        
+            for ur in urls:
+                soup = bs.BeautifulSoup(self._fetchUrl("http://"+self.getSiteDomain()+ur['href']),
+                                     selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
+        
+                div1 = soup.find('div', {'id' : 'story'})
+                
+                # appending next section
+                last=div.findAll('p')
+                next=div1.find('span', {'class' : 'conTag'}).nextSibling
+            
+                last[len(last)-1]=last[len(last)-1].append(next)
+                div.append(div1)
+            
+        # removing all the left-over stuff    
+        for a in div.findAll('span'):
+            a.extract() 
+            
+        for a in div.findAll('h1'):
+            a.extract()
+        for a in div.findAll('h2'):
+            a.extract()
+        for a in div.findAll('h3'):
+            a.extract()
+        for a in div.findAll('h4'):
+            a.extract()
+        for a in div.findAll('br'):
+            a.extract()
+        for a in div.findAll('div', {'class' : 'date'}):
+            a.extract()
+            
+        a = div.find('form')
+        if a != None:
+            b = a.nextSibling
+            while b != None:
+                a.extract()
+                a=b
+                b=b.nextSibling
+        
+
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+    
+        return self.utf8FromSoup(url,div)

fanficdownloader/adapters/adapter_test1.py

@@ -323,7 +323,7 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
 </div>
 '''
         elif self.story.getMetadata('storyId') == '0':
-            text=u'''
+            text=u'''<div>
 <h3>45. Pronglet Returns to Hogwarts: Chapter 7</h3>
 <br />
     eyes… but I’m not convinced we should automatically<br />
@@ -332,6 +332,7 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
 <br /><br />
     “Sure, invite her along. Does she have children?”<br />
 <br />
+</div>
 '''
         else:
             if self.story.getMetadata('storyId') == '667':

fanficdownloader/adapters/adapter_tthfanficorg.py

@@ -231,10 +231,10 @@ class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter):
         BtVS = True
         BtVSNonX = False
         for cat in verticaltable.findAll('a', href=re.compile(r"^/Category-")):
-            if cat.string not in ['General', 'Non-BtVS/AtS Stories', 'BtVS/AtS Non-Crossover', 'Non-BtVS Crossovers']:
+            if cat.string not in ['General', 'Non-BtVS/AtS Stories', 'Non-BTVS/AtS Stories', 'BtVS/AtS Non-Crossover', 'Non-BtVS Crossovers']:
                 self.story.addToList('category',cat.string)
             else:
-                if 'Non-BtVS' in cat.string:
+                if 'Non-BtVS' in cat.string or 'Non-BTVS' in cat.string:
                     BtVS = False
                 if 'BtVS/AtS Non-Crossover' == cat.string:
                     BtVSNonX = True

fanficdownloader/adapters/adapter_whoficcom.py

@@ -22,6 +22,7 @@ import re
 import urllib2
 
 from .. import BeautifulSoup as bs
+from ..htmlcleanup import stripHTML
 from .. import exceptions as exceptions
 
 from base_adapter import BaseSiteAdapter,  makeDate
@@ -227,7 +228,8 @@ class WhoficComSiteAdapter(BaseSiteAdapter):
 
         if None == span:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
-    
+
+        span.name='div'
         return self.utf8FromSoup(url,span)
 
 def getClass():

fanficdownloader/adapters/base_adapter.py

@@ -26,6 +26,7 @@ from functools import partial
 
 from .. import BeautifulSoup as bs
 from ..htmlcleanup import stripHTML
+from ..htmlheuristics import replace_br_with_p
 
 logger = logging.getLogger(__name__)
 
@@ -247,6 +248,10 @@ class BaseSiteAdapter(Configurable):
             self.metadataDone = True
         return self.story
 
+    def hookForUpdates(self,chaptercount):
+        "Usually not needed."
+        return chaptercount
+
     ###############################
     
     @staticmethod
@@ -347,16 +352,11 @@ class BaseSiteAdapter(Configurable):
             if t.name in ('center'):
                 t['class']=t.name
                 t.name='div'
-	    # removes paired, but empty tags.
-            if t.string != None and len(t.string.strip()) == 0 :
+	    # removes paired, but empty non paragraph tags.
+            if t.name not in ('p') and t.string != None and len(t.string.strip()) == 0 :
                 t.extract()
-                
-        retval = soup.__str__('utf8').decode('utf-8')
 
-        if self.getConfig('replace_hr'):
-            # replacing a self-closing tag with a container tag in the
-            # soup is more difficult than it first appears.  So cheat.
-            retval = retval.replace("<hr />","<div class='center'>* * *</div>")
+        retval = soup.__str__('utf8').decode('utf-8')
 
         if self.getConfig('nook_img_fix'):
             # if the <img> tag doesn't have a div or a p around it,
@@ -367,7 +367,19 @@ class BaseSiteAdapter(Configurable):
             
         # Don't want body tags in chapter html--writers add them.
         # This is primarily for epub updates.
-        return re.sub(r"</?body>\r?\n?","",retval)
+        retval = re.sub(r"</?body>\r?\n?","",retval)
+        
+        if self.getConfig("replace_br_with_p"):
+            # Apply heuristic processing to replace <br> paragraph
+            # breaks with <p> tags.
+            retval = replace_br_with_p(retval)
+            
+        if self.getConfig('replace_hr'):
+            # replacing a self-closing tag with a container tag in the
+            # soup is more difficult than it first appears.  So cheat.
+            retval = retval.replace("<hr />","<div class='center'>* * *</div>")
+
+        return retval
 
 def cachedfetch(realfetch,cache,url):
     if url in cache:
@@ -379,7 +391,7 @@ fullmon = {"January":"01", "February":"02", "March":"03", "April":"04", "May":"0
            "June":"06","July":"07", "August":"08", "September":"09", "October":"10",
            "November":"11", "December":"12" }
 
-def makeDate(string,format):
+def makeDate(string,dateform):
     # Surprise!  Abstracting this turned out to be more useful than
     # just saving bytes.
 
@@ -388,10 +400,10 @@ def makeDate(string,format):
     # there's non-english content. -- ficbook.net now makes that a
     # lie.  It has to do something even more complicated to get
     # Russian month names correct everywhere.
-    do_abbrev = "%b" in format
+    do_abbrev = "%b" in dateform
         
-    if "%B" in format or do_abbrev:
-        format = format.replace("%B","%m").replace("%b","%m")
+    if "%B" in dateform or do_abbrev:
+        dateform = dateform.replace("%B","%m").replace("%b","%m")
         for (name,num) in fullmon.items():
             if do_abbrev:
                 name = name[:3] # first three for abbrev
@@ -399,5 +411,5 @@ def makeDate(string,format):
                 string = string.replace(name,num)
                 break
             
-    return datetime.datetime.strptime(string,format)
+    return datetime.datetime.strptime(string,dateform)
 

fanficdownloader/epubutils.py

@@ -7,6 +7,9 @@ __license__   = 'GPL v3'
 __copyright__ = '2012, Jim Miller'
 __docformat__ = 'restructuredtext en'
 
+import logging
+logger = logging.getLogger(__name__)
+
 import re, os, traceback
 from zipfile import ZipFile
 from xml.dom.minidom import parseString
@@ -72,7 +75,7 @@ def get_update_data(inputio,
                 # remove all .. and the path part above it, if present.
                 # Mostly for epubs edited by Sigil.
                 src = re.sub(r"([^/]+/\.\./)","",src)
-                print("epubutils: found pre-existing cover image:%s"%src)
+                #print("epubutils: found pre-existing cover image:%s"%src)
                 oldcoverimghref = src
                 oldcoverimgdata = epub.read(src)
                 for item in contentdom.getElementsByTagName("item"):
@@ -81,8 +84,8 @@ def get_update_data(inputio,
                         break
                 oldcover = (oldcoverhtmlhref,oldcoverhtmltype,oldcoverhtmldata,oldcoverimghref,oldcoverimgtype,oldcoverimgdata)
             except Exception as e:
-                print("Cover Image %s not found"%src)
-                print("Exception: %s"%(unicode(e)))
+                logger.warn("Cover Image %s not found"%src)
+                logger.warn("Exception: %s"%(unicode(e)))
                 traceback.print_exc()
 
     filecount = 0
@@ -118,8 +121,8 @@ def get_update_data(inputio,
                                 images[longdesc] = data
                                 img['src'] = img['longdesc']
                             except Exception as e:
-                                print("Image %s not found!\n(originally:%s)"%(newsrc,longdesc))
-                                print("Exception: %s"%(unicode(e)))
+                                logger.warn("Image %s not found!\n(originally:%s)"%(newsrc,longdesc))
+                                logger.warn("Exception: %s"%(unicode(e)))
                                 traceback.print_exc()
                         soup = soup.find('body')
                         # ffdl epubs have chapter title h3
@@ -143,8 +146,8 @@ def get_update_data(inputio,
     except:
         pass
                     
-    for k in images.keys():
-        print("\tlongdesc:%s\n\tData len:%s\n"%(k,len(images[k])))
+    #for k in images.keys():
+        #print("\tlongdesc:%s\n\tData len:%s\n"%(k,len(images[k])))
     return (source,filecount,soups,images,oldcover,calibrebookmark,logfile)
 
 def get_path_part(n):

fanficdownloader/htmlcleanup.py

@@ -15,6 +15,9 @@
 # limitations under the License.
 #
 
+import logging
+logger = logging.getLogger(__name__)
+
 import re
 
 def _unirepl(match):
@@ -31,7 +34,7 @@ def _unirepl(match):
     except:
         # This way, at least if there's more of entities out there
         # that fail, it doesn't blow the entire download.
-        print "Numeric entity translation failed, skipping: &#x%s%s"%(match.group(1),match.group(2))
+        logger.warn("Numeric entity translation failed, skipping: &#x%s%s"%(match.group(1),match.group(2)))
         retval = ""
     return retval
 
@@ -69,8 +72,9 @@ def removeEntities(text):
     
     if text is None:
         return ""
-    if not (isinstance(text,str) or isinstance(text,unicode)):
-        return str(text)
+    
+    if not isinstance(text,basestring):
+        return unicode(text)
     
     try:
         t = text.decode('utf-8')

fanficdownloader/htmlheuristics.py

@@ -0,0 +1,283 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2013 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import logging
+logger = logging.getLogger(__name__)
+import re
+import BeautifulSoup as bs
+
+from . import exceptions as exceptions
+
+def replace_br_with_p(body):
+
+    # Ascii character (and Unicode as well) xA0 is a non-breaking space, ascii code 160.
+    # However, Python Regex does not recognize it as a whitespace, so we'll be changing it to a reagular space.
+    body = body.replace(u'\xa0', u' ')
+
+    if body.find('>') == -1 or body.rfind("<") == -1:
+        return body
+
+    # logger.debug(u'BODY start.: ' + body[:250])
+    # logger.debug(u'BODY end...: ' + body[-250:])
+    # logger.debug(u'BODY.......: ' + body)
+
+    # clean breaks (<br />), removing whitespaces between them.
+    body = re.sub(r'\s*<br[^>]*>\s*', r'<br />', body)
+
+    # change surrounding div to a p and remove attrs Top surrounding
+    # tag in all cases now should be div, to just strip the first and
+    # last tags.
+    if is_valid_block(body) and body.find('<div') == 0:
+        body = body[body.index('>')+1:body.rindex("<")]
+
+    body = soup_up_div(u'<div>' + body + u'</div>')
+
+    body = body[body.index('>')+1:body.rindex("<")]
+
+    # Find all bexisting blocks with p, pre and blockquote tags, we need to shields break tags inside those.
+    # This is for "lenient" mode, however it is also used to clear break tags before and after the block elements.
+    blocksRegex = re.compile(r'(\s*<br\ />\s*)*\s*<(pre|p|blockquote|table)([^>]*)>(.+?)</\2>\s*(\s*<br\ />\s*)*', re.DOTALL)
+    body = blocksRegex.sub(r'\n<\2\3>\4</\2>\n', body)
+
+    # if aggressive mode = true
+        # blocksRegex = re.compile(r'(\s*<br\ */*>\s*)*\s*<(pre)([^>]*)>(.+?)</\2>\s*(\s*<br\ */*>\s*)*', re.DOTALL)
+        # In aggressive mode, we also check breakes inside blockquotes, meaning we can get orphaned paragraph tags.
+        # body = re.sub(r'<blockquote([^>]*)>(.+?)</blockquote>', r'<blockquote\1><p>\2</p></blockquote>', body, re.DOTALL)
+    # end aggressive mode
+
+    blocks = blocksRegex.finditer(body)
+    # For our replacements to work, we need to work backwards, so we reverse the iterator.
+    blocksList = []
+    for match in blocks:
+        blocksList.insert(0, match)
+
+    for match in blocksList:
+        group4 =  match.group(4).replace(u'<br />', u'{br /}')
+        body = body[:match.start(4)] + group4 + body[match.end(4):]
+
+    # change surrounding div to a p and remove attrs Top surrounding
+    # tag in all cases now should be div, to just strip the first and
+    # last tags.
+    # body = u'<p>' + body + u'</p>'
+
+    # Nuke div tags surrounding a HR tag.
+    body = re.sub(r'<div[^>]+>\s*<hr[^>]+>\s*</div>', r'\n<hr />\n', body)
+
+    # So many people add formatting to their HR tags, and ePub does not allow those, we are supposed to use css.
+    # This nukes the hr tag attributes.
+    body = re.sub(r'\s*<hr[^>]+>\s*', r'\n<hr />\n', body)
+
+    # Remove leading and trailing breaks from HR tags
+    body = re.sub(r'\s*(<br\ \/>)*\s*<hr\ \/>\s*(<br\ \/>)*\s*', r'\n<hr />\n', body)
+    # Nuking breaks leading paragraps that may be in the body. They are eventually treated as <p><br /></p>
+    body = re.sub(r'\s*(<br\ \/>)+\s*<p', r'\n<p></p>\n<p', body)
+    # Nuking breaks trailing paragraps that may be in the body. They are eventually treated as <p><br /></p>
+    body = re.sub(r'</p>\s*(<br\ \/>)+\s*', r'</p>\n<p></p>\n', body)
+
+    # Because a leading or trailing non break tag will break the following code, we have to mess around rather badly for a few lines.
+    body = body.replace(u'[',u'&squareBracketStart;')
+    body = body.replace(u']',u'&squareBracketEnd;')
+    body = body.replace(u'<br />',u'[br /]')
+
+    breaksRegexp = [
+        re.compile(r'([^\]])(\[br\ \/\])([^\[])'),
+        re.compile(r'([^\]])(\[br\ \/\]){2}([^\[])'),
+        re.compile(r'([^\]])(\[br\ \/\]){3}([^\[])'),
+        re.compile(r'([^\]])(\[br\ \/\]){4}([^\[])'),
+        re.compile(r'([^\]])(\[br\ \/\]){5}([^\[])'),
+        re.compile(r'([^\]])(\[br\ \/\]){6}([^\[])'),
+        re.compile(r'([^\]])(\[br\ \/\]){7}([^\[])'),
+        re.compile(r'([^\]])(\[br\ \/\]){8}([^\[])'),
+        re.compile(r'(\[br\ \/\]){9,}')]
+
+    breaksCount = [
+        len(breaksRegexp[0].findall(body)),
+        len(breaksRegexp[1].findall(body)),
+        len(breaksRegexp[2].findall(body)),
+        len(breaksRegexp[3].findall(body)),
+        len(breaksRegexp[4].findall(body)),
+        len(breaksRegexp[5].findall(body)),
+        len(breaksRegexp[6].findall(body)),
+        len(breaksRegexp[7].findall(body))]
+
+    breaksMax = 0
+    breaksMaxIndex = 0;
+
+    for i in range(1,len(breaksCount)):
+        if breaksCount[i] >= breaksMax:
+            breaksMax = breaksCount[i]
+            breaksMaxIndex = i
+
+    lines = body.split(u'[br /]')
+    contentLines = 0;
+    contentLinesSum = 0;
+    longestLineLength = 0;
+    averageLineLength = 0;
+
+    for line in lines:
+        lineLen = len(line.strip())
+        if lineLen > 0:
+            contentLines += 1
+            contentLinesSum += lineLen
+            if lineLen > longestLineLength:
+                longestLineLength = lineLen
+
+    averageLineLength = contentLinesSum/contentLines
+
+    logger.debug(u'---')
+    logger.debug(u'Lines.............: ' + str(len(lines)))
+    logger.debug(u'contentLines......: ' + str(contentLines))
+    logger.debug(u'contentLinesSum...: ' + str(contentLinesSum))
+    logger.debug(u'longestLineLength.: ' + str(longestLineLength))
+    logger.debug(u'averageLineLength.: ' + str(averageLineLength))
+
+    if breaksMaxIndex == len(breaksCount)-1 and breaksMax < 2:
+        breaksMaxIndex = 0
+        breaksMax = breaksCount[0]
+
+    logger.debug(u'---')
+    logger.debug(u'breaks 1: ' + str(breaksCount[0]))
+    logger.debug(u'breaks 2: ' + str(breaksCount[1]))
+    logger.debug(u'breaks 3: ' + str(breaksCount[2]))
+    logger.debug(u'breaks 4: ' + str(breaksCount[3]))
+    logger.debug(u'breaks 5: ' + str(breaksCount[4]))
+    logger.debug(u'breaks 6: ' + str(breaksCount[5]))
+    logger.debug(u'breaks 7: ' + str(breaksCount[6]))
+    logger.debug(u'breaks 8: ' + str(breaksCount[7]))
+    logger.debug(u'----')
+    logger.debug(u'max found: ' + str(breaksMax))
+    logger.debug(u'max Index: ' + str(breaksMaxIndex))
+    logger.debug(u'----')
+
+    if breaksMaxIndex > 0 and breaksCount[0] > breaksMax and averageLineLength < 90:
+        body = breaksRegexp[0].sub(r'\1 \n\3', body)
+
+    # Find all instances of consecutive breaks less than otr equal to the max count use most often
+    #  replase those tags to inverted p tag pairs, those with more connsecutive breaks are replaced them with a horisontal line
+    for i in range(len(breaksCount)):
+        if i <= breaksMaxIndex:
+            logger.debug(str(i) + u' <= breaksMaxIndex (' + str(breaksMaxIndex) + u')')
+            body = breaksRegexp[i].sub(r'\1</p>\n<p>\3', body)
+        elif i == breaksMaxIndex+1:
+            logger.debug(str(i) + u' == breaksMaxIndex+1 (' + str(breaksMaxIndex+1) + u')')
+            body = breaksRegexp[i].sub(r'\1</p>\n<p><br/></p>\n<p>\3', body)
+        else:
+            logger.debug(str(i) + u' > breaksMaxIndex+1 (' + str(breaksMaxIndex+1) + u')')
+            body = breaksRegexp[i].sub(r'\1</p>\n<hr />\n<p>\3', body)
+
+    body = breaksRegexp[8].sub(r'</p>\n<hr />\n<p>', body)
+
+    # Reverting the square brackets
+    body = body.replace(u'[', u'<')
+    body = body.replace(u']', u'>')
+    body = body.replace(u'&squareBracketStart;', u'[')
+    body = body.replace(u'&squareBracketEnd;', u']')
+
+    body = body.replace(u'{p}', u'<p>')
+    body = body.replace(u'{/p}', u'</p>')
+
+    # If for some reason, a third break makes its way inside the paragraph, preplace that with the empty paragraph for the additional linespaing.
+    body = re.sub(r'<p>\s*(<br\ \/>)+', r'<p><br /></p>\n<p>', body)
+
+    # change empty p tags to include a br to force spacing.
+    body = re.sub(r'<p>\s*</p>', r'<p><br/></p>', body)
+
+    # Clean up hr tags, and add inverted p tag pairs
+    body = re.sub(r'(<div[^>]+>)*\s*<hr\ \/>\s*(</div>)*', r'\n<hr />\n', body)
+
+    # Clean up hr tags, and add inverted p tag pairs
+    body = re.sub(r'\s*<hr\ \/>\s*', r'</p>\n<hr />\n<p>', body)
+
+    # Because the previous regexp may cause trouble if the hr tag already had a p tag pair around it, w nee dot repair that.
+    # Repeated opening p tags are condenced to one. As we added the extra leading opening p tags, we can safely assume that
+    #  the last in such a chain must be the original. Lets keep its attributes if they are there.
+    body = re.sub(r'\s*(<p[^>]*>\s*)+<p([^>]*)>\s*', r'\n<p\2>', body)
+    # Repeated closing p tags are condenced to one
+    body = re.sub(r'\s*(<\/\s*p>\s*){2,}', r'</p>\n', body)
+
+    # superflous cleaning, remove whitespaces traling opening p tags. These does affect formatting.
+    body = re.sub(r'\s*<p([^>]*)>\s*', r'\n<p\1>', body)
+    # superflous cleaning, remove whitespaces leading closing p tags. These does not affect formatting.
+    body = re.sub(r'\s*</p>\s*', r'</p>\n', body)
+
+    # Remove empty tag pairs
+    body = re.sub(r'\s*<(\S+)[^>]*>\s*</\1>', r'', body)
+
+    body = body.replace(u'{br /}', u'<br />')
+    body = body.strip()
+
+    # re-wrap in div tag.
+    body = u'<div>\n' + body + u'</div>\n'
+
+    return body
+
+def is_valid_block(block):
+    return str(block).find('<') == 0 and str(block).find('<!') != 0
+
+def soup_up_div(body):
+    blockTags = ['address', 'blockquote', 'del', 'div', 'dl', 'fieldset', 'form', 'ins', 'noscript', 'ol', 'p', 'pre', 'table', 'ul']
+    recurseTags = ['blockquote', 'div', 'noscript']
+
+    tag = body[:body.index('>')+1]
+    tagend = body[body.rindex('<'):]
+
+    body = body.replace(u'<br />', u'[br /]')
+
+    soup = bs.BeautifulSoup(body)
+
+    body = u''
+    lastElement = 1 # 1 = block, 2 = nested, 3 = invalid
+
+    for i in soup.contents[0]:
+        if str(i).strip().__len__() > 0:
+            s = str(i)
+            if  type(i) == bs.Tag:
+                if  i.name in blockTags:
+                    if lastElement > 1:
+                        body = body.strip(r'\s*(\[br\ \/\]\s*)*\s*')
+                        body += u'{/p}'
+
+                    lastElement = 1
+
+                    if i.name in recurseTags:
+                        s = soup_up_div(s)
+
+                    body += s.strip() + '\n'
+                else:
+                    if lastElement == 1:
+                        body = body.strip(r'\s*(\[br\ \/\]\s*)*\s*')
+                        body += u'{p}'
+
+                    lastElement = 2
+                    body += s
+            elif type(i) == bs.Comment:
+                body += s
+            else:
+                if lastElement == 1:
+                    body = body.strip(r'\s*(\[br\ \/\]\s*)*\s*')
+                    body += u'{p}'
+
+                lastElement = 3
+                body += s
+
+    if lastElement > 1:
+        body = body.strip(r'\s*(\[br\ \/\]\s*)*\s*')
+        body += u'{/p}'
+
+    body = body.replace(u'[br /]', u'<br />')
+
+    return tag + body + tagend

fanficdownloader/story.py

@@ -349,6 +349,8 @@ class Story(Configurable):
                 return removeAllEntities(value)
             else:
                 return value
+        else: #if self.getConfig("default_value_"+key):
+            return self.getConfig("default_value_"+key)
         
     def getAllMetadata(self,
                        removeallentities=False,
@@ -485,6 +487,8 @@ class Story(Configurable):
 
         if None in subjectset:
             subjectset.remove(None)
+        if '' in subjectset:
+            subjectset.remove('')
 
         return list(subjectset | set(self.getConfigList("extratags")))
             
@@ -627,7 +631,7 @@ class Story(Configurable):
                     ext)
                 self.imgtuples.append({'newsrc':newsrc,'mime':mime,'data':data})
                 
-            logger.debug("\nimgurl:%s\nnewsrc:%s\nimage size:%d\n"%(imgurl,newsrc,len(data)))
+            #logger.debug("\nimgurl:%s\nnewsrc:%s\nimage size:%d\n"%(imgurl,newsrc,len(data)))
         else:
             newsrc = self.imgtuples[self.imgurls.index(imgurl)]['newsrc']
             

fanficdownloader/writers/base_writer.py

@@ -221,7 +221,7 @@ class BaseStoryWriter(Configurable):
                     lastupdated=self.story.getMetadataRaw('dateUpdated').date()
                     fileupdated=datetime.datetime.fromtimestamp(os.stat(outfilename)[8]).date()
                     if fileupdated > lastupdated:
-                        print "File(%s) Updated(%s) more recently than Story(%s) - Skipping" % (outfilename,fileupdated,lastupdated)
+                        logger.warn("File(%s) Updated(%s) more recently than Story(%s) - Skipping" % (outfilename,fileupdated,lastupdated))
                         return
             if not metaonly:        
                 self.story = self.adapter.getStory() # get full story

index.html

@@ -46,6 +46,17 @@
       {{yourfile}}
       <!--			</div> -->
 
+          <h3>fanfiction.net / fimfiction.net</h3>
+          <p>
+            Fanfiction.net appears to be blocking access from Google
+            App Engine, which prevents this web service.  There's
+            nothing I can do about it.  At the time of writing, the
+            latest CLI and calibre plugin versions worked.
+          </p>
+          <p>It appears that FimFiction.net is also blocking access from Google
+            App Engine now.
+          </p>
+	  
       {% if authorized %}
       <form action="/fdown" method="post">
 	<div id='urlbox'>
@@ -53,27 +64,15 @@
 	    <p>Hi, {{ nickname }}! This is FanFictionDownLoader, which makes reading stories from various websites
 	      much easier. </p>
 	  </div>
-	  <!-- put announcements here, h3 is a good title size.
+	  <!-- put announcements here, h3 is a good title size. -->
 	  <h3>Changes:</h3>
 	  <p>
 	    <ul>
-              <li>Better doc section override order in ini files.</li>
-              <li>Additional series as site specific data for AO3.</li>
-              <li>Fix for AO3 stories without series.</li>
-              <li>Fixes for changes to harrypotterfanfictioncom.</li>
-              <li>Add User-agent="FFDL/1.7" for all adapters for fanfiction.net changes.<br>
-                (Remove from specific adapters.)</li>
+              <li>Fixes for storiesonline.net, thanks davidfor</li>
+              <li>Additional fixes/enhancements for replace_br_with_p heuristic processing feature.  Turn on in personal User Config.  Thanks, Asbjørn Grandt.</li>
 	    </ul>
 	  </p>
-           -->
-          <h3>fanfiction.net</h3>
-          <p>
-            Fanfiction.net appears to be blocking access from Google
-            App Engine IPs, which prevents this web service.  There's
-            nothing I can do about it.  At the time of writing, the
-            latest CLI and calibre plugin versions worked.
-          </p>
-	  
+          
 	  <p>
 	    Questions?  Check out our 
 	    <a href="http://code.google.com/p/fanficdownloader/wiki/FanFictionDownloaderFAQs">FAQs</a>.
@@ -82,7 +81,7 @@
 	    If you have any problems with this application, please
 	    report them in
 	    the <a href="http://groups.google.com/group/fanfic-downloader">FanFictionDownLoader Google Group</a>.  The
-	    <a href="http://4-4-67.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
+	    <a href="http://4-4-81.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
 	  </p>
 	  <div id='error'>
 	    {{ error_message }}

plugin-defaults.ini

@@ -142,6 +142,14 @@ extratags: FanFiction
 ## Add this to genre if there's more than one category.
 #add_genre_when_multi_category: Crossover
 
+## default_value_(entry) can be used to set the value for a metadata
+## entry when no value has been found on the site.  For example, some
+## sites doesn't have a status metadatum.  If uncommented, this will
+## use 'Unknown' for status when no status is found.
+#default_value_status:Unknown
+## Can also be used for other metadata values
+#default_value_category:FanFiction
+
 ## number of seconds to sleep between calls to the story site.  May by
 ## useful if pulling large numbers of stories or if the site is slow.
 #slow_down_sleep_time:0.5
@@ -179,6 +187,11 @@ extratags: FanFiction
 ## doesn't work on some devices either.)
 #replace_hr: false
 
+## Some sites/authors/stories use br tags instead of p tags for
+## paragraphs.  This feature uses some heuristics to find and replace
+## br paragraphs with p tags while preserving scene breaks.
+#replace_br_with_p: false
+
 ## If you have the Generate Cover plugin installed, you can use the
 ## generate_cover_settings parameter to intelligently decide which GC
 ## setting to run.  There are three parts 1) a template of which
@@ -265,8 +278,7 @@ sort_ships:false
 #keep_in_order_author:true
 
 ## User-agent
-#user_agent:FFDL/1.7
-
+user_agent:FFDL/1.7
 
 ## Each output format has a section that overrides [defaults]
 [html]
@@ -746,6 +758,23 @@ extraships:Harry Potter/Hermione Granger
 #username:YourName
 #password:yourpassword
 
+[fictionpad.com]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+extra_valid_entries:followers,comments,views,likes,dislikes
+#extra_titlepage_entries:followers,comments,views,likes,dislikes
+
+followers_label:Followers
+comments_label:Comments
+views_label:Views
+likes_label:Likes
+dislikes_label:Dislikes
+
 [finestories.com]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -754,6 +783,24 @@ extraships:Harry Potter/Hermione Granger
 #username:YourName
 #password:yourpassword
 
+[storiesonline.net]
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
+## Clear FanFiction from defaults, site is original fiction.
+extratags:
+
+extra_valid_entries:size,universe,codes
+#extra_titlepage_entries:size,universe,codes
+
+size_label:Size
+universe_label:Universe
+codes_label:Codes
+
 [grangerenchanted.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter
@@ -796,6 +843,10 @@ extracategories:Highlander
 ## Site dedicated to these categories/characters/ships
 extracategories:In Death
 
+## some sites include images that we don't ever want becoming the
+## cover image.  This lets you exclude them.
+cover_exclusion_regexp:/public/style_emoticons/.*
+
 [ksarchive.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:Star Trek
@@ -1103,6 +1154,7 @@ context_label:Context
 type_label:Type of Couple
 
 [www.fanfiction.net]
+user_agent:
 ## fanfiction.net's 'cover' images are really just tiny thumbnails.
 ## Change this to false to use them anyway.
 never_make_cover: true
@@ -1111,6 +1163,10 @@ never_make_cover: true
 ## for examples of how to use them.
 extra_valid_entries:reviews,favs,follows
 
+## ffnet uses 'Pairings', not 'Relationship', stating they don't have
+## to be romantic pairings.
+ships_label:Pairings
+
 [www.fanfiktion.de]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In
@@ -1130,7 +1186,12 @@ extracategories:Harry Potter
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+## fictionalley.org doesn't have a status metadatum.  If uncommented,
+## this will be used for status.
+#default_value_status:Unknown
+
 [www.fictionpress.com]
+user_agent:
 ## Clear FanFiction from defaults, fictionpress.com is original fiction.
 extratags:
 
@@ -1152,12 +1213,13 @@ extracategories:My Little Pony: Friendship is Magic
 
 ## Extra metadata that this adapter knows about.  See [dramione.org]
 ## for examples of how to use them.
-extra_valid_entries:likes,dislikes,views,total_views,short_description
+extra_valid_entries:likes,dislikes,views,total_views,short_description,groups
 likes_label:Likes
 dislikes_label:Dislikes
 views_label:Highest Single Chapter Views
 total_views_label:Total Views
 short_description_label:Short Summary
+groups_label:Groups
 
 ## Some sites do not require a login, but do require the user to
 ## confirm they are adult for adult content.  In commandline version,
@@ -1169,6 +1231,13 @@ short_description_label:Short Summary
 ## when a password is required rather than prompting every time.
 #fail_on_password: false
 
+## fimfiction.net stories allow chapters to be added out of order.  So
+## the newest chapter may not be the last one.  FFDL update doesn't
+## like that.  If do_update_hook is uncommented and set true, the
+## adapter will discard all existing chapters from the newest one on
+## when updating to enforce accurate chapters.
+#do_update_hook:false
+
 [www.harrypotterfanfiction.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:Harry Potter

status.html

@@ -51,7 +51,17 @@
 	      by {{ fic.author }} ({{ fic.format }})
 	  {% endif %}
 	  {% if fic.failure %}
-	      <span id='error'>{{ fic.failure }}</span>
+          <h3>fanfiction.net / fimfiction.net</h3>
+          <p>
+            FYI, fanfiction.net appears to be blocking access from Google
+            App Engine, which prevents this web service.  There's
+            nothing I can do about it.  At the time of writing, the
+            latest CLI and calibre plugin versions worked.
+          </p>
+          <p>It appears that FimFiction.net is also blocking access from Google
+            App Engine now.
+          </p>
+	  <span id='error'>{{ fic.failure }}</span>
 	  {% endif %}
 	  {% if  not fic.completed and not fic.failure %}
 	      <p>Not done yet.  This page will periodically poll to see if your story has finished.</p>