Add *plugin* CLI via calibre's cli_main feature. --PI only. (list continues)

Add "marked:ffdl_success/failed" to added/updated/failed books. --PI only. Add "Show added/updated" pref using above. --PI only. Preserve log page in epub across overwrites as well as updates. --PI only. Add "add_to_" feature to ini config. Allow higher priority sections to *add* to any ini param rather than replace it.
2026-09-10 11:40:24 +08:00 · 2013-03-14 17:16:26 -05:00
parent fdb45183c7
commit 8cd3663d82
12 changed files with 326 additions and 271 deletions
@@ -80,11 +80,29 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
        if ac is not None:
            ac.apply_settings()

-# For testing, run from command line with this:
-# calibre-debug -e __init__.py
-# 
-if __name__ == '__main__':
-    from PyQt4.Qt import QApplication
-    from calibre.gui2.preferences import test_widget
-    app = QApplication([])
-    test_widget('Advanced', 'Plugins')
+    def cli_main(self,argv):
+        # I believe there's no performance hit loading these here when
+        # CLI--it would load everytime anyway.
+        from StringIO import StringIO
+        from calibre.library import db
+        from calibre_plugins.fanfictiondownloader_plugin.downloader import main as ffdl_main
+        from calibre_plugins.fanfictiondownloader_plugin.prefs import PrefsFacade
+        from calibre.utils.config import OptionParser, prefs as calibre_prefs
+
+        parser = OptionParser('%prog --run-plugin FanFictionDownLoader -- [options] <storyurl>')
+        go = parser.add_option_group(_('GLOBAL OPTIONS'))
+        go.add_option('--library-path', '--with-library', default=None, help=_('Path to the calibre library. Default is to use the path stored in the settings.'))
+        # go.add_option('--dont-notify-gui', default=False, action='store_true',
+        #               help=_('Do not notify the running calibre GUI (if any) that the database has'
+        #                      ' changed. Use with care, as it can lead to database corruption!'))
+
+        pargs = [x for x in argv if x.startswith('--with-library') or x.startswith('--library-path')
+                 or not x.startswith('-')]
+        opts, args = parser.parse_args(pargs)
+
+        ffdl_prefs = PrefsFacade(db(path=opts.library_path,
+                                    read_only=True))
+        ffdl_main(argv[1:],
+                  parser=parser,
+                  passed_defaultsini=StringIO(get_resources("defaults.ini")),
+                  passed_personalini=StringIO(ffdl_prefs["personal.ini"]))
@@ -15,10 +15,10 @@ from PyQt4.Qt import (QDialog, QWidget, QVBoxLayout, QHBoxLayout, QLabel,
                      QCheckBox, QPushButton, QTabWidget, QVariant, QScrollArea,
                      QDialogButtonBox )

-from calibre.gui2 import dynamic, info_dialog
 from calibre.utils.config import JSONConfig
 from calibre.gui2.ui import get_gui

+from calibre_plugins.fanfictiondownloader_plugin.prefs import prefs
 from calibre_plugins.fanfictiondownloader_plugin.dialogs \
    import (UPDATE, UPDATEALWAYS, OVERWRITE, collision_order, RejectListDialog,
            EditTextDialog)
@@ -27,126 +27,10 @@ from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.adapters \
    import (getConfigSections, getNormalStoryURL)

 from calibre_plugins.fanfictiondownloader_plugin.common_utils \
-    import ( get_library_uuid, KeyboardConfigDialog, PrefsViewerDialog )
+    import ( KeyboardConfigDialog, PrefsViewerDialog )

 from calibre.gui2.complete import MultiCompleteLineEdit

-PREFS_NAMESPACE = 'FanFictionDownLoaderPlugin'
-PREFS_KEY_SETTINGS = 'settings'
-
-# Set defaults used by all.  Library specific settings continue to
-# take from here.
-default_prefs = {}
-default_prefs['personal.ini'] = get_resources('plugin-example.ini')
-default_prefs['rejecturls'] = ''
-default_prefs['rejectreasons'] = '''Sucked
-Boring
-Dup from another site'''
-
-default_prefs['updatemeta'] = True
-default_prefs['updatecover'] = False
-default_prefs['updateepubcover'] = False
-default_prefs['keeptags'] = False
-default_prefs['urlsfromclip'] = True
-default_prefs['updatedefault'] = True
-default_prefs['fileform'] = 'epub'
-default_prefs['collision'] = OVERWRITE
-default_prefs['deleteotherforms'] = False
-default_prefs['adddialogstaysontop'] = False
-default_prefs['includeimages'] = False
-default_prefs['lookforurlinhtml'] = False
-default_prefs['injectseries'] = False
-
-default_prefs['send_lists'] = ''
-default_prefs['read_lists'] = ''
-default_prefs['addtolists'] = False
-default_prefs['addtoreadlists'] = False
-default_prefs['addtolistsonread'] = False
-
-default_prefs['gcnewonly'] = False
-default_prefs['gc_site_settings'] = {}
-default_prefs['allow_gc_from_ini'] = True
-
-default_prefs['countpagesstats'] = []
-
-default_prefs['errorcol'] = ''
-default_prefs['custom_cols'] = {}
-default_prefs['custom_cols_newonly'] = {}
-default_prefs['allow_custcol_from_ini'] = True
-
-default_prefs['std_cols_newonly'] = {}
-
-def set_library_config(library_config):
-    get_gui().current_db.prefs.set_namespaced(PREFS_NAMESPACE,
-                                              PREFS_KEY_SETTINGS,
-                                              library_config)
-    
-def get_library_config():
-    db = get_gui().current_db
-    library_id = get_library_uuid(db)
-    library_config = None
-    # Check whether this is a configuration needing to be migrated
-    # from json into database.  If so: get it, set it, rename it in json.
-    if library_id in old_prefs:
-        #print("get prefs from old_prefs")
-        library_config = old_prefs[library_id]
-        set_library_config(library_config)
-        old_prefs["migrated to library db %s"%library_id] = old_prefs[library_id]
-        del old_prefs[library_id]
-
-    if library_config is None:
-        #print("get prefs from db")
-        library_config = db.prefs.get_namespaced(PREFS_NAMESPACE, PREFS_KEY_SETTINGS,
-                                                 copy.deepcopy(default_prefs))
-    return library_config
-
-# This is where all preferences for this plugin *were* stored
-# Remember that this name (i.e. plugins/fanfictiondownloader_plugin) is also
-# in a global namespace, so make it as unique as possible.
-# You should always prefix your config file name with plugins/,
-# so as to ensure you dont accidentally clobber a calibre config file
-old_prefs = JSONConfig('plugins/fanfictiondownloader_plugin')
-
-# fake out so I don't have to change the prefs calls anywhere.  The
-# Java programmer in me is offended by op-overloading, but it's very
-# tidy.
-class PrefsFacade():
-    def __init__(self,default_prefs):
-        self.default_prefs = default_prefs
-        self.libraryid = None
-        self.current_prefs = None
-        
-    def _get_prefs(self):
-        libraryid = get_library_uuid(get_gui().current_db)
-        if self.current_prefs == None or self.libraryid != libraryid:
-            #print("self.current_prefs == None(%s) or self.libraryid != libraryid(%s)"%(self.current_prefs == None,self.libraryid != libraryid))
-            self.libraryid = libraryid
-            self.current_prefs = get_library_config()
-        return self.current_prefs
-        
-    def __getitem__(self,k):            
-        prefs = self._get_prefs()
-        if k not in prefs:
-            # pulls from default_prefs.defaults automatically if not set
-            # in default_prefs
-            return self.default_prefs[k]
-        return prefs[k]
-
-    def __setitem__(self,k,v):
-        prefs = self._get_prefs()
-        prefs[k]=v
-        # self._save_prefs(prefs)
-
-    def __delitem__(self,k):
-        prefs = self._get_prefs()
-        if k in prefs:
-            del prefs[k]
-
-    def save_to_db(self):
-        set_library_config(self._get_prefs())
-        
-prefs = PrefsFacade(default_prefs)
-
 class RejectURLList:
    def __init__(self,prefs):
        self.prefs = prefs
@@ -282,6 +166,7 @@ class ConfigWidget(QWidget):
        prefs['updatecover'] = self.basic_tab.updatecover.isChecked()
        prefs['updateepubcover'] = self.basic_tab.updateepubcover.isChecked()
        prefs['keeptags'] = self.basic_tab.keeptags.isChecked()
+        prefs['showmarked'] = self.basic_tab.showmarked.isChecked()
        prefs['urlsfromclip'] = self.basic_tab.urlsfromclip.isChecked()
        prefs['updatedefault'] = self.basic_tab.updatedefault.isChecked()
        prefs['deleteotherforms'] = self.basic_tab.deleteotherforms.isChecked()
@@ -448,6 +333,13 @@ class BasicTab(QWidget):

        self.l.addSpacing(10)        

+        self.showmarked = QCheckBox("Show added/updated books when finished?",self)
+        self.showmarked.setToolTip("Show added/updated books only when finished.\nYou can also manually search for 'marked:ffdl_success'.\n'marked:ffdl_failed' is also available, or search 'marked:ffdl' for both.")
+        self.showmarked.setChecked(prefs['showmarked'])
+        self.l.addWidget(self.showmarked)
+
+        self.l.addSpacing(10)        
+
        self.urlsfromclip = QCheckBox('Take URLs from Clipboard?',self)
        self.urlsfromclip.setToolTip('Prefill URLs from valid URLs in Clipboard when Adding New.')
        self.urlsfromclip.setChecked(prefs['urlsfromclip'])
@@ -557,84 +557,6 @@ class UpdateExistingDialog(SizePersistedDialog):
            'updateepubcover': self.updateepubcover.isChecked(),
            }

-def display_story_list(gui, header, prefs, icon, books,
-                       label_text='',
-                       save_size_name='fanfictiondownloader_plugin:display list dialog',
-                       offer_skip=False):
-    all_good = True
-    for b in books:
-        if not b['good']:
-            all_good=False
-            break
-        
-    ## 
-    if all_good and not dynamic.get(confirm_config_name(save_size_name), True):
-        return True
-        pass
-            ## fake accept?
-    d = DisplayStoryListDialog(gui, header, prefs, icon, books,
-                               label_text,
-                               save_size_name,
-                               offer_skip and all_good)
-    d.exec_()
-    return d.result() == d.Accepted
-
-class DisplayStoryListDialog(SizePersistedDialog):
-    def __init__(self, gui, header, prefs, icon, books,
-                 label_text='',
-                 save_size_name='fanfictiondownloader_plugin:display list dialog',
-                 offer_skip=False):
-        SizePersistedDialog.__init__(self, gui, save_size_name)
-        self.name = save_size_name
-        self.gui = gui
-        
-        self.setWindowTitle(header)
-        self.setWindowIcon(icon)
-        
-        layout = QVBoxLayout(self)
-        self.setLayout(layout)
-        title_layout = ImageTitleLayout(self, 'images/icon.png',
-                                        header)
-        layout.addLayout(title_layout)
-
-        self.books_table = StoryListTableWidget(self)
-        layout.addWidget(self.books_table)
-
-        options_layout = QHBoxLayout()
-        self.label = QLabel(label_text)
-        #self.label.setOpenExternalLinks(True)
-        #self.label.setWordWrap(True)
-        options_layout.addWidget(self.label)
-        
-        if offer_skip:
-            spacerItem1 = QtGui.QSpacerItem(2, 4, QtGui.QSizePolicy.Expanding, QtGui.QSizePolicy.Minimum)
-            options_layout.addItem(spacerItem1)
-            self.again = QCheckBox('Show this again?',self)
-            self.again.setChecked(True)
-            self.again.stateChanged.connect(self.toggle)
-            self.again.setToolTip('Uncheck to skip review and update stories immediately when no problems.')
-            options_layout.addWidget(self.again)
-        
-        button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
-        button_box.accepted.connect(self.accept)
-        button_box.rejected.connect(self.reject)
-
-        options_layout.addWidget(button_box)
-        
-        layout.addLayout(options_layout)
-        
-        # Cause our dialog size to be restored from prefs or created on first usage
-        self.resize_dialog()
-        self.books_table.populate_table(books)
-
-    def get_books(self):
-        return self.books_table.get_books()
-
-    def toggle(self, *args):
-        dynamic[confirm_config_name(self.name)] = self.again.isChecked()
-
-    
-
 class StoryListTableWidget(QTableWidget):

    def __init__(self, parent):
@@ -41,9 +41,10 @@ from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.epubutils impo
 from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.geturls import get_urls_from_page

 from calibre_plugins.fanfictiondownloader_plugin.ffdl_util import (get_ffdl_adapter, get_ffdl_config, get_ffdl_personalini)
-from calibre_plugins.fanfictiondownloader_plugin.config import (prefs, permitted_values, rejecturllist)
+from calibre_plugins.fanfictiondownloader_plugin.config import (permitted_values, rejecturllist)
+from calibre_plugins.fanfictiondownloader_plugin.prefs import prefs
 from calibre_plugins.fanfictiondownloader_plugin.dialogs import (
-    AddNewDialog, UpdateExistingDialog, display_story_list, DisplayStoryListDialog,
+    AddNewDialog, UpdateExistingDialog,
    LoopProgressDialog, UserPassDialog, AboutDialog, CollectURLDialog, RejectListDialog,
    OVERWRITE, OVERWRITEALWAYS, UPDATE, UPDATEALWAYS, ADDNEW, SKIP, CALIBREONLY,
    NotGoingToDownload )
@@ -551,7 +552,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
        
        for j, book in enumerate(update_books):
            url = book['url']
-            book['mergeorder'] = j
+            book['listorder'] = j
            if url in urlmapfile:
                #print("found epub for %s"%url)
                book['epub_for_update']=urlmapfile[url]
@@ -596,6 +597,9 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
        book_list = map( partial(self.make_book_id_only), self.gui.library_view.get_selected_ids() )
        #book_ids = self.gui.library_view.get_selected_ids()

+        for j, book in enumerate(book_list):
+            book['listorder'] = j
+            
        LoopProgressDialog(self.gui,
                           book_list,
                           partial(self.populate_book_from_calibre_id, db=self.gui.current_db),
@@ -872,8 +876,9 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                        raise NotGoingToDownload("Not Overwriting, web site is not newer.",'edit-undo.png')
        
                # For update, provide a tmp file copy of the existing epub so
-                # it can't change underneath us.
-                if collision in (UPDATE,UPDATEALWAYS) and \
+                # it can't change underneath us.  Now also overwrite for logpage preserve.
+                if collision in (UPDATE,UPDATEALWAYS,OVERWRITE,OVERWRITEALWAYS) and \
+                        fileform == 'epub' and \
                        db.has_format(book['calibre_id'],'EPUB',index_is_id=True):
                    tmp = PersistentTemporaryFile(prefix='old-%s-'%book['calibre_id'],
                                                  suffix='.epub',
@@ -930,18 +935,28 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                break
        else:
            ## No good stories to try to download, go straight to
-            ## list.
-            d = DisplayStoryListDialog(self.gui,
-                                       'Nothing to Download',
-                                       prefs,
-                                       self.qaction.icon(),
-                                       book_list,
-                                       label_text='None of the URLs/stories given can be/need to be downloaded.'
-                                       )
-            d.exec_()
-
-            self.update_error_column(book_list,options)
+            ## updating error col.
+            msg = '''
+<p>None of the <b>%d</b> URLs/stories given can be/need to be downloaded.</p>
+<p>See log for details.</p>
+<p>Proceed with updating your library(Error Column, if configured)?</p>
+'''%len(book_list)
+    
+            htmllog='<html><body><table border="1"><tr><th>Status</th><th>Title</th><th>Author</th><th>Comment</th><th>URL</th></tr>'
+            for book in book_list:
+                if 'status' in book:
+                    status = book['status']
+                else:
+                    status = 'Bad'
+                htmllog = htmllog + '<tr><td>' + '</td><td>'.join([escapehtml(status),escapehtml(book['title']),escapehtml(", ".join(book['author'])),escapehtml(book['comment']),book['url']]) + '</td></tr>'
            
+            htmllog = htmllog + '</table></body></html>'
+
+            payload = ([], book_list, options)
+            self.gui.proceed_question(self.update_error_column,
+                                      payload, htmllog,
+                                      'FFDL log', 'FFDL download ended', msg,
+                                      show_copy_button=False)
            return
            
        func = 'arbitrary_n'
@@ -961,6 +976,18 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                              'collision':ADDNEW,
                              'updatemeta':True,
                              'updateepubcover':True}):
+        custom_columns = self.gui.library_view.model().custom_columns
+        if book['calibre_id'] and prefs['errorcol'] != '' and prefs['errorcol'] in custom_columns:
+            label = custom_columns[prefs['errorcol']]['label']
+            if not book['good']:
+                print("record/update error message column %s %s"%(book['title'],book['url']))
+                db.set_custom(book['calibre_id'], book['comment'], label=label, commit=True) # book['comment']
+            else:
+                db.set_custom(book['calibre_id'], '', label=label, commit=True) # book['comment']
+                
+        if not book['good']:
+            return # only update errorcol on error.
+        
        print("add/update %s %s"%(book['title'],book['url']))
        mi = self.make_mi_from_book(book)
                
@@ -979,8 +1006,10 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
        add_ids = [ x['calibre_id'] for x in add_list ]
        update_list = filter(lambda x : x['good'] and not x['added'], book_list)
        update_ids = [ x['calibre_id'] for x in update_list ]
-        all_ids = add_ids
-        all_ids.extend(update_ids)
+        all_ids = add_ids + update_ids
+
+        failed_list = filter(lambda x : not x['good'] , book_list)
+        failed_ids = [ x['calibre_id'] for x in failed_list ]

        if options['collision'] != CALIBREONLY and \
                (prefs['addtolists'] or prefs['addtoreadlists']):
@@ -999,19 +1028,26 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
        
        if self.gui.cover_flow:
            self.gui.cover_flow.dataChanged()
-        
+
+        if showlist: # don't use with anthology
+            db = self.gui.current_db
+            marked_ids = dict()
+            marked_text = "ffdl_success"
+            for index, book_id in enumerate(all_ids):
+                marked_ids[book_id] = '%s_%04d' % (marked_text, index)
+            for index, book_id in enumerate(failed_ids):
+                marked_ids[book_id] = 'ffdl_failed_%04d' % index
+            # Mark the results in our database
+            db.set_marked_ids(marked_ids)
+            
+            if prefs['showmarked']: # show add/update
+                # Search to display the list contents
+                self.gui.search.set_search_string('marked:' + marked_text)
+                # Sort by our marked column to display the books in order
+                self.gui.library_view.sort_by_named_field('marked', True)
+                
        self.gui.status_bar.show_message(_('Finished Adding/Updating %d books.'%(len(update_list) + len(add_list))), 3000)
            
-        if showlist and (len(update_list) + len(add_list) != len(book_list)):
-            d = DisplayStoryListDialog(self.gui,
-                                       'Updates completed, final status',
-                                       prefs,
-                                       self.qaction.icon(),
-                                       book_list,
-                                       label_text='Stories have be added or updated in Calibre, some had additional problems.'
-                                       )
-            d.exec_()
-    
        print("all done, remove temp dir.")
        remove_dir(options['tdir'])

@@ -1030,6 +1066,8 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
        book_list = job.result
        good_list = filter(lambda x : x['good'], book_list)
        bad_list = filter(lambda x : not x['good'], book_list)
+        good_list = sorted(good_list,key=lambda x : x['listorder'])
+        bad_list = sorted(bad_list,key=lambda x : x['listorder'])
        #print("book_list:%s"%book_list)
        payload = (good_list, bad_list, options)
        
@@ -1051,7 +1089,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
            msg = msg + '<p>Proceed with updating this anthology and your library?</p>'
    
            htmllog='<html><body><table border="1"><tr><th>Status</th><th>Title</th><th>Author</th><th>Comment</th><th>URL</th></tr>'
-            for book in sorted(good_list+bad_list,key=lambda x : x['mergeorder']):
+            for book in sorted(good_list+bad_list,key=lambda x : x['listorder']):
                if 'status' in book:
                    status = book['status']
                else:
@@ -1106,10 +1144,10 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
        (good_list,bad_list,options) = payload
        total_good = len(good_list)

-        print("merge titles:\n%s"%"\n".join([ "%s %s"%(x['title'],x['mergeorder']) for x in good_list ]))
+        print("merge titles:\n%s"%"\n".join([ "%s %s"%(x['title'],x['listorder']) for x in good_list ]))

-        good_list = sorted(good_list,key=lambda x : x['mergeorder'])
-        bad_list = sorted(bad_list,key=lambda x : x['mergeorder'])
+        good_list = sorted(good_list,key=lambda x : x['listorder'])
+        bad_list = sorted(bad_list,key=lambda x : x['listorder'])
        
        self.gui.status_bar.show_message(_('Merging %s books.'%total_good))

@@ -1149,26 +1187,23 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
    def do_download_list_update(self, payload):
        
        (good_list,bad_list,options) = payload
-        total_good = len(good_list)
+        good_list = sorted(good_list,key=lambda x : x['listorder'])
+        bad_list = sorted(bad_list,key=lambda x : x['listorder'])

-        self.gui.status_bar.show_message(_('Adding/Updating %s books.'%total_good))
+        self.gui.status_bar.show_message(_('FFDL Adding/Updating books.'))

-        if total_good > 0:
+        if good_list or (bad_list and prefs['errorcol'] != '' and prefs['errorcol'] in self.gui.library_view.model().custom_columns):
            LoopProgressDialog(self.gui,
-                               good_list,
+                               good_list+bad_list,
                               partial(self.update_books_loop, options=options, db=self.gui.current_db),
                               partial(self.update_books_finish, options=options),
                               init_label="Updating calibre for FanFiction stories...",
                               win_title="Update calibre for FanFiction stories",
                               status_prefix="Updated")

-        total_bad = len(bad_list)
-
-        if total_bad > 0:
-            self.update_error_column(bad_list,options)
-
-    def update_error_column(self,book_list,options):
+    def update_error_column(self,payload):
        '''Update custom error column if configured.'''
+        (empty_list,book_list,options)=payload
        custom_columns = self.gui.library_view.model().custom_columns
        if prefs['errorcol'] != '' and prefs['errorcol'] in custom_columns:
            self.previous = self.gui.library_view.currentIndex() # used by update_books_finish.
@@ -1391,11 +1426,6 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                realmi = db.get_metadata(book_id, index_is_id=True)
                gc_plugin.generate_cover_for_book(realmi,saved_setting_name=setting_name)

-        ## if error column set.
-        if prefs['errorcol'] != '' and prefs['errorcol'] in custom_columns:
-            label = custom_columns[prefs['errorcol']]['label']
-            db.set_custom(book['calibre_id'], '', label=label, commit=True) # book['comment']
-
    def get_clean_reading_lists(self,lists):
        if lists == None or lists.strip() == "" :
            return []
@@ -1488,8 +1518,8 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                book['good'] = False
                book['comment'] = "Same story already included."
            uniqueurls.add(book['url'])
-            book['mergeorder']=i # BG d/l jobs don't come back in order.
-                                 # Didn't matter until anthologies.
+            book['listorder']=i # BG d/l jobs don't come back in order.
+                                # Didn't matter until anthologies & 'marked' successes
            books.append(book)
        return books

@@ -11,7 +11,7 @@ from StringIO import StringIO

 from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapters, exceptions
 from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.configurable import Configuration
-from calibre_plugins.fanfictiondownloader_plugin.config import (prefs)
+from calibre_plugins.fanfictiondownloader_plugin.prefs import (prefs)

 def get_ffdl_personalini():
    if prefs['includeimages']:
@@ -153,6 +153,22 @@ def do_download_for_worker(book,options):
        elif options['collision'] in (ADDNEW, SKIP, OVERWRITE, OVERWRITEALWAYS) or \
                ('epub_for_update' not in book and options['collision'] in (UPDATE, UPDATEALWAYS)):

+            # preserve logfile even on overwrite.
+            if 'epub_for_update' in book:
+                (urlignore,
+                 chaptercountignore,
+                 oldchaptersignore,
+                 oldimgsignore,
+                 oldcoverignore,
+                 calibrebookmarkignore,
+                 # only logfile set in adapter, so others aren't used.
+                 adapter.logfile) = get_update_data(book['epub_for_update'])
+
+                # change the existing entries id to notid so
+                # write_epub writes a whole new set to indicate overwrite.
+                if adapter.logfile:
+                    adapter.logfile = adapter.logfile.replace("span id","span notid")
+            
            print("write to %s"%outfile)
            writer.writeStory(outfilename=outfile, forceOverwrite=True)
            book['comment'] = 'Download %s completed, %s chapters.'%(options['fileform'],story.getMetadata("numChapters"))
@@ -0,0 +1,141 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2013, Jim Miller'
+__docformat__ = 'restructuredtext en'
+
+import copy
+
+from calibre.utils.config import JSONConfig
+from calibre.gui2.ui import get_gui
+
+from calibre_plugins.fanfictiondownloader_plugin.dialogs import OVERWRITE
+from calibre_plugins.fanfictiondownloader_plugin.common_utils import get_library_uuid
+PREFS_NAMESPACE = 'FanFictionDownLoaderPlugin'
+PREFS_KEY_SETTINGS = 'settings'
+
+# Set defaults used by all.  Library specific settings continue to
+# take from here.
+default_prefs = {}
+default_prefs['personal.ini'] = get_resources('plugin-example.ini')
+default_prefs['rejecturls'] = ''
+default_prefs['rejectreasons'] = '''Sucked
+Boring
+Dup from another site'''
+
+default_prefs['updatemeta'] = True
+default_prefs['updatecover'] = False
+default_prefs['updateepubcover'] = False
+default_prefs['keeptags'] = False
+default_prefs['showmarked'] = False
+default_prefs['urlsfromclip'] = True
+default_prefs['updatedefault'] = True
+default_prefs['fileform'] = 'epub'
+default_prefs['collision'] = OVERWRITE
+default_prefs['deleteotherforms'] = False
+default_prefs['adddialogstaysontop'] = False
+default_prefs['includeimages'] = False
+default_prefs['lookforurlinhtml'] = False
+default_prefs['injectseries'] = False
+
+default_prefs['send_lists'] = ''
+default_prefs['read_lists'] = ''
+default_prefs['addtolists'] = False
+default_prefs['addtoreadlists'] = False
+default_prefs['addtolistsonread'] = False
+
+default_prefs['gcnewonly'] = False
+default_prefs['gc_site_settings'] = {}
+default_prefs['allow_gc_from_ini'] = True
+
+default_prefs['countpagesstats'] = []
+
+default_prefs['errorcol'] = ''
+default_prefs['custom_cols'] = {}
+default_prefs['custom_cols_newonly'] = {}
+default_prefs['allow_custcol_from_ini'] = True
+
+default_prefs['std_cols_newonly'] = {}
+
+def set_library_config(library_config,db):
+    db.prefs.set_namespaced(PREFS_NAMESPACE,
+                            PREFS_KEY_SETTINGS,
+                            library_config)
+
+def get_library_config(db):
+    library_id = get_library_uuid(db)
+    library_config = None
+    # Check whether this is a configuration needing to be migrated
+    # from json into database.  If so: get it, set it, rename it in json.
+    if library_id in old_prefs:
+        #print("get prefs from old_prefs")
+        library_config = old_prefs[library_id]
+        set_library_config(library_config)
+        old_prefs["migrated to library db %s"%library_id] = old_prefs[library_id]
+        del old_prefs[library_id]
+
+    if library_config is None:
+        #print("get prefs from db")
+        library_config = db.prefs.get_namespaced(PREFS_NAMESPACE, PREFS_KEY_SETTINGS,
+                                                 copy.deepcopy(default_prefs))
+    return library_config
+
+# This is where all preferences for this plugin *were* stored
+# Remember that this name (i.e. plugins/fanfictiondownloader_plugin) is also
+# in a global namespace, so make it as unique as possible.
+# You should always prefix your config file name with plugins/,
+# so as to ensure you dont accidentally clobber a calibre config file
+old_prefs = JSONConfig('plugins/fanfictiondownloader_plugin')
+
+# fake out so I don't have to change the prefs calls anywhere.  The
+# Java programmer in me is offended by op-overloading, but it's very
+# tidy.
+class PrefsFacade():
+    def _get_db(self):
+        if self.passed_db:
+            return self.passed_db
+        else:
+            # In the GUI plugin we want current db so we detect when
+            # it's changed.  CLI plugin calls need to pass db in.
+            return get_gui().current_db
+    
+    def __init__(self,passed_db=None):
+        self.default_prefs = default_prefs
+        self.libraryid = None
+        self.current_prefs = None
+        self.passed_db=passed_db
+        
+    def _get_prefs(self):
+        libraryid = get_library_uuid(self._get_db())
+        if self.current_prefs == None or self.libraryid != libraryid:
+            #print("self.current_prefs == None(%s) or self.libraryid != libraryid(%s)"%(self.current_prefs == None,self.libraryid != libraryid))
+            self.libraryid = libraryid
+            self.current_prefs = get_library_config(self._get_db())
+        return self.current_prefs
+        
+    def __getitem__(self,k):            
+        prefs = self._get_prefs()
+        if k not in prefs:
+            # pulls from default_prefs.defaults automatically if not set
+            # in default_prefs
+            return self.default_prefs[k]
+        return prefs[k]
+
+    def __setitem__(self,k,v):
+        prefs = self._get_prefs()
+        prefs[k]=v
+        # self._save_prefs(prefs)
+
+    def __delitem__(self,k):
+        prefs = self._get_prefs()
+        if k in prefs:
+            del prefs[k]
+
+    def save_to_db(self):
+        set_library_config(self._get_prefs(),self._get_db())
+        
+prefs = PrefsFacade()
+
@@ -369,7 +369,7 @@ output_css:
 ## It can be either a 'file:' or 'http:' url.
 ## Note that if you enable make_firstimage_cover in [epub], but want
 ## to use default_cover_image for a specific site, use the site:format
-## section, for example: [www.ficwad.com:epub]
+## section, for example: [ficwad.com:epub]
 ## default_cover_image is a python string Template string with
 ## ${title}, ${author} etc, same as titlepage_entries. Unless
 ## allow_unsafe_filename is true, invalid filename chars will be
@@ -32,10 +32,17 @@ if sys.version_info >= (2, 7):
    loghandler.setFormatter(logging.Formatter("(=====)(levelname)s:%(message)s"))
    rootlogger.addHandler(loghandler)

-from fanficdownloader import adapters,writers,exceptions
-from fanficdownloader.configurable import Configuration
-from fanficdownloader.epubutils import get_dcsource_chaptercount, get_update_data
-from fanficdownloader.geturls import get_urls_from_page
+try:
+    from fanficdownloader import adapters,writers,exceptions
+    from fanficdownloader.configurable import Configuration
+    from fanficdownloader.epubutils import get_dcsource_chaptercount, get_update_data
+    from fanficdownloader.geturls import get_urls_from_page
+except:
+    # running under calibre
+    from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapters,writers,exceptions
+    from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.configurable import Configuration
+    from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.epubutils import get_dcsource_chaptercount, get_update_data
+    from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.geturls import get_urls_from_page

 if sys.version_info < (2, 5):
    print "This program requires Python 2.5 or newer."
@@ -48,15 +55,23 @@ def writeStory(config,adapter,writeformat,metaonly=False,outstream=None):
    del writer
    return output_filename

-def main():
+def main(argv,
+         parser=None,
+         passed_defaultsini=None,
+         passed_personalini=None):
   # read in args, anything starting with -- will be treated as --<varible>=<value>
-   usage = "usage: %prog [options] storyurl"
-   parser = OptionParser(usage)
+   if not parser:
+       parser = OptionParser("usage: %prog [options] storyurl")
   parser.add_option("-f", "--format", dest="format", default="epub",
-                     help="write story as FORMAT, epub(default), text or html", metavar="FORMAT")
+                     help="write story as FORMAT, epub(default), mobi, text or html", metavar="FORMAT")
+
+   if passed_defaultsini:
+       config_help="read config from specified file(s) in addition to calibre plugin personal.ini, ~/.fanficdownloader/personal.ini, and ./personal.ini"
+   else:
+       config_help="read config from specified file(s) in addition to ~/.fanficdownloader/defaults.ini, ~/.fanficdownloader/personal.ini, ./defaults.ini, and ./personal.ini"
   parser.add_option("-c", "--config",
                     action="append", dest="configfile", default=None,
-                     help="read config from specified file(s) in addition to ~/.fanficdownloader/defaults.ini, ~/.fanficdownloader/personal.ini, ./defaults.ini, ./personal.ini", metavar="CONFIG")
+                     help=config_help, metavar="CONFIG")
   parser.add_option("-b", "--begin", dest="begin", default=None,
                     help="Begin with Chapter START", metavar="START")
   parser.add_option("-e", "--end", dest="end", default=None,
@@ -83,7 +98,7 @@ def main():
                     action="store_true", dest="debug",
                     help="Show debug output while downloading.",)
   
-   (options, args) = parser.parse_args()
+   (options, args) = parser.parse_args(argv)

   if not options.debug:
       logger = logging.getLogger("fanficdownloader")
@@ -107,12 +122,18 @@ def main():

   conflist = []
   homepath = join(expanduser("~"),".fanficdownloader")
+
+   if passed_defaultsini:
+       configuration.readfp(passed_defaultsini)
   
   if isfile(join(homepath,"defaults.ini")):
       conflist.append(join(homepath,"defaults.ini"))
   if isfile("defaults.ini"):
       conflist.append("defaults.ini")
       
+   if passed_personalini:
+       configuration.readfp(passed_personalini)
+   
   if isfile(join(homepath,"personal.ini")):
       conflist.append(join(homepath,"personal.ini"))
   if isfile("personal.ini"):
@@ -240,5 +261,5 @@ def main():
 if __name__ == "__main__":
    #import time
    #start = time.time()
-    main()
+    main(sys.argv[1:])
    #print("Total time seconds:%f"%(time.time()-start))
@@ -94,7 +94,12 @@ class Configuration(ConfigParser.SafeConfigParser):
                #print("found %s in section [%s]"%(key,section))
                return True
            except:
-                pass
+                try:
+                    self.get(section,"add_to_"+key)
+                    #print("found add_to_%s in section [%s]"%(key,section))
+                    return True
+                except:
+                    pass

        return False
        
@@ -106,16 +111,24 @@ class Configuration(ConfigParser.SafeConfigParser):
                if val and val.lower() == "false":
                    val = False
                #print "getConfig(%s)=[%s]%s" % (key,section,val)
-                return val
+                break
            except (ConfigParser.NoOptionError, ConfigParser.NoSectionError), e:
                pass

+        for section in self.sectionslist[::-1]:
+            # 'martian smiley' [::-1] reverses list by slicing whole list with -1 step.
+            try:
+                val = val + self.get(section,"add_to_"+key)
+                #print "getConfig(add_to_%s)=[%s]%s" % (key,section,val)
+            except (ConfigParser.NoOptionError, ConfigParser.NoSectionError), e:
+                pass
+            
        return val

    # split and strip each.
    def getConfigList(self, key):
        vlist = self.getConfig(key).split(',')
-        vlist = [ v.strip() for v in vlist ]
+        vlist = filter( lambda x : x !='', [ v.strip() for v in vlist ])
        #print "vlist("+key+"):"+str(vlist)
        return vlist        

@@ -27,7 +27,7 @@ if __name__=="__main__":
    exclude=['*.pyc','*~','*.xcf','*[0-9].png']
    # from top dir. 'w' for overwrite
    createZipFile(filename,"w",
-                  ['plugin-defaults.ini','plugin-example.ini','fanficdownloader'],
+                  ['plugin-defaults.ini','plugin-example.ini','fanficdownloader','downloader.py','defaults.ini'],
                  exclude=exclude)
    #from calibre-plugin dir. 'a' for append
    os.chdir('calibre-plugin')
@@ -269,6 +269,8 @@ include_tocpage: false
 ## dateUpdated,numChapters,numWords at a minimum) will be shown.
 ## Great for tracking when chapters came out and when the description,
 ## etc changed.
+## Plugin will now preserve the log page when the epub is overwritten,
+## too.
 include_logpage: false
 ## If set to 'smart', logpage will only be included if the story is
 ## status:In-Progress or already had a logpage.  That way you don't