From 8cd3663d828b1ed032a7700daf618c7ae3e72375 Mon Sep 17 00:00:00 2001 From: Jim Miller Date: Thu, 14 Mar 2013 17:16:26 -0500 Subject: [PATCH] Add *plugin* CLI via calibre's cli_main feature. --PI only. (list continues) Add "marked:ffdl_success/failed" to added/updated/failed books. --PI only. Add "Show added/updated" pref using above. --PI only. Preserve log page in epub across overwrites as well as updates. --PI only. Add "add_to_" feature to ini config. Allow higher priority sections to *add* to any ini param rather than replace it. --- calibre-plugin/__init__.py | 34 ++++++-- calibre-plugin/config.py | 128 +++------------------------- calibre-plugin/dialogs.py | 78 ----------------- calibre-plugin/ffdl_plugin.py | 130 +++++++++++++++++----------- calibre-plugin/ffdl_util.py | 2 +- calibre-plugin/jobs.py | 16 ++++ calibre-plugin/prefs.py | 141 +++++++++++++++++++++++++++++++ defaults.ini | 2 +- downloader.py | 43 +++++++--- fanficdownloader/configurable.py | 19 ++++- makeplugin.py | 2 +- plugin-defaults.ini | 2 + 12 files changed, 326 insertions(+), 271 deletions(-) create mode 100644 calibre-plugin/prefs.py diff --git a/calibre-plugin/__init__.py b/calibre-plugin/__init__.py index 51da39f..0fccc23 100644 --- a/calibre-plugin/__init__.py +++ b/calibre-plugin/__init__.py @@ -80,11 +80,29 @@ class FanFictionDownLoaderBase(InterfaceActionBase): if ac is not None: ac.apply_settings() -# For testing, run from command line with this: -# calibre-debug -e __init__.py -# -if __name__ == '__main__': - from PyQt4.Qt import QApplication - from calibre.gui2.preferences import test_widget - app = QApplication([]) - test_widget('Advanced', 'Plugins') + def cli_main(self,argv): + # I believe there's no performance hit loading these here when + # CLI--it would load everytime anyway. + from StringIO import StringIO + from calibre.library import db + from calibre_plugins.fanfictiondownloader_plugin.downloader import main as ffdl_main + from calibre_plugins.fanfictiondownloader_plugin.prefs import PrefsFacade + from calibre.utils.config import OptionParser, prefs as calibre_prefs + + parser = OptionParser('%prog --run-plugin FanFictionDownLoader -- [options] ') + go = parser.add_option_group(_('GLOBAL OPTIONS')) + go.add_option('--library-path', '--with-library', default=None, help=_('Path to the calibre library. Default is to use the path stored in the settings.')) + # go.add_option('--dont-notify-gui', default=False, action='store_true', + # help=_('Do not notify the running calibre GUI (if any) that the database has' + # ' changed. Use with care, as it can lead to database corruption!')) + + pargs = [x for x in argv if x.startswith('--with-library') or x.startswith('--library-path') + or not x.startswith('-')] + opts, args = parser.parse_args(pargs) + + ffdl_prefs = PrefsFacade(db(path=opts.library_path, + read_only=True)) + ffdl_main(argv[1:], + parser=parser, + passed_defaultsini=StringIO(get_resources("defaults.ini")), + passed_personalini=StringIO(ffdl_prefs["personal.ini"])) diff --git a/calibre-plugin/config.py b/calibre-plugin/config.py index 17a02bb..4ebfeb7 100644 --- a/calibre-plugin/config.py +++ b/calibre-plugin/config.py @@ -15,10 +15,10 @@ from PyQt4.Qt import (QDialog, QWidget, QVBoxLayout, QHBoxLayout, QLabel, QCheckBox, QPushButton, QTabWidget, QVariant, QScrollArea, QDialogButtonBox ) -from calibre.gui2 import dynamic, info_dialog from calibre.utils.config import JSONConfig from calibre.gui2.ui import get_gui +from calibre_plugins.fanfictiondownloader_plugin.prefs import prefs from calibre_plugins.fanfictiondownloader_plugin.dialogs \ import (UPDATE, UPDATEALWAYS, OVERWRITE, collision_order, RejectListDialog, EditTextDialog) @@ -27,126 +27,10 @@ from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.adapters \ import (getConfigSections, getNormalStoryURL) from calibre_plugins.fanfictiondownloader_plugin.common_utils \ - import ( get_library_uuid, KeyboardConfigDialog, PrefsViewerDialog ) + import ( KeyboardConfigDialog, PrefsViewerDialog ) from calibre.gui2.complete import MultiCompleteLineEdit -PREFS_NAMESPACE = 'FanFictionDownLoaderPlugin' -PREFS_KEY_SETTINGS = 'settings' - -# Set defaults used by all. Library specific settings continue to -# take from here. -default_prefs = {} -default_prefs['personal.ini'] = get_resources('plugin-example.ini') -default_prefs['rejecturls'] = '' -default_prefs['rejectreasons'] = '''Sucked -Boring -Dup from another site''' - -default_prefs['updatemeta'] = True -default_prefs['updatecover'] = False -default_prefs['updateepubcover'] = False -default_prefs['keeptags'] = False -default_prefs['urlsfromclip'] = True -default_prefs['updatedefault'] = True -default_prefs['fileform'] = 'epub' -default_prefs['collision'] = OVERWRITE -default_prefs['deleteotherforms'] = False -default_prefs['adddialogstaysontop'] = False -default_prefs['includeimages'] = False -default_prefs['lookforurlinhtml'] = False -default_prefs['injectseries'] = False - -default_prefs['send_lists'] = '' -default_prefs['read_lists'] = '' -default_prefs['addtolists'] = False -default_prefs['addtoreadlists'] = False -default_prefs['addtolistsonread'] = False - -default_prefs['gcnewonly'] = False -default_prefs['gc_site_settings'] = {} -default_prefs['allow_gc_from_ini'] = True - -default_prefs['countpagesstats'] = [] - -default_prefs['errorcol'] = '' -default_prefs['custom_cols'] = {} -default_prefs['custom_cols_newonly'] = {} -default_prefs['allow_custcol_from_ini'] = True - -default_prefs['std_cols_newonly'] = {} - -def set_library_config(library_config): - get_gui().current_db.prefs.set_namespaced(PREFS_NAMESPACE, - PREFS_KEY_SETTINGS, - library_config) - -def get_library_config(): - db = get_gui().current_db - library_id = get_library_uuid(db) - library_config = None - # Check whether this is a configuration needing to be migrated - # from json into database. If so: get it, set it, rename it in json. - if library_id in old_prefs: - #print("get prefs from old_prefs") - library_config = old_prefs[library_id] - set_library_config(library_config) - old_prefs["migrated to library db %s"%library_id] = old_prefs[library_id] - del old_prefs[library_id] - - if library_config is None: - #print("get prefs from db") - library_config = db.prefs.get_namespaced(PREFS_NAMESPACE, PREFS_KEY_SETTINGS, - copy.deepcopy(default_prefs)) - return library_config - -# This is where all preferences for this plugin *were* stored -# Remember that this name (i.e. plugins/fanfictiondownloader_plugin) is also -# in a global namespace, so make it as unique as possible. -# You should always prefix your config file name with plugins/, -# so as to ensure you dont accidentally clobber a calibre config file -old_prefs = JSONConfig('plugins/fanfictiondownloader_plugin') - -# fake out so I don't have to change the prefs calls anywhere. The -# Java programmer in me is offended by op-overloading, but it's very -# tidy. -class PrefsFacade(): - def __init__(self,default_prefs): - self.default_prefs = default_prefs - self.libraryid = None - self.current_prefs = None - - def _get_prefs(self): - libraryid = get_library_uuid(get_gui().current_db) - if self.current_prefs == None or self.libraryid != libraryid: - #print("self.current_prefs == None(%s) or self.libraryid != libraryid(%s)"%(self.current_prefs == None,self.libraryid != libraryid)) - self.libraryid = libraryid - self.current_prefs = get_library_config() - return self.current_prefs - - def __getitem__(self,k): - prefs = self._get_prefs() - if k not in prefs: - # pulls from default_prefs.defaults automatically if not set - # in default_prefs - return self.default_prefs[k] - return prefs[k] - - def __setitem__(self,k,v): - prefs = self._get_prefs() - prefs[k]=v - # self._save_prefs(prefs) - - def __delitem__(self,k): - prefs = self._get_prefs() - if k in prefs: - del prefs[k] - - def save_to_db(self): - set_library_config(self._get_prefs()) - -prefs = PrefsFacade(default_prefs) - class RejectURLList: def __init__(self,prefs): self.prefs = prefs @@ -282,6 +166,7 @@ class ConfigWidget(QWidget): prefs['updatecover'] = self.basic_tab.updatecover.isChecked() prefs['updateepubcover'] = self.basic_tab.updateepubcover.isChecked() prefs['keeptags'] = self.basic_tab.keeptags.isChecked() + prefs['showmarked'] = self.basic_tab.showmarked.isChecked() prefs['urlsfromclip'] = self.basic_tab.urlsfromclip.isChecked() prefs['updatedefault'] = self.basic_tab.updatedefault.isChecked() prefs['deleteotherforms'] = self.basic_tab.deleteotherforms.isChecked() @@ -448,6 +333,13 @@ class BasicTab(QWidget): self.l.addSpacing(10) + self.showmarked = QCheckBox("Show added/updated books when finished?",self) + self.showmarked.setToolTip("Show added/updated books only when finished.\nYou can also manually search for 'marked:ffdl_success'.\n'marked:ffdl_failed' is also available, or search 'marked:ffdl' for both.") + self.showmarked.setChecked(prefs['showmarked']) + self.l.addWidget(self.showmarked) + + self.l.addSpacing(10) + self.urlsfromclip = QCheckBox('Take URLs from Clipboard?',self) self.urlsfromclip.setToolTip('Prefill URLs from valid URLs in Clipboard when Adding New.') self.urlsfromclip.setChecked(prefs['urlsfromclip']) diff --git a/calibre-plugin/dialogs.py b/calibre-plugin/dialogs.py index 768bff6..e1981bb 100644 --- a/calibre-plugin/dialogs.py +++ b/calibre-plugin/dialogs.py @@ -557,84 +557,6 @@ class UpdateExistingDialog(SizePersistedDialog): 'updateepubcover': self.updateepubcover.isChecked(), } -def display_story_list(gui, header, prefs, icon, books, - label_text='', - save_size_name='fanfictiondownloader_plugin:display list dialog', - offer_skip=False): - all_good = True - for b in books: - if not b['good']: - all_good=False - break - - ## - if all_good and not dynamic.get(confirm_config_name(save_size_name), True): - return True - pass - ## fake accept? - d = DisplayStoryListDialog(gui, header, prefs, icon, books, - label_text, - save_size_name, - offer_skip and all_good) - d.exec_() - return d.result() == d.Accepted - -class DisplayStoryListDialog(SizePersistedDialog): - def __init__(self, gui, header, prefs, icon, books, - label_text='', - save_size_name='fanfictiondownloader_plugin:display list dialog', - offer_skip=False): - SizePersistedDialog.__init__(self, gui, save_size_name) - self.name = save_size_name - self.gui = gui - - self.setWindowTitle(header) - self.setWindowIcon(icon) - - layout = QVBoxLayout(self) - self.setLayout(layout) - title_layout = ImageTitleLayout(self, 'images/icon.png', - header) - layout.addLayout(title_layout) - - self.books_table = StoryListTableWidget(self) - layout.addWidget(self.books_table) - - options_layout = QHBoxLayout() - self.label = QLabel(label_text) - #self.label.setOpenExternalLinks(True) - #self.label.setWordWrap(True) - options_layout.addWidget(self.label) - - if offer_skip: - spacerItem1 = QtGui.QSpacerItem(2, 4, QtGui.QSizePolicy.Expanding, QtGui.QSizePolicy.Minimum) - options_layout.addItem(spacerItem1) - self.again = QCheckBox('Show this again?',self) - self.again.setChecked(True) - self.again.stateChanged.connect(self.toggle) - self.again.setToolTip('Uncheck to skip review and update stories immediately when no problems.') - options_layout.addWidget(self.again) - - button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel) - button_box.accepted.connect(self.accept) - button_box.rejected.connect(self.reject) - - options_layout.addWidget(button_box) - - layout.addLayout(options_layout) - - # Cause our dialog size to be restored from prefs or created on first usage - self.resize_dialog() - self.books_table.populate_table(books) - - def get_books(self): - return self.books_table.get_books() - - def toggle(self, *args): - dynamic[confirm_config_name(self.name)] = self.again.isChecked() - - - class StoryListTableWidget(QTableWidget): def __init__(self, parent): diff --git a/calibre-plugin/ffdl_plugin.py b/calibre-plugin/ffdl_plugin.py index df4afb4..ad27de4 100644 --- a/calibre-plugin/ffdl_plugin.py +++ b/calibre-plugin/ffdl_plugin.py @@ -41,9 +41,10 @@ from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.epubutils impo from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.geturls import get_urls_from_page from calibre_plugins.fanfictiondownloader_plugin.ffdl_util import (get_ffdl_adapter, get_ffdl_config, get_ffdl_personalini) -from calibre_plugins.fanfictiondownloader_plugin.config import (prefs, permitted_values, rejecturllist) +from calibre_plugins.fanfictiondownloader_plugin.config import (permitted_values, rejecturllist) +from calibre_plugins.fanfictiondownloader_plugin.prefs import prefs from calibre_plugins.fanfictiondownloader_plugin.dialogs import ( - AddNewDialog, UpdateExistingDialog, display_story_list, DisplayStoryListDialog, + AddNewDialog, UpdateExistingDialog, LoopProgressDialog, UserPassDialog, AboutDialog, CollectURLDialog, RejectListDialog, OVERWRITE, OVERWRITEALWAYS, UPDATE, UPDATEALWAYS, ADDNEW, SKIP, CALIBREONLY, NotGoingToDownload ) @@ -551,7 +552,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction): for j, book in enumerate(update_books): url = book['url'] - book['mergeorder'] = j + book['listorder'] = j if url in urlmapfile: #print("found epub for %s"%url) book['epub_for_update']=urlmapfile[url] @@ -596,6 +597,9 @@ class FanFictionDownLoaderPlugin(InterfaceAction): book_list = map( partial(self.make_book_id_only), self.gui.library_view.get_selected_ids() ) #book_ids = self.gui.library_view.get_selected_ids() + for j, book in enumerate(book_list): + book['listorder'] = j + LoopProgressDialog(self.gui, book_list, partial(self.populate_book_from_calibre_id, db=self.gui.current_db), @@ -872,8 +876,9 @@ class FanFictionDownLoaderPlugin(InterfaceAction): raise NotGoingToDownload("Not Overwriting, web site is not newer.",'edit-undo.png') # For update, provide a tmp file copy of the existing epub so - # it can't change underneath us. - if collision in (UPDATE,UPDATEALWAYS) and \ + # it can't change underneath us. Now also overwrite for logpage preserve. + if collision in (UPDATE,UPDATEALWAYS,OVERWRITE,OVERWRITEALWAYS) and \ + fileform == 'epub' and \ db.has_format(book['calibre_id'],'EPUB',index_is_id=True): tmp = PersistentTemporaryFile(prefix='old-%s-'%book['calibre_id'], suffix='.epub', @@ -930,18 +935,28 @@ class FanFictionDownLoaderPlugin(InterfaceAction): break else: ## No good stories to try to download, go straight to - ## list. - d = DisplayStoryListDialog(self.gui, - 'Nothing to Download', - prefs, - self.qaction.icon(), - book_list, - label_text='None of the URLs/stories given can be/need to be downloaded.' - ) - d.exec_() - - self.update_error_column(book_list,options) + ## updating error col. + msg = ''' +

None of the %d URLs/stories given can be/need to be downloaded.

+

See log for details.

+

Proceed with updating your library(Error Column, if configured)?

+'''%len(book_list) + + htmllog='' + for book in book_list: + if 'status' in book: + status = book['status'] + else: + status = 'Bad' + htmllog = htmllog + '' + htmllog = htmllog + '
StatusTitleAuthorCommentURL
' + ''.join([escapehtml(status),escapehtml(book['title']),escapehtml(", ".join(book['author'])),escapehtml(book['comment']),book['url']]) + '
' + + payload = ([], book_list, options) + self.gui.proceed_question(self.update_error_column, + payload, htmllog, + 'FFDL log', 'FFDL download ended', msg, + show_copy_button=False) return func = 'arbitrary_n' @@ -961,6 +976,18 @@ class FanFictionDownLoaderPlugin(InterfaceAction): 'collision':ADDNEW, 'updatemeta':True, 'updateepubcover':True}): + custom_columns = self.gui.library_view.model().custom_columns + if book['calibre_id'] and prefs['errorcol'] != '' and prefs['errorcol'] in custom_columns: + label = custom_columns[prefs['errorcol']]['label'] + if not book['good']: + print("record/update error message column %s %s"%(book['title'],book['url'])) + db.set_custom(book['calibre_id'], book['comment'], label=label, commit=True) # book['comment'] + else: + db.set_custom(book['calibre_id'], '', label=label, commit=True) # book['comment'] + + if not book['good']: + return # only update errorcol on error. + print("add/update %s %s"%(book['title'],book['url'])) mi = self.make_mi_from_book(book) @@ -979,8 +1006,10 @@ class FanFictionDownLoaderPlugin(InterfaceAction): add_ids = [ x['calibre_id'] for x in add_list ] update_list = filter(lambda x : x['good'] and not x['added'], book_list) update_ids = [ x['calibre_id'] for x in update_list ] - all_ids = add_ids - all_ids.extend(update_ids) + all_ids = add_ids + update_ids + + failed_list = filter(lambda x : not x['good'] , book_list) + failed_ids = [ x['calibre_id'] for x in failed_list ] if options['collision'] != CALIBREONLY and \ (prefs['addtolists'] or prefs['addtoreadlists']): @@ -999,19 +1028,26 @@ class FanFictionDownLoaderPlugin(InterfaceAction): if self.gui.cover_flow: self.gui.cover_flow.dataChanged() - + + if showlist: # don't use with anthology + db = self.gui.current_db + marked_ids = dict() + marked_text = "ffdl_success" + for index, book_id in enumerate(all_ids): + marked_ids[book_id] = '%s_%04d' % (marked_text, index) + for index, book_id in enumerate(failed_ids): + marked_ids[book_id] = 'ffdl_failed_%04d' % index + # Mark the results in our database + db.set_marked_ids(marked_ids) + + if prefs['showmarked']: # show add/update + # Search to display the list contents + self.gui.search.set_search_string('marked:' + marked_text) + # Sort by our marked column to display the books in order + self.gui.library_view.sort_by_named_field('marked', True) + self.gui.status_bar.show_message(_('Finished Adding/Updating %d books.'%(len(update_list) + len(add_list))), 3000) - if showlist and (len(update_list) + len(add_list) != len(book_list)): - d = DisplayStoryListDialog(self.gui, - 'Updates completed, final status', - prefs, - self.qaction.icon(), - book_list, - label_text='Stories have be added or updated in Calibre, some had additional problems.' - ) - d.exec_() - print("all done, remove temp dir.") remove_dir(options['tdir']) @@ -1030,6 +1066,8 @@ class FanFictionDownLoaderPlugin(InterfaceAction): book_list = job.result good_list = filter(lambda x : x['good'], book_list) bad_list = filter(lambda x : not x['good'], book_list) + good_list = sorted(good_list,key=lambda x : x['listorder']) + bad_list = sorted(bad_list,key=lambda x : x['listorder']) #print("book_list:%s"%book_list) payload = (good_list, bad_list, options) @@ -1051,7 +1089,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction): msg = msg + '

Proceed with updating this anthology and your library?

' htmllog='' - for book in sorted(good_list+bad_list,key=lambda x : x['mergeorder']): + for book in sorted(good_list+bad_list,key=lambda x : x['listorder']): if 'status' in book: status = book['status'] else: @@ -1106,10 +1144,10 @@ class FanFictionDownLoaderPlugin(InterfaceAction): (good_list,bad_list,options) = payload total_good = len(good_list) - print("merge titles:\n%s"%"\n".join([ "%s %s"%(x['title'],x['mergeorder']) for x in good_list ])) + print("merge titles:\n%s"%"\n".join([ "%s %s"%(x['title'],x['listorder']) for x in good_list ])) - good_list = sorted(good_list,key=lambda x : x['mergeorder']) - bad_list = sorted(bad_list,key=lambda x : x['mergeorder']) + good_list = sorted(good_list,key=lambda x : x['listorder']) + bad_list = sorted(bad_list,key=lambda x : x['listorder']) self.gui.status_bar.show_message(_('Merging %s books.'%total_good)) @@ -1149,26 +1187,23 @@ class FanFictionDownLoaderPlugin(InterfaceAction): def do_download_list_update(self, payload): (good_list,bad_list,options) = payload - total_good = len(good_list) + good_list = sorted(good_list,key=lambda x : x['listorder']) + bad_list = sorted(bad_list,key=lambda x : x['listorder']) - self.gui.status_bar.show_message(_('Adding/Updating %s books.'%total_good)) + self.gui.status_bar.show_message(_('FFDL Adding/Updating books.')) - if total_good > 0: + if good_list or (bad_list and prefs['errorcol'] != '' and prefs['errorcol'] in self.gui.library_view.model().custom_columns): LoopProgressDialog(self.gui, - good_list, + good_list+bad_list, partial(self.update_books_loop, options=options, db=self.gui.current_db), partial(self.update_books_finish, options=options), init_label="Updating calibre for FanFiction stories...", win_title="Update calibre for FanFiction stories", status_prefix="Updated") - total_bad = len(bad_list) - - if total_bad > 0: - self.update_error_column(bad_list,options) - - def update_error_column(self,book_list,options): + def update_error_column(self,payload): '''Update custom error column if configured.''' + (empty_list,book_list,options)=payload custom_columns = self.gui.library_view.model().custom_columns if prefs['errorcol'] != '' and prefs['errorcol'] in custom_columns: self.previous = self.gui.library_view.currentIndex() # used by update_books_finish. @@ -1391,11 +1426,6 @@ class FanFictionDownLoaderPlugin(InterfaceAction): realmi = db.get_metadata(book_id, index_is_id=True) gc_plugin.generate_cover_for_book(realmi,saved_setting_name=setting_name) - ## if error column set. - if prefs['errorcol'] != '' and prefs['errorcol'] in custom_columns: - label = custom_columns[prefs['errorcol']]['label'] - db.set_custom(book['calibre_id'], '', label=label, commit=True) # book['comment'] - def get_clean_reading_lists(self,lists): if lists == None or lists.strip() == "" : return [] @@ -1488,8 +1518,8 @@ class FanFictionDownLoaderPlugin(InterfaceAction): book['good'] = False book['comment'] = "Same story already included." uniqueurls.add(book['url']) - book['mergeorder']=i # BG d/l jobs don't come back in order. - # Didn't matter until anthologies. + book['listorder']=i # BG d/l jobs don't come back in order. + # Didn't matter until anthologies & 'marked' successes books.append(book) return books diff --git a/calibre-plugin/ffdl_util.py b/calibre-plugin/ffdl_util.py index adaef1e..c169d80 100644 --- a/calibre-plugin/ffdl_util.py +++ b/calibre-plugin/ffdl_util.py @@ -11,7 +11,7 @@ from StringIO import StringIO from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapters, exceptions from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.configurable import Configuration -from calibre_plugins.fanfictiondownloader_plugin.config import (prefs) +from calibre_plugins.fanfictiondownloader_plugin.prefs import (prefs) def get_ffdl_personalini(): if prefs['includeimages']: diff --git a/calibre-plugin/jobs.py b/calibre-plugin/jobs.py index 9d95afc..88bdf7b 100644 --- a/calibre-plugin/jobs.py +++ b/calibre-plugin/jobs.py @@ -153,6 +153,22 @@ def do_download_for_worker(book,options): elif options['collision'] in (ADDNEW, SKIP, OVERWRITE, OVERWRITEALWAYS) or \ ('epub_for_update' not in book and options['collision'] in (UPDATE, UPDATEALWAYS)): + # preserve logfile even on overwrite. + if 'epub_for_update' in book: + (urlignore, + chaptercountignore, + oldchaptersignore, + oldimgsignore, + oldcoverignore, + calibrebookmarkignore, + # only logfile set in adapter, so others aren't used. + adapter.logfile) = get_update_data(book['epub_for_update']) + + # change the existing entries id to notid so + # write_epub writes a whole new set to indicate overwrite. + if adapter.logfile: + adapter.logfile = adapter.logfile.replace("span id","span notid") + print("write to %s"%outfile) writer.writeStory(outfilename=outfile, forceOverwrite=True) book['comment'] = 'Download %s completed, %s chapters.'%(options['fileform'],story.getMetadata("numChapters")) diff --git a/calibre-plugin/prefs.py b/calibre-plugin/prefs.py new file mode 100644 index 0000000..f529534 --- /dev/null +++ b/calibre-plugin/prefs.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2013, Jim Miller' +__docformat__ = 'restructuredtext en' + +import copy + +from calibre.utils.config import JSONConfig +from calibre.gui2.ui import get_gui + +from calibre_plugins.fanfictiondownloader_plugin.dialogs import OVERWRITE +from calibre_plugins.fanfictiondownloader_plugin.common_utils import get_library_uuid +PREFS_NAMESPACE = 'FanFictionDownLoaderPlugin' +PREFS_KEY_SETTINGS = 'settings' + +# Set defaults used by all. Library specific settings continue to +# take from here. +default_prefs = {} +default_prefs['personal.ini'] = get_resources('plugin-example.ini') +default_prefs['rejecturls'] = '' +default_prefs['rejectreasons'] = '''Sucked +Boring +Dup from another site''' + +default_prefs['updatemeta'] = True +default_prefs['updatecover'] = False +default_prefs['updateepubcover'] = False +default_prefs['keeptags'] = False +default_prefs['showmarked'] = False +default_prefs['urlsfromclip'] = True +default_prefs['updatedefault'] = True +default_prefs['fileform'] = 'epub' +default_prefs['collision'] = OVERWRITE +default_prefs['deleteotherforms'] = False +default_prefs['adddialogstaysontop'] = False +default_prefs['includeimages'] = False +default_prefs['lookforurlinhtml'] = False +default_prefs['injectseries'] = False + +default_prefs['send_lists'] = '' +default_prefs['read_lists'] = '' +default_prefs['addtolists'] = False +default_prefs['addtoreadlists'] = False +default_prefs['addtolistsonread'] = False + +default_prefs['gcnewonly'] = False +default_prefs['gc_site_settings'] = {} +default_prefs['allow_gc_from_ini'] = True + +default_prefs['countpagesstats'] = [] + +default_prefs['errorcol'] = '' +default_prefs['custom_cols'] = {} +default_prefs['custom_cols_newonly'] = {} +default_prefs['allow_custcol_from_ini'] = True + +default_prefs['std_cols_newonly'] = {} + +def set_library_config(library_config,db): + db.prefs.set_namespaced(PREFS_NAMESPACE, + PREFS_KEY_SETTINGS, + library_config) + +def get_library_config(db): + library_id = get_library_uuid(db) + library_config = None + # Check whether this is a configuration needing to be migrated + # from json into database. If so: get it, set it, rename it in json. + if library_id in old_prefs: + #print("get prefs from old_prefs") + library_config = old_prefs[library_id] + set_library_config(library_config) + old_prefs["migrated to library db %s"%library_id] = old_prefs[library_id] + del old_prefs[library_id] + + if library_config is None: + #print("get prefs from db") + library_config = db.prefs.get_namespaced(PREFS_NAMESPACE, PREFS_KEY_SETTINGS, + copy.deepcopy(default_prefs)) + return library_config + +# This is where all preferences for this plugin *were* stored +# Remember that this name (i.e. plugins/fanfictiondownloader_plugin) is also +# in a global namespace, so make it as unique as possible. +# You should always prefix your config file name with plugins/, +# so as to ensure you dont accidentally clobber a calibre config file +old_prefs = JSONConfig('plugins/fanfictiondownloader_plugin') + +# fake out so I don't have to change the prefs calls anywhere. The +# Java programmer in me is offended by op-overloading, but it's very +# tidy. +class PrefsFacade(): + def _get_db(self): + if self.passed_db: + return self.passed_db + else: + # In the GUI plugin we want current db so we detect when + # it's changed. CLI plugin calls need to pass db in. + return get_gui().current_db + + def __init__(self,passed_db=None): + self.default_prefs = default_prefs + self.libraryid = None + self.current_prefs = None + self.passed_db=passed_db + + def _get_prefs(self): + libraryid = get_library_uuid(self._get_db()) + if self.current_prefs == None or self.libraryid != libraryid: + #print("self.current_prefs == None(%s) or self.libraryid != libraryid(%s)"%(self.current_prefs == None,self.libraryid != libraryid)) + self.libraryid = libraryid + self.current_prefs = get_library_config(self._get_db()) + return self.current_prefs + + def __getitem__(self,k): + prefs = self._get_prefs() + if k not in prefs: + # pulls from default_prefs.defaults automatically if not set + # in default_prefs + return self.default_prefs[k] + return prefs[k] + + def __setitem__(self,k,v): + prefs = self._get_prefs() + prefs[k]=v + # self._save_prefs(prefs) + + def __delitem__(self,k): + prefs = self._get_prefs() + if k in prefs: + del prefs[k] + + def save_to_db(self): + set_library_config(self._get_prefs(),self._get_db()) + +prefs = PrefsFacade() + diff --git a/defaults.ini b/defaults.ini index c2af98f..bd18b80 100644 --- a/defaults.ini +++ b/defaults.ini @@ -369,7 +369,7 @@ output_css: ## It can be either a 'file:' or 'http:' url. ## Note that if you enable make_firstimage_cover in [epub], but want ## to use default_cover_image for a specific site, use the site:format -## section, for example: [www.ficwad.com:epub] +## section, for example: [ficwad.com:epub] ## default_cover_image is a python string Template string with ## ${title}, ${author} etc, same as titlepage_entries. Unless ## allow_unsafe_filename is true, invalid filename chars will be diff --git a/downloader.py b/downloader.py index 90a9874..d440392 100644 --- a/downloader.py +++ b/downloader.py @@ -32,10 +32,17 @@ if sys.version_info >= (2, 7): loghandler.setFormatter(logging.Formatter("(=====)(levelname)s:%(message)s")) rootlogger.addHandler(loghandler) -from fanficdownloader import adapters,writers,exceptions -from fanficdownloader.configurable import Configuration -from fanficdownloader.epubutils import get_dcsource_chaptercount, get_update_data -from fanficdownloader.geturls import get_urls_from_page +try: + from fanficdownloader import adapters,writers,exceptions + from fanficdownloader.configurable import Configuration + from fanficdownloader.epubutils import get_dcsource_chaptercount, get_update_data + from fanficdownloader.geturls import get_urls_from_page +except: + # running under calibre + from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapters,writers,exceptions + from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.configurable import Configuration + from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.epubutils import get_dcsource_chaptercount, get_update_data + from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.geturls import get_urls_from_page if sys.version_info < (2, 5): print "This program requires Python 2.5 or newer." @@ -48,15 +55,23 @@ def writeStory(config,adapter,writeformat,metaonly=False,outstream=None): del writer return output_filename -def main(): +def main(argv, + parser=None, + passed_defaultsini=None, + passed_personalini=None): # read in args, anything starting with -- will be treated as --= - usage = "usage: %prog [options] storyurl" - parser = OptionParser(usage) + if not parser: + parser = OptionParser("usage: %prog [options] storyurl") parser.add_option("-f", "--format", dest="format", default="epub", - help="write story as FORMAT, epub(default), text or html", metavar="FORMAT") + help="write story as FORMAT, epub(default), mobi, text or html", metavar="FORMAT") + + if passed_defaultsini: + config_help="read config from specified file(s) in addition to calibre plugin personal.ini, ~/.fanficdownloader/personal.ini, and ./personal.ini" + else: + config_help="read config from specified file(s) in addition to ~/.fanficdownloader/defaults.ini, ~/.fanficdownloader/personal.ini, ./defaults.ini, and ./personal.ini" parser.add_option("-c", "--config", action="append", dest="configfile", default=None, - help="read config from specified file(s) in addition to ~/.fanficdownloader/defaults.ini, ~/.fanficdownloader/personal.ini, ./defaults.ini, ./personal.ini", metavar="CONFIG") + help=config_help, metavar="CONFIG") parser.add_option("-b", "--begin", dest="begin", default=None, help="Begin with Chapter START", metavar="START") parser.add_option("-e", "--end", dest="end", default=None, @@ -83,7 +98,7 @@ def main(): action="store_true", dest="debug", help="Show debug output while downloading.",) - (options, args) = parser.parse_args() + (options, args) = parser.parse_args(argv) if not options.debug: logger = logging.getLogger("fanficdownloader") @@ -107,12 +122,18 @@ def main(): conflist = [] homepath = join(expanduser("~"),".fanficdownloader") + + if passed_defaultsini: + configuration.readfp(passed_defaultsini) if isfile(join(homepath,"defaults.ini")): conflist.append(join(homepath,"defaults.ini")) if isfile("defaults.ini"): conflist.append("defaults.ini") + if passed_personalini: + configuration.readfp(passed_personalini) + if isfile(join(homepath,"personal.ini")): conflist.append(join(homepath,"personal.ini")) if isfile("personal.ini"): @@ -240,5 +261,5 @@ def main(): if __name__ == "__main__": #import time #start = time.time() - main() + main(sys.argv[1:]) #print("Total time seconds:%f"%(time.time()-start)) diff --git a/fanficdownloader/configurable.py b/fanficdownloader/configurable.py index dd16992..261c7a3 100644 --- a/fanficdownloader/configurable.py +++ b/fanficdownloader/configurable.py @@ -94,7 +94,12 @@ class Configuration(ConfigParser.SafeConfigParser): #print("found %s in section [%s]"%(key,section)) return True except: - pass + try: + self.get(section,"add_to_"+key) + #print("found add_to_%s in section [%s]"%(key,section)) + return True + except: + pass return False @@ -106,16 +111,24 @@ class Configuration(ConfigParser.SafeConfigParser): if val and val.lower() == "false": val = False #print "getConfig(%s)=[%s]%s" % (key,section,val) - return val + break except (ConfigParser.NoOptionError, ConfigParser.NoSectionError), e: pass + for section in self.sectionslist[::-1]: + # 'martian smiley' [::-1] reverses list by slicing whole list with -1 step. + try: + val = val + self.get(section,"add_to_"+key) + #print "getConfig(add_to_%s)=[%s]%s" % (key,section,val) + except (ConfigParser.NoOptionError, ConfigParser.NoSectionError), e: + pass + return val # split and strip each. def getConfigList(self, key): vlist = self.getConfig(key).split(',') - vlist = [ v.strip() for v in vlist ] + vlist = filter( lambda x : x !='', [ v.strip() for v in vlist ]) #print "vlist("+key+"):"+str(vlist) return vlist diff --git a/makeplugin.py b/makeplugin.py index c8c10e4..d13439b 100644 --- a/makeplugin.py +++ b/makeplugin.py @@ -27,7 +27,7 @@ if __name__=="__main__": exclude=['*.pyc','*~','*.xcf','*[0-9].png'] # from top dir. 'w' for overwrite createZipFile(filename,"w", - ['plugin-defaults.ini','plugin-example.ini','fanficdownloader'], + ['plugin-defaults.ini','plugin-example.ini','fanficdownloader','downloader.py','defaults.ini'], exclude=exclude) #from calibre-plugin dir. 'a' for append os.chdir('calibre-plugin') diff --git a/plugin-defaults.ini b/plugin-defaults.ini index 7933b2a..0195a38 100644 --- a/plugin-defaults.ini +++ b/plugin-defaults.ini @@ -269,6 +269,8 @@ include_tocpage: false ## dateUpdated,numChapters,numWords at a minimum) will be shown. ## Great for tracking when chapters came out and when the description, ## etc changed. +## Plugin will now preserve the log page when the epub is overwritten, +## too. include_logpage: false ## If set to 'smart', logpage will only be included if the story is ## status:In-Progress or already had a logpage. That way you don't
StatusTitleAuthorCommentURL