From 36eb6c2e2c8d3d883b2a248457518dc68ad7a61e Mon Sep 17 00:00:00 2001 From: Jim Miller Date: Sun, 8 Jan 2012 13:42:27 -0600 Subject: [PATCH] Plugin 1.1.1, only allow update when epub, restruct add/update code. --- calibre-plugin/__init__.py | 6 +- calibre-plugin/config.py | 19 +++- calibre-plugin/dialogs.py | 90 ++++++++---------- calibre-plugin/ffdl_plugin.py | 168 ++++++++++++++++++++++------------ calibre-plugin/jobs.py | 7 +- 5 files changed, 172 insertions(+), 118 deletions(-) diff --git a/calibre-plugin/__init__.py b/calibre-plugin/__init__.py index d952652..d995756 100644 --- a/calibre-plugin/__init__.py +++ b/calibre-plugin/__init__.py @@ -11,7 +11,9 @@ __docformat__ = 'restructuredtext en' # The class that all Interface Action plugin wrappers must inherit from from calibre.customize import InterfaceActionBase -class InterfacePluginDemo(InterfaceActionBase): +## Apparently the name for this class doesn't matter--it was still +## 'demo' for the first few versions. +class FanFictionDownLoaderBase(InterfaceActionBase): ''' This class is a simple wrapper that provides information about the actual plugin class. The actual interface plugin class is called @@ -25,7 +27,7 @@ class InterfacePluginDemo(InterfaceActionBase): description = 'UI plugin to download FanFiction stories from various sites.' supported_platforms = ['windows', 'osx', 'linux'] author = 'Jim Miller' - version = (1, 1, 0) + version = (1, 1, 1) minimum_calibre_version = (0, 8, 30) #: This field defines the GUI plugin class that contains all the code diff --git a/calibre-plugin/config.py b/calibre-plugin/config.py index a91f731..7d09338 100644 --- a/calibre-plugin/config.py +++ b/calibre-plugin/config.py @@ -51,6 +51,7 @@ class ConfigWidget(QWidget): self.fileform.addItem('txt') self.fileform.setCurrentIndex(self.fileform.findText(prefs['fileform'])) self.fileform.setToolTip('Choose output format to create. May set default from plugin configuration.') + self.fileform.activated.connect(self.set_collisions) label.setBuddy(self.fileform) horz.addWidget(self.fileform) self.l.addLayout(horz) @@ -60,9 +61,11 @@ class ConfigWidget(QWidget): label.setToolTip("What to do if there's already an existing story with the same title and author.") horz.addWidget(label) self.collision = QComboBox(self) - for o in collision_order: - self.collision.addItem(o) - self.collision.setCurrentIndex(self.collision.findText(prefs['collision'])) + # add collision options + self.set_collisions() + i = self.collision.findText(prefs['collision']) + if i > -1: + self.collision.setCurrentIndex(i) # self.collision.setToolTip('Overwrite will replace the existing story. Add New will create a new story with the same title and author.') label.setBuddy(self.collision) horz.addWidget(self.collision) @@ -113,6 +116,16 @@ class ConfigWidget(QWidget): reset_confirmation_button.clicked.connect(self.reset_dialogs) self.l.addWidget(reset_confirmation_button) + def set_collisions(self): + prev=self.collision.currentText() + self.collision.clear() + for o in collision_order: + if self.fileform.currentText() == 'epub' or o not in [UPDATE,UPDATEALWAYS]: + self.collision.addItem(o) + i = self.collision.findText(prev) + if i > -1: + self.collision.setCurrentIndex(i) + def save_settings(self): prefs['fileform'] = unicode(self.fileform.currentText()) prefs['collision'] = unicode(self.collision.currentText()) diff --git a/calibre-plugin/dialogs.py b/calibre-plugin/dialogs.py index a14c53c..3fe2544 100644 --- a/calibre-plugin/dialogs.py +++ b/calibre-plugin/dialogs.py @@ -66,15 +66,6 @@ class AddNewDialog(SizePersistedDialog): self.url.setText(url_list_text) self.l.addWidget(self.url) - # self.ffdl_button = QPushButton( - # 'Download Stories', self) - # self.ffdl_button.setToolTip('Start download(s).') - # self.ffdl_button.clicked.connect(self.ffdl) - # # if there's already URL(s), focus 'go' button - # if url_list_text: - # self.ffdl_button.setFocus() - # self.l.addWidget(self.ffdl_button) - horz = QHBoxLayout() label = QLabel('Output &Format:') horz.addWidget(label) @@ -85,6 +76,8 @@ class AddNewDialog(SizePersistedDialog): self.fileform.addItem('txt') self.fileform.setCurrentIndex(self.fileform.findText(prefs['fileform'])) self.fileform.setToolTip('Choose output format to create. May set default from plugin configuration.') + self.fileform.activated.connect(self.set_collisions) + label.setBuddy(self.fileform) horz.addWidget(self.fileform) self.l.addLayout(horz) @@ -95,10 +88,10 @@ class AddNewDialog(SizePersistedDialog): horz.addWidget(label) self.collision = QComboBox(self) # add collision options - for o in collision_order: - self.collision.addItem(o) - - self.collision.setCurrentIndex(self.collision.findText(prefs['collision'])) + self.set_collisions() + i = self.collision.findText(prefs['collision']) + if i > -1: + self.collision.setCurrentIndex(i) # self.collision.setToolTip(OVERWRITE+' will replace the existing story.\n'+ # UPDATE+' will download new chapters only and add to existing EPUB.\n'+ # ADDNEW+' will create a new story with the same title and author.\n'+ @@ -113,22 +106,6 @@ class AddNewDialog(SizePersistedDialog): self.updatemeta.setChecked(prefs['updatemeta']) self.l.addWidget(self.updatemeta) - # self.onlyoverwriteifnewer = QCheckBox('Only Overwrite Story if Newer',self) - # self.onlyoverwriteifnewer.setToolTip("Don't overwrite existing book unless the story on the web site is newer.\n"+ - # "From the same day counts as 'newer' because the sites don't give update time.") - # self.onlyoverwriteifnewer.setChecked(prefs['onlyoverwriteifnewer']) - # self.l.addWidget(self.onlyoverwriteifnewer) - - # horz = QHBoxLayout() - # self.about_button = QPushButton('About', self) - # self.about_button.clicked.connect(self.about) - # horz.addWidget(self.about_button) - # self.conf_button = QPushButton( - # 'Configure this plugin', self) - # self.conf_button.clicked.connect(self.config) - # horz.addWidget(self.conf_button) - # self.l.addLayout(horz) - button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel) button_box.accepted.connect(self.accept) button_box.rejected.connect(self.reject) @@ -136,10 +113,21 @@ class AddNewDialog(SizePersistedDialog): if url_list_text: button_box.button(QDialogButtonBox.Ok).setFocus() - + + # restore saved size. self.resize_dialog() #self.resize(self.sizeHint()) + def set_collisions(self): + prev=self.collision.currentText() + self.collision.clear() + for o in collision_order: + if self.fileform.currentText() == 'epub' or o not in [UPDATE,UPDATEALWAYS]: + self.collision.addItem(o) + i = self.collision.findText(prev) + if i > -1: + self.collision.setCurrentIndex(i) + def get_ffdl_options(self): return { 'fileform': unicode(self.fileform.currentText()), @@ -171,6 +159,7 @@ class UserPassDialog(QDialog): self.l.addWidget(QLabel("Password:"),2,0) self.passwd = QLineEdit(self) + self.passwd.setEchoMode(QLineEdit.Password) self.l.addWidget(self.passwd,2,1) self.ok_button = QPushButton('OK', self) @@ -223,7 +212,6 @@ class MetadataProgressDialog(QProgressDialog): print(self.labelText()) def do_loop(self): - print("self.i:%d"%self.i) if self.i == 0: self.setValue(0) @@ -242,9 +230,8 @@ class MetadataProgressDialog(QProgressDialog): except Exception as e: book['good']=False book['comment']=unicode(e) - print("%s:%s"%(book,unicode(e))) - # XXX trace for not-expected exceptions - #traceback.print_exc() + print("Exception: %s:%s"%(book,unicode(e))) + traceback.print_exc() self.updateStatus() self.i += 1 @@ -354,6 +341,7 @@ class UpdateExistingDialog(SizePersistedDialog): self.fileform.addItem('txt') self.fileform.setCurrentIndex(self.fileform.findText(prefs['fileform'])) self.fileform.setToolTip('Choose output format to create. May set default from plugin configuration.') + self.fileform.activated.connect(self.set_collisions) label.setBuddy(self.fileform) options_layout.addWidget(self.fileform) @@ -361,10 +349,11 @@ class UpdateExistingDialog(SizePersistedDialog): label.setToolTip("What sort of update to perform. May set default from plugin configuration.") options_layout.addWidget(label) self.collision = QComboBox(self) - for o in collision_order: - if o not in [ADDNEW,SKIP]: - self.collision.addItem(o) - self.collision.setCurrentIndex(self.collision.findText(prefs['collision'])) + # add collision options + self.set_collisions() + i = self.collision.findText(prefs['collision']) + if i > -1: + self.collision.setCurrentIndex(i) # self.collision.setToolTip('Overwrite will replace the existing story. Add New will create a new story with the same title and author.') label.setBuddy(self.collision) options_layout.addWidget(self.collision) @@ -384,6 +373,17 @@ class UpdateExistingDialog(SizePersistedDialog): # Cause our dialog size to be restored from prefs or created on first usage self.resize_dialog() self.books_table.populate_table(books) + + def set_collisions(self): + prev=self.collision.currentText() + self.collision.clear() + for o in collision_order: + if o not in [ADDNEW,SKIP] and \ + (self.fileform.currentText() == 'epub' or o not in [UPDATE,UPDATEALWAYS]): + self.collision.addItem(o) + i = self.collision.findText(prev) + if i > -1: + self.collision.setCurrentIndex(i) def remove_from_list(self): self.books_table.remove_selected_rows() @@ -507,26 +507,12 @@ class StoryListTableWidget(QTableWidget): #comment_cell.setData(Qt.UserRole, QVariant(book)) self.setItem(row, 4, comment_cell) - # def get_calibre_ids(self): - # ids = [] - # for row in range(self.rowCount()): - # ids.append(self.item(row, 1).data(Qt.UserRole).toPyObject()) - # return ids - - # def get_urls(self): - # urls = [] - # for row in range(self.rowCount()): - # urls.append(self.item(row, 2).data(Qt.UserRole).toPyObject()) - # return urls - def get_books(self): books = [] #print("=========================\nbooks:%s"%self.books) for row in range(self.rowCount()): rnum = self.item(row, 1).data(Qt.UserRole).toPyObject() - #print("get_books rnum:%s"%rnum) book = self.books[rnum] - #if book['good']: books.append(book) return books diff --git a/calibre-plugin/ffdl_plugin.py b/calibre-plugin/ffdl_plugin.py index b47337d..17f2d0a 100644 --- a/calibre-plugin/ffdl_plugin.py +++ b/calibre-plugin/ffdl_plugin.py @@ -7,7 +7,7 @@ __license__ = 'GPL v3' __copyright__ = '2012, Jim Miller' __docformat__ = 'restructuredtext en' -import time, os +import time, os, copy from ConfigParser import SafeConfigParser from StringIO import StringIO from functools import partial @@ -78,6 +78,8 @@ class FanFictionDownLoaderPlugin(InterfaceAction): # Preferences->Plugins, which is why the do_user_config # method is defined on the base plugin class do_user_config = self.interface_action_base_plugin.do_user_config + base = self.interface_action_base_plugin + self.version = base.name+" v%d.%d.%d"%base.version # Set the icon for this interface action # The get_icons function is a builtin function defined for all your @@ -187,7 +189,11 @@ class FanFictionDownLoaderPlugin(InterfaceAction): #print("add_books:%s"%add_books) #print("options:%s"%d.get_ffdl_options()) - self.start_downloads( d.get_ffdl_options(),add_books ) + options = d.get_ffdl_options() + options['version'] = self.version + print(self.version) + + self.start_downloads( options, add_books ) def update_existing(self): #print("update_existing()") @@ -213,7 +219,10 @@ class FanFictionDownLoaderPlugin(InterfaceAction): #print("options:%s"%d.get_ffdl_options()) # only if there's some good ones. if 0 < len(filter(lambda x : x['good'], update_books)): - self.start_downloads( d.get_ffdl_options(), update_books ) + options = d.get_ffdl_options() + options['version'] = self.version + print(self.version) + self.start_downloads( options, update_books ) def get_urls_clip(self): url_list = [] @@ -272,6 +281,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction): return url = book['url'] + print("url:%s"%url) ## was self.ffdlconfig, but we need to be able to change it ## when doing epub update. @@ -333,7 +343,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction): if book['calibre_id'] != None: # updating an existing book. Update mode applies. - print("update id:%s"%book['calibre_id']) + print("update existing id:%s"%book['calibre_id']) book_id = book['calibre_id'] # No handling needed: OVERWRITEALWAYS,CALIBREONLY @@ -347,12 +357,12 @@ class FanFictionDownLoaderPlugin(InterfaceAction): (story.getMetadata("author", removeallentities=True),)) # author is a list. identicalbooks = db.find_identical_books(mi) ## removed for being overkill. - # for ib in db.find_identical_books(mi): + # for ib in identicalbooks: # # only *really* identical if URL matches, too. # # XXX make an option? # if self._get_story_url(db,ib) == url: # identicalbooks.append(ib) - print("identicalbooks:%s"%identicalbooks) + #print("identicalbooks:%s"%identicalbooks) if collision == SKIP and identicalbooks: raise NotGoingToDownload("Skipping duplicate story.","list_remove.png") @@ -418,6 +428,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction): tmp = PersistentTemporaryFile(prefix='new-%s-'%book['calibre_id'], suffix='.'+options['fileform'], dir=options['tdir']) + print("title:"+book['title']) print("outfile:"+tmp.name) book['outfile'] = tmp.name @@ -443,6 +454,18 @@ class FanFictionDownLoaderPlugin(InterfaceAction): self.download_list_completed(notjob,options=options) return + # ## XXX show list before starting download. + # d = DisplayStoryListDialog(self.gui, + # 'Download List', + # prefs, + # self.qaction.icon(), + # book_list, + # label_text='Status of stories to be downloaded' + # ) + # d.exec_() + # if d.result() != d.Accepted: + # return + for book in book_list: if book['good']: break @@ -496,71 +519,52 @@ class FanFictionDownLoaderPlugin(InterfaceAction): # for b in book_list: # print("d list: %s"%b['title']) - update_list = filter(lambda x : x['good'] and x['calibre_id'] != None, - book_list) + # update_list = filter(lambda x : x['good'] and x['calibre_id'] != None, + # book_list) - add_list = filter(lambda x : x['good'] and x['calibre_id'] == None, - book_list) + # add_list = filter(lambda x : x['good'] and x['calibre_id'] == None, + # book_list) - total_good = len(update_list)+len(add_list) + good_list = filter(lambda x : x['good'], book_list) + + total_good = len(good_list) #update_list)+len(add_list) self.gui.status_bar.show_message(_('Adding/Updating %s books.'%total_good), 3000) #print("==================================================") - addfiles,addfileforms,addmis=[],[],[] + # addfiles,addfileforms,addmis=[],[],[] + + list_000_ids = [] - added=0 - for book in add_list: - print("adding %s %s"%(book['title'],book['url'])) - addfiles.append(book['outfile']) - addfileforms.append(options['fileform']) - addmis.append(self._make_mi_from_book(book)) - - (notadded,added)=db.add_books(addfiles,addfileforms,addmis, - add_duplicates=True) - - #print("==================================================") - updated=0 - updated_ids = set() - for book in update_list: - print("updating (%s)%s %s"%(book['calibre_id'],book['title'],book['url'])) - - if options['collision'] == CALIBREONLY: - updated += 1 - else: - if not db.add_format_with_hooks(book['calibre_id'], options['fileform'], book['outfile'], index_is_id=True): - book['comment'] = "Adding format to book failed for some reason..." - book['good']=False - book['icon']='dialog_error.png' - else: - updated += 1 - updated_ids.add(book['calibre_id']) - # get all formats. - if prefs['deleteotherforms']: - fmts = set([x.lower() for x in db.formats(book['calibre_id'], index_is_id=True).split(',')]) - for fmt in fmts: - if fmt != options['fileform']: - print("remove f:"+fmt) - db.remove_format(book['calibre_id'], fmt, index_is_id=True)#, notify=False + for book in good_list: + print("add/update %s %s"%(book['title'],book['url'])) + mi = self._make_mi_from_book(book) + + if options['collision'] != CALIBREONLY: + self._add_or_update_book(book,options,prefs,mi) + # book000 = copy.copy(book) + # book000['title'] = "000 %s"%book000['title'] + # book000['calibre_id'] = self._find_existing_book_id(db,book000) + # list_000_ids.append(self._add_or_update_book(book000,options,prefs)) if options['collision'] == CALIBREONLY or \ - (options['updatemeta'] or book['good']) : - updated_ids.add(book['calibre_id']) - db.set_metadata(book['calibre_id'], - self._make_mi_from_book(book)) + (options['updatemeta'] and book['good']) : + db.set_metadata(book['calibre_id'],mi) - if added: + add_list = filter(lambda x : x['good'] and x['added'], book_list) + update_list = filter(lambda x : x['good'] and not x['added'], book_list) + update_ids = [ x['calibre_id'] for x in update_list ] + + if len(add_list)+len(list_000_ids): ## even shows up added to searchs. Nice. - self.gui.library_view.model().books_added(added) + self.gui.library_view.model().books_added(len(add_list)+len(list_000_ids)) - # the refresh causes problems sometimes? Switching libraries - # and back cleared it?--No problems with BG proc, but clears selected and search? - if updated_ids: - self.gui.library_view.model().refresh_ids(updated_ids) + if update_ids: + self.gui.library_view.model().refresh_ids(update_ids+list_000_ids) - self.gui.status_bar.show_message(_('Finished Adding/Updating %d books.'%(updated + added)), 3000) + self.gui.status_bar.show_message(_('Finished Adding/Updating %d books.'%(len(update_list) + len(add_list))), 3000) - if updated + added != total_good: + if len(update_list) + len(add_list) != total_good: d = DisplayStoryListDialog(self.gui, 'Updates completed, final status', prefs, @@ -573,6 +577,54 @@ class FanFictionDownLoaderPlugin(InterfaceAction): print("all done, remove temp dir.") remove_dir(options['tdir']) + def _add_or_update_book(self,book,options,prefs,mi=None): + db = self.gui.current_db + + if mi == None: + mi = self._make_mi_from_book(book) + + book_id = book['calibre_id'] + if book_id == None: + book_id = db.create_book_entry(mi, + add_duplicates=True) + book['calibre_id'] = book_id + book['added'] = True + else: + book['added'] = False + + if not db.add_format_with_hooks(book_id, + options['fileform'], + book['outfile'], index_is_id=True): + book['comment'] = "Adding format to book failed for some reason..." + book['good']=False + book['icon']='dialog_error.png' + + if prefs['deleteotherforms']: + fmts = db.formats(book['calibre_id'], index_is_id=True).split(',') + for fmt in fmts: + if fmt != formmapping[options['fileform']]: + print("remove f:"+fmt) + db.remove_format(book['calibre_id'], fmt, index_is_id=True)#, notify=False + + # rl_plugin = self.gui.iactions['Reading List'] + # rl_plugin.add_books_to_list("Send", + # [book_id], + # refresh_screen=False, + # display_warnings=False) + + return book_id + + def _find_existing_book_id(self,db,book,matchurl=True): + mi = MetaInformation(book["title"],(book["author"],)) # author is a list. + identicalbooks = db.find_identical_books(mi) + if matchurl: # only *really* identical if URL matches, too. + for ib in identicalbooks: + if self._get_story_url(db,ib) == book['url']: + return ib + if identicalbooks: + return identicalbooks.pop() + + def _make_mi_from_book(self,book): mi = MetaInformation(book['title'],(book['author'],)) # author is a list. @@ -598,6 +650,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction): book['comment'] = '' book['url'] = '' + book['added'] = False self._set_book_url_and_comment(book,url) @@ -622,6 +675,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction): book['comment'] = '' book['url'] = "" + book['added'] = False url = self._get_story_url(db,book_id) self._set_book_url_and_comment(book,url) diff --git a/calibre-plugin/jobs.py b/calibre-plugin/jobs.py index 3acfac4..071f560 100644 --- a/calibre-plugin/jobs.py +++ b/calibre-plugin/jobs.py @@ -42,11 +42,11 @@ def do_download_worker(book_list, options, ''' server = Server(pool_size=cpus) + print(options['version']) total = 0 # Queue all the jobs for book in book_list: if book['good']: - print("add job for %s"%book['url']) total += 1 args = ['calibre_plugins.fanfictiondownloader_plugin.jobs', 'do_download_for_worker', @@ -103,7 +103,6 @@ def do_download_for_worker(book,options): when run as a worker job ''' try: - print("do_download_for_worker") # print("is_adult:%s"%book['is_adult']) # print("personal.ini len:%s"%len(options['personal.ini'])) # print("defaults.ini len:%s"%len(get_resources("defaults.ini"))) @@ -148,6 +147,7 @@ def do_download_for_worker(book,options): striptitletoc=True, forceunique=False) print("Do update - epub(%d) vs url(%d)" % (chaptercount, urlchaptercount)) + print("write to %s"%outfile) ## Get updated title page/metadata by itself in an epub. ## Even if the title page isn't included, this carries the metadata. @@ -184,8 +184,7 @@ def do_download_for_worker(book,options): book['good']=False book['comment']=unicode(e) book['icon']='dialog_error.png' - print("%s:%s"%(book,unicode(e))) - # XXX trace for not-expected exceptions + print("Exception: %s:%s"%(book,unicode(e))) traceback.print_exc() #time.sleep(10)