From cbc8cadd91bc41a848a6310d24ab933766d60095 Mon Sep 17 00:00:00 2001 From: Jim Miller Date: Mon, 19 Dec 2011 14:59:20 -0600 Subject: [PATCH] First calibre-plugin version with working background downloading. --- calibre-plugin/__init__.py | 9 ++ calibre-plugin/jobs.py | 72 +++++++++ calibre-plugin/plugin.py | 168 +++++++++------------ calibre-plugin/ui.py | 164 +++++++++++++++++++- fanficdownloader/adapters/adapter_test1.py | 4 +- fanficdownloader/adapters/base_adapter.py | 1 + 6 files changed, 309 insertions(+), 109 deletions(-) create mode 100644 calibre-plugin/jobs.py diff --git a/calibre-plugin/__init__.py b/calibre-plugin/__init__.py index 73485ed..c66c7f4 100644 --- a/calibre-plugin/__init__.py +++ b/calibre-plugin/__init__.py @@ -79,3 +79,12 @@ class InterfacePluginDemo(InterfaceActionBase): ac.apply_settings() + + +# For testing, run from command line with this: +# calibre-debug -e __init__.py +if __name__ == '__main__': + from PyQt4.Qt import QApplication + from calibre.gui2.preferences import test_widget + app = QApplication([]) + test_widget('Advanced', 'Plugins') diff --git a/calibre-plugin/jobs.py b/calibre-plugin/jobs.py new file mode 100644 index 0000000..30c92ae --- /dev/null +++ b/calibre-plugin/jobs.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2011, Jim Miller' +__docformat__ = 'restructuredtext en' + +import os, traceback, time + +import ConfigParser + +from calibre.ebooks import DRMError +from calibre.ptempfile import PersistentTemporaryFile +from calibre.ebooks.metadata import MetaInformation + +from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapters,writers,exceptions + +def do_story_downloads(adaptertuple_list, fileform, db, + abort=None, log=None, notifications=[]): # lambda x,y:x lambda makes small anonymous function. + ''' + Master job, to launch child jobs to download this list of stories + ''' + print("do_story_downloads") + notifications.put((0.01, 'Start Downloading Stories')) + count = 0 + total = len(adaptertuple_list) + # Queue all the jobs + for (url,adapter) in adaptertuple_list: + do_story_download(adapter,fileform,db) + count = count + 1 + notifications.put((float(count)/total, 'Downloading Stories')) + # return the map as the job result + # return book_pages_map, book_words_map + return {},{} + +def do_story_download(adapter,fileform,db): + print("do_story_download") + +# ffdlconfig = ConfigParser.SafeConfigParser() +# adapter = adapters.getAdapter(ffdlconfig,url) + + story = adapter.getStoryMetadataOnly() + + mi = MetaInformation(story.getMetadata("title"), + (story.getMetadata("author"),)) # author is a list. + + writer = writers.getWriter(fileform,adapter.config,adapter) + tmp = PersistentTemporaryFile("."+fileform) + print("tmp: "+tmp.name) + + writer.writeStory(tmp) + + print("post write tmp: "+tmp.name) + + mi.set_identifiers({'url':story.getMetadata("storyUrl")}) + mi.publisher = story.getMetadata("site") + + mi.tags = writer.getTags() + mi.languages = ['en'] + mi.pubdate = story.getMetadataRaw('datePublished').strftime("%Y-%m-%d") + mi.timestamp = story.getMetadataRaw('dateCreated').strftime("%Y-%m-%d") + mi.comments = story.getMetadata("description") + + (notadded,addedcount)=db.add_books([tmp],[fileform],[mi], add_duplicates=True) + # Otherwise list of books doesn't update right away. + #self.gui.library_view.model().books_added(addedcount) + + del adapter + del writer + diff --git a/calibre-plugin/plugin.py b/calibre-plugin/plugin.py index 78d2154..b171af1 100644 --- a/calibre-plugin/plugin.py +++ b/calibre-plugin/plugin.py @@ -1,38 +1,26 @@ #!/usr/bin/env python # vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai -from __future__ import (unicode_literals, division, absolute_import, +from __future__ import (unicode_literals, division, print_function) __license__ = 'GPL v3' __copyright__ = '2011, Fanficdownloader team' __docformat__ = 'restructuredtext en' -from StringIO import StringIO +from PyQt4.Qt import (QDialog, QMessageBox, QVBoxLayout, QGridLayout, QPushButton, QProgressDialog, QString, + QLabel, QLineEdit, QInputDialog, QComboBox, QProgressDialog, QTimer ) -from PyQt4.Qt import (QDialog, QVBoxLayout, QGridLayout, QPushButton, - QLabel, QLineEdit, QInputDialog, QComboBox ) +from calibre.gui2 import error_dialog, warning_dialog, question_dialog -from calibre.ptempfile import PersistentTemporaryFile -from calibre.ebooks.metadata import MetaInformation -from calibre.gui2 import question_dialog - -from calibre_plugins.fanfictiondownloader_plugin.config import prefs from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapters,writers,exceptions -import ConfigParser +class DownloadDialog(QDialog): -class DemoDialog(QDialog): - - def __init__(self, gui, icon, do_user_config): + def __init__(self, gui, icon, do_user_config, pluginaction): QDialog.__init__(self, gui) self.gui = gui self.do_user_config = do_user_config - - # The current database shown in the GUI - # db is an instance of the class LibraryDatabase2 from database.py - # This class has many, many methods that allow you to do a lot of - # things. - self.db = gui.current_db + self.pluginaction = pluginaction self.l = QVBoxLayout() self.setLayout(self.l) @@ -46,12 +34,12 @@ class DemoDialog(QDialog): self.l.addWidget(self.url) self.l.addWidget(QLabel('Output Format:')) - self.format = QComboBox(self) - self.format.addItem('epub') - self.format.addItem('mobi') - self.format.addItem('html') - self.format.addItem('txt') - self.l.addWidget(self.format) + self.fileform = QComboBox(self) + self.fileform.addItem('epub') + self.fileform.addItem('mobi') + self.fileform.addItem('html') + self.fileform.addItem('txt') + self.l.addWidget(self.fileform) self.ffdl_button = QPushButton( 'Download Story', self) @@ -84,81 +72,9 @@ class DemoDialog(QDialog): text.decode('utf-8')) def ffdl(self): - - config = ConfigParser.SafeConfigParser() - config.readfp(StringIO(get_resources("defaults.ini"))) - config.readfp(StringIO(prefs['personal.ini'])) - print("URL:"+unicode(self.url.text())) - - adapter = adapters.getAdapter(config,unicode(self.url.text())) - - try: - adapter.getStoryMetadataOnly() - except exceptions.FailedToLogin: - print("Login Failed, Need Username/Password.") - userpass = UserPassDialog(self.gui,adapter.getSiteDomain()) - userpass.exec_() # exec_ will make it act modal - if userpass.status: - adapter.username = userpass.user.text() - adapter.password = userpass.passwd.text() - else: - del adapter - return - except exceptions.AdultCheckRequired: - if question_dialog(self.gui, 'Are You Adult?', '

'+ - "This story requires that you be an adult. Please confirm you are an adult in your locale:", - show_copy_button=False): - adapter.is_adult=True - else: - del adapter - return -# except exceptions.StoryDoesNotExist - - story = adapter.getStoryMetadataOnly() - fileform = unicode(self.format.currentText()) - - mi = MetaInformation(story.getMetadata("title"), - (story.getMetadata("author"),)) # author is a list. - - add=True - identicalbooks = self.db.find_identical_books(mi) - if identicalbooks: - add=False - if question_dialog(self.gui, 'Add Duplicate?', '

'+ - "That story is already in your library. Create a new one?", - show_copy_button=False): - add=True - - if add: - writer = writers.getWriter(fileform,config,adapter) - tmp = PersistentTemporaryFile("."+fileform) - print("tmp: "+tmp.name) - - writer.writeStory(tmp) - - mi.set_identifiers({'url':story.getMetadata("storyUrl")}) - mi.publisher = story.getMetadata("site") - - mi.tags = writer.getTags() - mi.languages = ['en'] - mi.pubdate = story.getMetadataRaw('datePublished').strftime("%Y-%m-%d") - mi.timestamp = story.getMetadataRaw('dateCreated').strftime("%Y-%m-%d") - mi.comments = story.getMetadata("description") - - (notadded,addedcount)=self.db.add_books([tmp],[fileform],[mi], add_duplicates=True) - # Otherwise list of books doesn't update right away. - self.gui.library_view.model().books_added(addedcount) - + self.pluginaction.start_downloads(unicode(self.url.text()), + unicode(self.fileform.currentText())) self.hide() - - # QMessageBox.about(self, 'FFDL Metadata', - # str(adapter.getStoryMetadataOnly()).decode('utf-8')) - del adapter - try: - del writer - except: - pass - def config(self): self.do_user_config(parent=self) @@ -203,3 +119,57 @@ class UserPassDialog(QDialog): def cancel(self): self.status=False self.hide() + +class QueueProgressDialog(QProgressDialog): + + def __init__(self, gui, title, loop_list, fileform, loop_function, enqueue_function, db): + QProgressDialog.__init__(self, title, QString(), 0, len(loop_list), gui) + self.setWindowTitle(title) + self.setMinimumWidth(500) + self.gui = gui + self.db = db + self.loop_list = loop_list + self.fileform = fileform + self.loop_function = loop_function + self.enqueue_function = enqueue_function + # self.book_ids, self.tdir, self.format_order, self.queue, self.db = \ + # book_ids, tdir, format_order, queue, db + # self.pages_algorithm, self.pages_custom_column = pages_algorithm, pages_col + # self.words_algorithm, self.words_custom_column = words_algorithm, words_col + self.i, self.loop_bad, self.loop_good = 0, [], [] + self.setValue(0) + self.setLabelText("Fetching metadata for %d of %d"%(0,len(self.loop_list))) + QTimer.singleShot(0, self.do_loop) + self.exec_() + + def do_loop(self): + current = self.loop_list[self.i] + self.i += 1 + + try: + retval = self.loop_function(current,self.fileform) + self.loop_good.append((current,retval)) + except Exception as e: + self.loop_bad.append(current) + + self.setValue(self.i) + self.setLabelText("Fetching metadata for %d of %d"%(self.i,len(self.loop_list))) + if self.i >= len(self.loop_list): + return self.do_queue() + else: + QTimer.singleShot(0, self.do_loop) + + def do_queue(self): + self.hide() + if self.loop_bad != []: + res = [] + for current in self.loop_bad: + res.append('%s'%current) + msg = '%s' % '\n'.join(res) + warning_dialog(self.gui, _('Could not get metadata for some stories'), + _('Could not get metadata for %d of %d stories.') % + (len(self.loop_bad), len(self.loop_list)), + msg).exec_() + self.gui = None + # Queue a job to process these ePub/Mobi books + self.enqueue_function(self.loop_good,self.fileform) diff --git a/calibre-plugin/ui.py b/calibre-plugin/ui.py index 8b5f94c..c5dca7c 100644 --- a/calibre-plugin/ui.py +++ b/calibre-plugin/ui.py @@ -7,15 +7,23 @@ __license__ = 'GPL v3' __copyright__ = '2011, Fanficdownloader team' __docformat__ = 'restructuredtext en' -if False: - # This is here to keep my python error checker from complaining about - # the builtin functions that will be defined by the plugin loading system - # You do not need this code in your plugins - get_icons = get_resources = None +from StringIO import StringIO +import ConfigParser + +from PyQt4.Qt import (QMessageBox) # The class that all interface action plugins must inherit from from calibre.gui2.actions import InterfaceAction -from calibre_plugins.fanfictiondownloader_plugin.plugin import DemoDialog +from calibre.ebooks.metadata import MetaInformation +from calibre.gui2 import error_dialog, warning_dialog, question_dialog, info_dialog +from calibre.gui2.threaded_jobs import ThreadedJobServer, ThreadedJob + +from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapters,writers,exceptions + +from calibre_plugins.fanfictiondownloader_plugin.config import prefs +from calibre_plugins.fanfictiondownloader_plugin.plugin import (DownloadDialog, QueueProgressDialog,UserPassDialog) + +from calibre_plugins.fanfictiondownloader_plugin.jobs import do_story_downloads class InterfacePlugin(InterfaceAction): @@ -59,15 +67,155 @@ class InterfacePlugin(InterfaceAction): # method is defined on the base plugin class do_user_config = base_plugin_object.do_user_config + # The current database shown in the GUI + # db is an instance of the class LibraryDatabase2 from database.py + # This class has many, many methods that allow you to do a lot of + # things. + self.db = self.gui.current_db + # self.gui is the main calibre GUI. It acts as the gateway to access # all the elements of the calibre user interface, it should also be the # parent of the dialog - d = DemoDialog(self.gui, self.qaction.icon(), do_user_config) + d = DownloadDialog(self.gui, self.qaction.icon(), do_user_config, self) d.show() def apply_settings(self): - from calibre_plugins.fanfictiondownloader_plugin.config import prefs # In an actual non trivial plugin, you would probably need to # do something based on the settings in prefs prefs + + def start_downloads(self,url,fileform): + self.ffdlconfig = ConfigParser.SafeConfigParser() + self.ffdlconfig.readfp(StringIO(get_resources("defaults.ini"))) + self.ffdlconfig.readfp(StringIO(prefs['personal.ini'])) + + ## XXX including code for Things to Come, namely, a list of + ## URLs rather than just one. + url_list = [url, + #"http://test1.com?sid=6700", + #"http://test1.com?sid=6701", + # "http://test1.com?sid=6702", + # "http://test1.com?sid=6703", + # "http://test1.com?sid=6704", + # "http://test1.com?sid=6705", + # "http://test1.com?sid=6706" + ] + + self.fetchmeta_qpd = \ + QueueProgressDialog(self.gui, + "Getting Metadata for Stories", + url_list, + fileform, + self.get_adapter_for_story, + self.enqueue_story_list_for_download, + self.db) + + def get_adapter_for_story(self,url,fileform): + ''' + Returns adapter object for story at URL. To be called from + QueueProgressDialog 'loop' to build up list of adapters. Also + pops dialogs for is adult, user/pass, duplicate + ''' + + print("URL:"+url) + adapter = adapters.getAdapter(self.ffdlconfig,url) + + try: + adapter.getStoryMetadataOnly() + except exceptions.FailedToLogin: + print("Login Failed, Need Username/Password.") + userpass = UserPassDialog(self.gui,url) + userpass.exec_() # exec_ will make it act modal + if userpass.status: + adapter.username = userpass.user.text() + adapter.password = userpass.passwd.text() + # else: + # del adapter + # return + except exceptions.AdultCheckRequired: + if question_dialog(self.gui, 'Are You Adult?', '

'+ + "%s requires that you be an adult. Please confirm you are an adult in your locale:"%url, + show_copy_button=False): + adapter.is_adult=True + # else: + # del adapter + # return + + # let exceptions percolate up. + story = adapter.getStoryMetadataOnly() + + mi = MetaInformation(story.getMetadata("title"), + (story.getMetadata("author"),)) # author is a list. + + add=True + identicalbooks = self.db.find_identical_books(mi) + if identicalbooks: + add=False + if question_dialog(self.gui, 'Add Duplicate?', '

'+ + "That story is already in your library. Create a new one?", + show_copy_button=False): + add=True + + if add: + return adapter + else: + return None + + def enqueue_story_list_for_download(self,adaptertuple_list,fileform): + ''' + Called by QueueProgressDialog to enqueue story downloads for BG processing. + adapter_list is a list of tuples of (url,adapter) + ''' + print("enqueue_story_list_for_download") + print(adaptertuple_list) + + func = 'arbitrary_n' + args = ['calibre_plugins.fanfictiondownloader_plugin.jobs', 'do_story_downloads', + (adaptertuple_list, fileform)] # adaptertuple_list + desc = 'Download FanFiction Stories' + + print("pre self.gui.job_manager.run_job") + # job = self.gui.job_manager.run_job( + # self.Dispatcher(self._get_stories_completed), func, args=args, + # description=desc) + + job = ThreadedJob('FanFictionDownload', + 'Downloading FanFiction Stories', + func=do_story_downloads, + args=(adaptertuple_list, fileform, self.db), + kwargs={}, + callback=self._get_stories_completed) + + self.gui.job_manager.run_threaded_job(job) + print("post self.gui.job_manager.run_job") + # job.tdir = tdir + # job.pages_custom_column = pages_custom_column + # job.words_custom_column = words_custom_column + self.gui.status_bar.show_message('Downloading %d stories'%len(adaptertuple_list)) + + def _get_stories_completed(self, job): + print("_get_stories_completed") + # remove_dir(job.tdir) + if job.failed: + return self.gui.job_exception(job, dialog_title='Failed to download stories') +# self.gui.status_bar.show_message('Download Stories completed', 3000) + # not really, but leave it for now. + book_pages_map, book_words_map = job.result + + # if len(book_pages_map) + len(book_words_map) == 0: + # # Must have been some sort of error in processing this book + # msg = 'Failed to generate any statistics. View Log for details' + # p = ErrorNotification(job.details, 'Count log', 'Count Pages failed', msg, + # show_copy_button=False, parent=self.gui) + # else: + # payload = (book_pages_map, job.pages_custom_column, book_words_map, job.words_custom_column) + # all_ids = set(book_pages_map.keys()) | set(book_words_map.keys()) + # msg = '

Count Pages plugin found %d statistics(s). ' % len(all_ids) + \ + # 'Proceed with updating your library?' + # p = ProceedNotification(self._update_database_columns, + # payload, job.details, + # 'Count log', 'Count complete', msg, + # show_copy_button=False, parent=self.gui) + # p.show() + #info_dialog(self.gui,'FFDL Complete','It worked?') diff --git a/fanficdownloader/adapters/adapter_test1.py b/fanficdownloader/adapters/adapter_test1.py index 2bc8668..f498cc0 100644 --- a/fanficdownloader/adapters/adapter_test1.py +++ b/fanficdownloader/adapters/adapter_test1.py @@ -54,7 +54,7 @@ class TestSiteAdapter(BaseSiteAdapter): if self.story.getMetadata('storyId') == '666': raise exceptions.StoryDoesNotExist(self.url) - if self.story.getMetadata('storyId') == '670': + if self.story.getMetadata('storyId').startswith('670'): time.sleep(2.0) if self.getConfig("username"): @@ -131,7 +131,7 @@ Some more longer description. "I suck at summaries!" "Better than it sounds!" if self.story.getMetadata('storyId') == '667': raise exceptions.FailedToDownload("Error downloading Chapter: %s!" % url) - if self.story.getMetadata('storyId') == '670': + if self.story.getMetadata('storyId').startswith('670'): time.sleep(2.0) if "chapter=1" in url : diff --git a/fanficdownloader/adapters/base_adapter.py b/fanficdownloader/adapters/base_adapter.py index 6c9d3a1..4a4642e 100644 --- a/fanficdownloader/adapters/base_adapter.py +++ b/fanficdownloader/adapters/base_adapter.py @@ -63,6 +63,7 @@ class BaseSiteAdapter(Configurable): return re.match(self.getSiteURLPattern(), self.url) def __init__(self, config, url): + self.config = config Configurable.__init__(self, config) self.addConfigSection(self.getSiteDomain()) self.addConfigSection("overrides")