From 56ac9127063cb0be91d7c76d448a7b3ad17646e1 Mon Sep 17 00:00:00 2001 From: Jim Miller Date: Mon, 19 Dec 2011 21:18:55 -0600 Subject: [PATCH] Rename some files, further improvements--just about usable for new stories only. --- calibre-plugin/__init__.py | 10 +- calibre-plugin/{plugin.py => dialogs.py} | 66 +++++++--- calibre-plugin/{ui.py => ffdl_plugin.py} | 138 +++++++++------------ fanficdownloader/adapters/adapter_test1.py | 12 +- 4 files changed, 122 insertions(+), 104 deletions(-) rename calibre-plugin/{plugin.py => dialogs.py} (75%) rename calibre-plugin/{ui.py => ffdl_plugin.py} (72%) diff --git a/calibre-plugin/__init__.py b/calibre-plugin/__init__.py index 66899cb..5ae0bde 100644 --- a/calibre-plugin/__init__.py +++ b/calibre-plugin/__init__.py @@ -13,10 +13,10 @@ from calibre.customize import InterfaceActionBase class InterfacePluginDemo(InterfaceActionBase): ''' - This class is a simple wrapper that provides information about the actual - plugin class. The actual interface plugin class is called InterfacePlugin - and is defined in the ui.py file, as specified in the actual_plugin field - below. + This class is a simple wrapper that provides information about the + actual plugin class. The actual interface plugin class is called + InterfacePlugin and is defined in the ffdl_plugin.py file, as + specified in the actual_plugin field below. The reason for having two classes is that it allows the command line calibre utilities to run without needing to load the GUI libraries. @@ -31,7 +31,7 @@ class InterfacePluginDemo(InterfaceActionBase): #: This field defines the GUI plugin class that contains all the code #: that actually does something. Its format is module_path:class_name #: The specified class must be defined in the specified module. - actual_plugin = 'calibre_plugins.fanfictiondownloader_plugin.ui:FanFictionDownLoaderPlugin' + actual_plugin = 'calibre_plugins.fanfictiondownloader_plugin.ffdl_plugin:FanFictionDownLoaderPlugin' def is_customizable(self): ''' diff --git a/calibre-plugin/plugin.py b/calibre-plugin/dialogs.py similarity index 75% rename from calibre-plugin/plugin.py rename to calibre-plugin/dialogs.py index 2e956a2..b68f5b7 100644 --- a/calibre-plugin/plugin.py +++ b/calibre-plugin/dialogs.py @@ -12,7 +12,7 @@ from PyQt4.Qt import (QDialog, QMessageBox, QVBoxLayout, QGridLayout, QTextEdit, QLineEdit, QInputDialog, QComboBox, QProgressDialog, QTimer ) -from calibre.gui2 import error_dialog, warning_dialog, question_dialog +from calibre.gui2 import error_dialog, warning_dialog, question_dialog, info_dialog from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapters,writers,exceptions @@ -34,7 +34,17 @@ class DownloadDialog(QDialog): self.l.addWidget(QLabel('Story URL(s), one per line:')) self.url = QTextEdit(self) self.url.setLineWrapMode(QTextEdit.NoWrap) - self.url.setText('http://test1.com?sid=12345') +# self.url.setText('''http://test1.com?sid=6700 +# http://test1.com?sid=6701 +# http://test1.com?sid=6702 +# http://test1.com?sid=6703 +# http://test1.com?sid=6704 +# http://test1.com?sid=6705 +# http://test1.com?sid=6706 +# http://test1.com?sid=6707 +# http://test1.com?sid=6708 +# http://test1.com?sid=6709 +# ''') self.l.addWidget(self.url) # self.url = QLineEdit(self) @@ -133,9 +143,11 @@ class MetadataProgressDialog(QProgressDialog): ''' ProgressDialog displayed while fetching metadata for each story. ''' - def __init__(self, gui, title, loop_list, fileform, getadapter_function, download_list_function, db): - QProgressDialog.__init__(self, title, QString(), 0, len(loop_list), gui) - self.setWindowTitle(title) + def __init__(self, gui, loop_list, fileform, getadapter_function, download_list_function, db): + QProgressDialog.__init__(self, + "Fetching metadata for stories...", + QString(), 0, len(loop_list), gui) + self.setWindowTitle("Downloading metadata for stories") self.setMinimumWidth(500) self.gui = gui self.db = db @@ -144,32 +156,44 @@ class MetadataProgressDialog(QProgressDialog): self.getadapter_function = getadapter_function self.download_list_function = download_list_function self.i, self.loop_bad, self.loop_good = 0, [], [] + + ## self.do_loop does QTimer.singleShot on self.do_loop also. + ## A weird way to do a loop, but that was the example I had. QTimer.singleShot(0, self.do_loop) self.exec_() - def bump(self): - self.i += 1 - self.setValue(self.i) - self.setLabelText("Fetching metadata for %d of %d"%(0,len(self.loop_list))) - + def updateStatus(self): + self.setLabelText("Fetched metadata for %d of %d"%(self.i+1,len(self.loop_list))) + self.setValue(self.i+1) + print(self.labelText()) def do_loop(self): - current = self.loop_list[self.i] + print("self.i:%d"%self.i) - self.bump() - try: - retval = self.getadapter_function(current,self.fileform) - self.loop_good.append((current,retval)) - except Exception as e: - self.loop_bad.append((current,e)) + if self.i == 0: + self.setValue(0) - if self.i >= len(self.loop_list): + if self.i >= len(self.loop_list) or self.wasCanceled(): return self.do_when_finished() + else: + current = self.loop_list[self.i] + try: + retval = self.getadapter_function(current,self.fileform) + self.loop_good.append((current,retval)) + except Exception as e: + self.loop_bad.append((current,e)) + + self.updateStatus() + self.i += 1 QTimer.singleShot(0, self.do_loop) def do_when_finished(self): self.hide() + + # Queues a job to process these ePub/Mobi books in the background. + self.download_list_function(self.loop_good,self.fileform) + if self.loop_bad != []: res = [] for j in self.loop_bad: @@ -179,6 +203,8 @@ class MetadataProgressDialog(QProgressDialog): _('Could not get metadata for %d of %d stories.') % (len(self.loop_bad), len(self.loop_list)), msg).exec_() + else: + info_dialog(self.gui, "Starting Downloads", + "Got metadata and started download for %d stories."%len(self.loop_good), + show_copy_button=False).exec_() self.gui = None - # Queue a job to process these ePub/Mobi books - self.download_list_function(self.loop_good,self.fileform) diff --git a/calibre-plugin/ui.py b/calibre-plugin/ffdl_plugin.py similarity index 72% rename from calibre-plugin/ui.py rename to calibre-plugin/ffdl_plugin.py index 478e6de..e14fa63 100644 --- a/calibre-plugin/ui.py +++ b/calibre-plugin/ffdl_plugin.py @@ -19,9 +19,7 @@ from calibre.gui2.threaded_jobs import ThreadedJob from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapters, writers, exceptions from calibre_plugins.fanfictiondownloader_plugin.config import prefs -from calibre_plugins.fanfictiondownloader_plugin.plugin import DownloadDialog, MetadataProgressDialog, UserPassDialog - -#from calibre_plugins.fanfictiondownloader_plugin.jobs import do_story_downloads +from calibre_plugins.fanfictiondownloader_plugin.dialogs import DownloadDialog, MetadataProgressDialog, UserPassDialog class FanFictionDownLoaderPlugin(InterfaceAction): @@ -94,9 +92,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction): self.ffdlconfig.readfp(StringIO(get_resources("defaults.ini"))) self.ffdlconfig.readfp(StringIO(prefs['personal.ini'])) - ## XXX including code for Things to Come, namely, a list of - ## URLs rather than just one. - url_list = urls.splitlines() + url_list = get_url_list(urls) ''' http://test1.com?sid=6700 http://test1.com?sid=6701 @@ -109,7 +105,6 @@ class FanFictionDownLoaderPlugin(InterfaceAction): self.fetchmeta_qpd = \ MetadataProgressDialog(self.gui, - "Getting Metadata for Stories", url_list, fileform, self.get_adapter_for_story, @@ -158,7 +153,8 @@ class FanFictionDownLoaderPlugin(InterfaceAction): if identicalbooks: add=False if question_dialog(self.gui, 'Add Duplicate?', '

'+ - "That story is already in your library. Create a new one?", + "%s by %s is already in your library. Create a new one?"% + (story.getMetadata("title"),story.getMetadata("author")), show_copy_button=False): add=True @@ -169,34 +165,20 @@ class FanFictionDownLoaderPlugin(InterfaceAction): def download_list(self,adaptertuple_list,fileform): ''' - Called by MetadataProgressDialog to enqueue story downloads for BG processing. + Called by MetadataProgressDialog to start story downloads BG processing. adapter_list is a list of tuples of (url,adapter) ''' - print("enqueue_story_list_for_download") - print(adaptertuple_list) + print("download_list") - func = 'arbitrary_n' - args = ['calibre_plugins.fanfictiondownloader_plugin.jobs', 'do_story_downloads', - (adaptertuple_list, fileform)] # adaptertuple_list - desc = 'Download FanFiction Stories' - - print("pre self.gui.job_manager.run_job") - # job = self.gui.job_manager.run_job( - # self.Dispatcher(self._get_stories_completed), func, args=args, - # description=desc) - job = ThreadedJob('FanFictionDownload', 'Downloading FanFiction Stories', - func=do_story_downloads, + func=self.do_story_downloads, args=(adaptertuple_list, fileform, self.db), kwargs={}, callback=self._get_stories_completed) self.gui.job_manager.run_threaded_job(job) - print("post self.gui.job_manager.run_job") - # job.tdir = tdir - # job.pages_custom_column = pages_custom_column - # job.words_custom_column = words_custom_column + self.gui.status_bar.show_message('Downloading %d stories'%len(adaptertuple_list)) def _get_stories_completed(self, job): @@ -225,57 +207,61 @@ class FanFictionDownLoaderPlugin(InterfaceAction): # p.show() #info_dialog(self.gui,'FFDL Complete','It worked?') -def do_story_downloads(adaptertuple_list, fileform, db, - abort=None, log=None, notifications=[]): # lambda x,y:x lambda makes small anonymous function. - ''' - Master job, to launch child jobs to download this list of stories - ''' - print("do_story_downloads") - notifications.put((0.01, 'Start Downloading Stories')) - count = 0 - total = len(adaptertuple_list) - # Queue all the jobs - for (url,adapter) in adaptertuple_list: - do_story_download(adapter,fileform,db) - count = count + 1 - notifications.put((float(count)/total, 'Downloading Stories')) - # return the map as the job result - # return book_pages_map, book_words_map - return {},{} - -def do_story_download(adapter,fileform,db): - print("do_story_download") - -# ffdlconfig = ConfigParser.SafeConfigParser() -# adapter = adapters.getAdapter(ffdlconfig,url) - - story = adapter.getStoryMetadataOnly() - - mi = MetaInformation(story.getMetadata("title"), - (story.getMetadata("author"),)) # author is a list. + def do_story_downloads(self, adaptertuple_list, fileform, db, + **kwargs): # lambda x,y:x lambda makes small anonymous function. + # abort=None, log=None, + ''' + Master job, loop to download this list of stories + ''' + print("do_story_downloads") + notifications=kwargs['notifications'] + notifications.put((0.01, 'Start Downloading Stories')) + count = 0 + total = len(adaptertuple_list) + # Queue all the jobs + for (url,adapter) in adaptertuple_list: + self.do_story_download(adapter,fileform,db) + count = count + 1 + notifications.put((float(count)/total, 'Downloading Stories')) + # return the map as the job result + # return book_pages_map, book_words_map + return {},{} - writer = writers.getWriter(fileform,adapter.config,adapter) - tmp = PersistentTemporaryFile("."+fileform) - print("tmp: "+tmp.name) + def do_story_download(self,adapter,fileform,db): + print("do_story_download") - writer.writeStory(tmp) + story = adapter.getStoryMetadataOnly() - print("post write tmp: "+tmp.name) - - mi.set_identifiers({'url':story.getMetadata("storyUrl")}) - mi.publisher = story.getMetadata("site") - - mi.tags = writer.getTags() - mi.languages = ['en'] - mi.pubdate = story.getMetadataRaw('datePublished').strftime("%Y-%m-%d") - mi.timestamp = story.getMetadataRaw('dateCreated').strftime("%Y-%m-%d") - mi.comments = story.getMetadata("description") - - (notadded,addedcount)=db.add_books([tmp],[fileform],[mi], add_duplicates=True) - # Otherwise list of books doesn't update right away. - #self.gui.library_view.model().books_added(addedcount) - - del adapter - del writer - + mi = MetaInformation(story.getMetadata("title"), + (story.getMetadata("author"),)) # author is a list. + writer = writers.getWriter(fileform,adapter.config,adapter) + tmp = PersistentTemporaryFile("."+fileform) + print("tmp: "+tmp.name) + + writer.writeStory(tmp) + + print("post write tmp: "+tmp.name) + + mi.set_identifiers({'url':story.getMetadata("storyUrl")}) + mi.publisher = story.getMetadata("site") + + mi.tags = writer.getTags() + mi.languages = ['en'] + mi.pubdate = story.getMetadataRaw('datePublished').strftime("%Y-%m-%d") + mi.timestamp = story.getMetadataRaw('dateCreated').strftime("%Y-%m-%d") + mi.comments = story.getMetadata("description") + + (notadded,addedcount)=db.add_books([tmp],[fileform],[mi], add_duplicates=True) + # Otherwise list of books doesn't update right away. + self.gui.library_view.model().books_added(addedcount) + + del adapter + del writer + +def f(x): + if x.strip(): return True + else: return False + +def get_url_list(urls): + return filter(f,urls.strip().splitlines()) diff --git a/fanficdownloader/adapters/adapter_test1.py b/fanficdownloader/adapters/adapter_test1.py index f498cc0..ef33e42 100644 --- a/fanficdownloader/adapters/adapter_test1.py +++ b/fanficdownloader/adapters/adapter_test1.py @@ -55,7 +55,10 @@ class TestSiteAdapter(BaseSiteAdapter): raise exceptions.StoryDoesNotExist(self.url) if self.story.getMetadata('storyId').startswith('670'): - time.sleep(2.0) + time.sleep(1.0) + + if self.story.getMetadata('storyId').startswith('671'): + time.sleep(1.0) if self.getConfig("username"): self.username = self.getConfig("username") @@ -131,8 +134,9 @@ Some more longer description. "I suck at summaries!" "Better than it sounds!" if self.story.getMetadata('storyId') == '667': raise exceptions.FailedToDownload("Error downloading Chapter: %s!" % url) - if self.story.getMetadata('storyId').startswith('670'): - time.sleep(2.0) + if self.story.getMetadata('storyId').startswith('670') or \ + self.story.getMetadata('storyId').startswith('672'): + time.sleep(1.0) if "chapter=1" in url : text=u''' @@ -146,6 +150,8 @@ Some more longer description. "I suck at summaries!" "Better than it sounds!"

http://test1.com?sid=668 - raises FailedToLogin unless username='Me'

http://test1.com?sid=669 - Succeeds with Updated Date=now

http://test1.com?sid=670 - Succeeds, but sleeps 2sec on each chapter

+

http://test1.com?sid=671 - Succeeds, but sleeps 2sec metadata only

+

http://test1.com?sid=672 - Succeeds, quick meta, sleeps 2sec chapters only

And other storyId will succeed with the same output.

'''