Rename some files, further improvements--just about usable for new stories only.

2026-09-13 12:11:20 +08:00 · 2011-12-19 21:18:55 -06:00
parent b75c9d4946
commit 56ac912706
4 changed files with 122 additions and 104 deletions
@@ -13,10 +13,10 @@ from calibre.customize import InterfaceActionBase

 class InterfacePluginDemo(InterfaceActionBase):
    '''
-    This class is a simple wrapper that provides information about the actual
-    plugin class. The actual interface plugin class is called InterfacePlugin
-    and is defined in the ui.py file, as specified in the actual_plugin field
-    below.
+    This class is a simple wrapper that provides information about the
+    actual plugin class. The actual interface plugin class is called
+    InterfacePlugin and is defined in the ffdl_plugin.py file, as
+    specified in the actual_plugin field below.

    The reason for having two classes is that it allows the command line
    calibre utilities to run without needing to load the GUI libraries.
@@ -31,7 +31,7 @@ class InterfacePluginDemo(InterfaceActionBase):
    #: This field defines the GUI plugin class that contains all the code
    #: that actually does something. Its format is module_path:class_name
    #: The specified class must be defined in the specified module.
-    actual_plugin       = 'calibre_plugins.fanfictiondownloader_plugin.ui:FanFictionDownLoaderPlugin'
+    actual_plugin       = 'calibre_plugins.fanfictiondownloader_plugin.ffdl_plugin:FanFictionDownLoaderPlugin'

    def is_customizable(self):
        '''
@@ -12,7 +12,7 @@ from PyQt4.Qt import (QDialog, QMessageBox, QVBoxLayout, QGridLayout,
                      QTextEdit, QLineEdit, QInputDialog, QComboBox,
                      QProgressDialog, QTimer )

-from calibre.gui2 import error_dialog, warning_dialog, question_dialog
+from calibre.gui2 import error_dialog, warning_dialog, question_dialog, info_dialog

 from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapters,writers,exceptions

@@ -34,7 +34,17 @@ class DownloadDialog(QDialog):
        self.l.addWidget(QLabel('Story URL(s), one per line:'))
        self.url = QTextEdit(self)
        self.url.setLineWrapMode(QTextEdit.NoWrap)
-        self.url.setText('http://test1.com?sid=12345')
+#         self.url.setText('''http://test1.com?sid=6700
+# http://test1.com?sid=6701
+# http://test1.com?sid=6702
+# http://test1.com?sid=6703
+# http://test1.com?sid=6704
+# http://test1.com?sid=6705
+# http://test1.com?sid=6706
+# http://test1.com?sid=6707
+# http://test1.com?sid=6708
+# http://test1.com?sid=6709
+# ''')
        self.l.addWidget(self.url)
        
        # self.url = QLineEdit(self)
@@ -133,9 +143,11 @@ class MetadataProgressDialog(QProgressDialog):
    '''
    ProgressDialog displayed while fetching metadata for each story.
    '''
-    def __init__(self, gui, title, loop_list, fileform, getadapter_function, download_list_function, db):
-        QProgressDialog.__init__(self, title, QString(), 0, len(loop_list), gui)
-        self.setWindowTitle(title)
+    def __init__(self, gui, loop_list, fileform, getadapter_function, download_list_function, db):
+        QProgressDialog.__init__(self,
+                                 "Fetching metadata for stories...",
+                                 QString(), 0, len(loop_list), gui)
+        self.setWindowTitle("Downloading metadata for stories")
        self.setMinimumWidth(500)
        self.gui = gui
        self.db = db
@@ -144,32 +156,44 @@ class MetadataProgressDialog(QProgressDialog):
        self.getadapter_function = getadapter_function
        self.download_list_function = download_list_function
        self.i, self.loop_bad, self.loop_good = 0, [], []
+        
+        ## self.do_loop does QTimer.singleShot on self.do_loop also.
+        ## A weird way to do a loop, but that was the example I had.
        QTimer.singleShot(0, self.do_loop)
        self.exec_()

-    def bump(self):
-        self.i += 1
-        self.setValue(self.i)
-        self.setLabelText("Fetching metadata for %d of %d"%(0,len(self.loop_list)))
-        
+    def updateStatus(self):
+        self.setLabelText("Fetched metadata for %d of %d"%(self.i+1,len(self.loop_list)))
+        self.setValue(self.i+1)
+        print(self.labelText())

    def do_loop(self):
-        current = self.loop_list[self.i]
+        print("self.i:%d"%self.i)

-        self.bump()
-        try:
-            retval = self.getadapter_function(current,self.fileform)
-            self.loop_good.append((current,retval))
-        except Exception as e:
-            self.loop_bad.append((current,e))
+        if self.i == 0:
+            self.setValue(0)

-        if self.i >= len(self.loop_list):
+        if self.i >= len(self.loop_list) or self.wasCanceled():
            return self.do_when_finished()
+
        else:
+            current = self.loop_list[self.i]
+            try:
+                retval = self.getadapter_function(current,self.fileform)
+                self.loop_good.append((current,retval))
+            except Exception as e:
+                self.loop_bad.append((current,e))
+            
+            self.updateStatus()
+            self.i += 1
            QTimer.singleShot(0, self.do_loop)

    def do_when_finished(self):
        self.hide()
+        
+        # Queues a job to process these ePub/Mobi books in the background.
+        self.download_list_function(self.loop_good,self.fileform)
+        
        if self.loop_bad != []:
            res = []
            for j in self.loop_bad:
@@ -179,6 +203,8 @@ class MetadataProgressDialog(QProgressDialog):
                _('Could not get metadata for %d of %d stories.') %
                (len(self.loop_bad), len(self.loop_list)),
                msg).exec_()
+        else:
+            info_dialog(self.gui, "Starting Downloads",
+                        "Got metadata and started download for %d stories."%len(self.loop_good),
+                        show_copy_button=False).exec_()
        self.gui = None
-        # Queue a job to process these ePub/Mobi books
-        self.download_list_function(self.loop_good,self.fileform)
@@ -19,9 +19,7 @@ from calibre.gui2.threaded_jobs import ThreadedJob

 from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapters, writers, exceptions
 from calibre_plugins.fanfictiondownloader_plugin.config import prefs
-from calibre_plugins.fanfictiondownloader_plugin.plugin import DownloadDialog, MetadataProgressDialog, UserPassDialog
-
-#from calibre_plugins.fanfictiondownloader_plugin.jobs import do_story_downloads
+from calibre_plugins.fanfictiondownloader_plugin.dialogs import DownloadDialog, MetadataProgressDialog, UserPassDialog

 class FanFictionDownLoaderPlugin(InterfaceAction):

@@ -94,9 +92,7 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
        self.ffdlconfig.readfp(StringIO(get_resources("defaults.ini")))
        self.ffdlconfig.readfp(StringIO(prefs['personal.ini']))

-        ## XXX including code for Things to Come, namely, a list of
-        ## URLs rather than just one.
-        url_list = urls.splitlines()
+        url_list = get_url_list(urls)
        '''
        http://test1.com?sid=6700
        http://test1.com?sid=6701
@@ -109,7 +105,6 @@ class FanFictionDownLoaderPlugin(InterfaceAction):

        self.fetchmeta_qpd = \
            MetadataProgressDialog(self.gui,
-                                "Getting Metadata for Stories",
                                url_list,
                                fileform,
                                self.get_adapter_for_story,
@@ -158,7 +153,8 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
        if identicalbooks:
            add=False
            if question_dialog(self.gui, 'Add Duplicate?', '<p>'+
-                               "That story is already in your library.  Create a new one?",
+                               "%s by %s is already in your library.  Create a new one?"%
+                               (story.getMetadata("title"),story.getMetadata("author")),
                               show_copy_button=False):
                add=True

@@ -169,34 +165,20 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
        
    def download_list(self,adaptertuple_list,fileform):
        '''
-        Called by MetadataProgressDialog to enqueue story downloads for BG processing.
+        Called by MetadataProgressDialog to start story downloads BG processing.
        adapter_list is a list of tuples of (url,adapter)
        '''
-        print("enqueue_story_list_for_download")
-        print(adaptertuple_list)
+        print("download_list")
        
-        func = 'arbitrary_n'
-        args = ['calibre_plugins.fanfictiondownloader_plugin.jobs', 'do_story_downloads',
-                (adaptertuple_list, fileform)] # adaptertuple_list
-        desc = 'Download FanFiction Stories'
-
-        print("pre self.gui.job_manager.run_job")
-        # job = self.gui.job_manager.run_job(
-        #         self.Dispatcher(self._get_stories_completed), func, args=args,
-        #             description=desc)
-
        job = ThreadedJob('FanFictionDownload',
                          'Downloading FanFiction Stories',
-                          func=do_story_downloads,
+                          func=self.do_story_downloads,
                          args=(adaptertuple_list, fileform, self.db),
                          kwargs={},
                          callback=self._get_stories_completed)
        
        self.gui.job_manager.run_threaded_job(job)
-        print("post self.gui.job_manager.run_job")
-        # job.tdir = tdir
-        # job.pages_custom_column = pages_custom_column
-        # job.words_custom_column = words_custom_column
+        
        self.gui.status_bar.show_message('Downloading %d stories'%len(adaptertuple_list))

    def _get_stories_completed(self, job):
@@ -225,57 +207,61 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
        # p.show()
        #info_dialog(self.gui,'FFDL Complete','It worked?')        

-def do_story_downloads(adaptertuple_list, fileform, db,
-                       abort=None, log=None, notifications=[]): # lambda x,y:x lambda makes small anonymous function.
-    '''
-    Master job, to launch child jobs to download this list of stories
-    '''
-    print("do_story_downloads")
-    notifications.put((0.01, 'Start Downloading Stories'))
-    count = 0
-    total = len(adaptertuple_list)
-    # Queue all the jobs
-    for (url,adapter) in adaptertuple_list:
-        do_story_download(adapter,fileform,db)
-        count = count + 1
-        notifications.put((float(count)/total, 'Downloading Stories'))
-    # return the map as the job result
-    # return book_pages_map, book_words_map
-    return {},{}
-
-def do_story_download(adapter,fileform,db):
-    print("do_story_download")
-
-#    ffdlconfig = ConfigParser.SafeConfigParser()
-#    adapter = adapters.getAdapter(ffdlconfig,url)
-
-    story = adapter.getStoryMetadataOnly()
-
-    mi = MetaInformation(story.getMetadata("title"),
-                         (story.getMetadata("author"),)) # author is a list.
+    def do_story_downloads(self, adaptertuple_list, fileform, db,
+                           **kwargs): # lambda x,y:x lambda makes small anonymous function.
+        # abort=None, log=None, 
+        '''
+        Master job, loop to download this list of stories
+        '''
+        print("do_story_downloads")
+        notifications=kwargs['notifications']
+        notifications.put((0.01, 'Start Downloading Stories'))
+        count = 0
+        total = len(adaptertuple_list)
+        # Queue all the jobs
+        for (url,adapter) in adaptertuple_list:
+            self.do_story_download(adapter,fileform,db)
+            count = count + 1
+            notifications.put((float(count)/total, 'Downloading Stories'))
+        # return the map as the job result
+        # return book_pages_map, book_words_map
+        return {},{}
    
-    writer = writers.getWriter(fileform,adapter.config,adapter)
-    tmp = PersistentTemporaryFile("."+fileform)
-    print("tmp: "+tmp.name)
+    def do_story_download(self,adapter,fileform,db):
+        print("do_story_download")
    
-    writer.writeStory(tmp)
+        story = adapter.getStoryMetadataOnly()
    
-    print("post write tmp: "+tmp.name)
-    
-    mi.set_identifiers({'url':story.getMetadata("storyUrl")})
-    mi.publisher = story.getMetadata("site")
-
-    mi.tags = writer.getTags()
-    mi.languages = ['en']
-    mi.pubdate = story.getMetadataRaw('datePublished').strftime("%Y-%m-%d")
-    mi.timestamp = story.getMetadataRaw('dateCreated').strftime("%Y-%m-%d")
-    mi.comments = story.getMetadata("description")
-
-    (notadded,addedcount)=db.add_books([tmp],[fileform],[mi], add_duplicates=True)
-    # Otherwise list of books doesn't update right away.
-    #self.gui.library_view.model().books_added(addedcount)
-
-    del adapter
-    del writer
-
+        mi = MetaInformation(story.getMetadata("title"),
+                             (story.getMetadata("author"),)) # author is a list.
        
+        writer = writers.getWriter(fileform,adapter.config,adapter)
+        tmp = PersistentTemporaryFile("."+fileform)
+        print("tmp: "+tmp.name)
+        
+        writer.writeStory(tmp)
+        
+        print("post write tmp: "+tmp.name)
+        
+        mi.set_identifiers({'url':story.getMetadata("storyUrl")})
+        mi.publisher = story.getMetadata("site")
+    
+        mi.tags = writer.getTags()
+        mi.languages = ['en']
+        mi.pubdate = story.getMetadataRaw('datePublished').strftime("%Y-%m-%d")
+        mi.timestamp = story.getMetadataRaw('dateCreated').strftime("%Y-%m-%d")
+        mi.comments = story.getMetadata("description")
+    
+        (notadded,addedcount)=db.add_books([tmp],[fileform],[mi], add_duplicates=True)
+        # Otherwise list of books doesn't update right away.
+        self.gui.library_view.model().books_added(addedcount)
+    
+        del adapter
+        del writer
+
+def f(x):
+    if x.strip(): return True
+    else: return False
+    
+def get_url_list(urls):
+    return filter(f,urls.strip().splitlines())
@@ -55,7 +55,10 @@ class TestSiteAdapter(BaseSiteAdapter):
            raise exceptions.StoryDoesNotExist(self.url)

        if self.story.getMetadata('storyId').startswith('670'):
-            time.sleep(2.0)
+            time.sleep(1.0)
+            
+        if self.story.getMetadata('storyId').startswith('671'):
+            time.sleep(1.0)
            
        if self.getConfig("username"):
            self.username = self.getConfig("username")
@@ -131,8 +134,9 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
        if self.story.getMetadata('storyId') == '667':
            raise exceptions.FailedToDownload("Error downloading Chapter: %s!" % url)

-        if self.story.getMetadata('storyId').startswith('670'):
-            time.sleep(2.0)
+        if self.story.getMetadata('storyId').startswith('670') or \
+                self.story.getMetadata('storyId').startswith('672'):
+            time.sleep(1.0)

        if "chapter=1" in url :
            text=u'''
@@ -146,6 +150,8 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
 <p>http://test1.com?sid=668 - raises FailedToLogin unless username='Me'</p>
 <p>http://test1.com?sid=669 - Succeeds with Updated Date=now</p>
 <p>http://test1.com?sid=670 - Succeeds, but sleeps 2sec on each chapter</p>
+<p>http://test1.com?sid=671 - Succeeds, but sleeps 2sec metadata only</p>
+<p>http://test1.com?sid=672 - Succeeds, quick meta, sleeps 2sec chapters only</p>
 <p>And other storyId will succeed with the same output.</p>
 </div>
 '''