Plugin only, optionally search epub text for story URL, option to GC only new.

calibre-plugin/__init__.py

@@ -27,7 +27,7 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
     description         = 'UI plugin to download FanFiction stories from various sites.'
     supported_platforms = ['windows', 'osx', 'linux']
     author              = 'Jim Miller'
-    version             = (1, 5, 20)
+    version             = (1, 5, 21)
     minimum_calibre_version = (0, 8, 30)
 
     #: This field defines the GUI plugin class that contains all the code

calibre-plugin/config.py

@@ -47,6 +47,7 @@ all_prefs.defaults['collision'] = OVERWRITE
 all_prefs.defaults['deleteotherforms'] = False
 all_prefs.defaults['adddialogstaysontop'] = False
 all_prefs.defaults['includeimages'] = False
+all_prefs.defaults['lookforurlinhtml'] = False
 
 all_prefs.defaults['send_lists'] = ''
 all_prefs.defaults['read_lists'] = ''
@@ -54,6 +55,7 @@ all_prefs.defaults['addtolists'] = False
 all_prefs.defaults['addtoreadlists'] = False
 all_prefs.defaults['addtolistsonread'] = False
 
+all_prefs.defaults['gcnewonly'] = False
 all_prefs.defaults['gc_site_settings'] = {}
 all_prefs.defaults['allow_gc_from_ini'] = True
 
@@ -72,6 +74,8 @@ copylist = ['personal.ini',
             'deleteotherforms',
             'adddialogstaysontop',
             'includeimages',
+            'lookforurlinhtml',
+            'gcnewonly',
             'gc_site_settings',
             'allow_gc_from_ini']
 
@@ -176,6 +180,7 @@ class ConfigWidget(QWidget):
         prefs['deleteotherforms'] = self.basic_tab.deleteotherforms.isChecked()
         prefs['adddialogstaysontop'] = self.basic_tab.adddialogstaysontop.isChecked()
         prefs['includeimages'] = self.basic_tab.includeimages.isChecked()
+        prefs['lookforurlinhtml'] = self.basic_tab.lookforurlinhtml.isChecked()
 
         if self.readinglist_tab:
             # lists
@@ -196,6 +201,7 @@ class ConfigWidget(QWidget):
             prefs['personal.ini'] = get_resources('plugin-example.ini')
 
         # Generate Covers tab
+        prefs['gcnewonly'] = self.generatecover_tab.gcnewonly.isChecked()
         gc_site_settings = {}
         for (site,combo) in self.generatecover_tab.gc_dropdowns.iteritems():
             val = unicode(combo.itemData(combo.currentIndex()).toString())
@@ -309,6 +315,11 @@ class BasicTab(QWidget):
         self.includeimages.setChecked(prefs['includeimages'])
         self.l.addWidget(self.includeimages)
 
+        self.lookforurlinhtml = QCheckBox("Search EPUB text for Story URL?",self)
+        self.lookforurlinhtml.setToolTip("Look for first valid story URL inside EPUB text if not found in metadata.\nSomewhat risky, could find wrong URL depending on EPUB content.\nAlso finds and corrects bad ffnet URLs from ficsaver.com files.")
+        self.lookforurlinhtml.setChecked(prefs['lookforurlinhtml'])
+        self.l.addWidget(self.lookforurlinhtml)
+
         self.l.insertStretch(-1)
         
     def set_collisions(self):
@@ -511,6 +522,11 @@ class GenerateCoverTab(QWidget):
             horz.addWidget(dropdown)
             self.sl.addLayout(horz)
         
+        self.gcnewonly = QCheckBox("Run Generate Cover Only on New Books",self)
+        self.gcnewonly.setToolTip("Default is to run GC any time the calibre metadata is updated.")
+        self.gcnewonly.setChecked(prefs['gcnewonly'])
+        self.l.addWidget(self.gcnewonly)
+
         self.allow_gc_from_ini = QCheckBox('Allow generate_cover_settings from personal.ini to override.',self)
         self.allow_gc_from_ini.setToolTip("The INI parameter generate_cover_settings allows you to choose a GC setting based on metadata rather than site,\nbut it's much more complex.  generate_cover_settings is ignored when this is off.")
         self.allow_gc_from_ini.setChecked(prefs['allow_gc_from_ini'])

calibre-plugin/ffdl_plugin.py

@@ -36,7 +36,7 @@ from calibre_plugins.fanfictiondownloader_plugin.common_utils import (set_plugin
 
 from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapters, writers, exceptions
 from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.htmlcleanup import stripHTML
-from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.epubutils import get_dcsource, get_dcsource_chaptercount
+from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.epubutils import get_dcsource, get_dcsource_chaptercount, get_story_url_from_html
 
 from calibre_plugins.fanfictiondownloader_plugin.config import (prefs, permitted_values)
 from calibre_plugins.fanfictiondownloader_plugin.dialogs import (
@@ -815,8 +815,8 @@ make_firstimage_cover:true
 
         db.commit()
         
-        if 'Generate Cover' in self.gui.iactions:
-
+        print("book['added']:%s"%book['added'])
+        if 'Generate Cover' in self.gui.iactions and (book['added'] or not prefs['gcnewonly']):
             gc_plugin = self.gui.iactions['Generate Cover']
             setting_name = None
             if prefs['allow_gc_from_ini']:
@@ -1018,8 +1018,12 @@ make_firstimage_cover:true
                 if 'url' in identifiers:
                     #print("url from epub:"+identifiers['url'].replace('|',':'))
                     return identifiers['url'].replace('|',':')
-                # look for dc:source
-                return get_dcsource(existingepub)
+                # look for dc:source first, then scan HTML if 
+                link = get_dcsource(existingepub)
+                if link:
+                    return link
+                elif prefs['lookforurlinhtml']:
+                    return get_story_url_from_html(existingepub,self._is_good_downloader_url)
         return None
 
     def _is_good_downloader_url(self,url):

fanficdownloader/epubutils.py

@@ -94,3 +94,40 @@ def get_path_part(n):
     if( len(relpath) > 0 ):
         relpath=relpath+"/"
     return relpath
+
+def get_story_url_from_html(inputio,_is_good_url=None):
+
+    #print("get_story_url_from_html called")
+    epub = ZipFile(inputio, 'r')
+
+    ## Find the .opf file.
+    container = epub.read("META-INF/container.xml")
+    containerdom = parseString(container)
+    rootfilenodelist = containerdom.getElementsByTagName("rootfile")
+    rootfilename = rootfilenodelist[0].getAttribute("full-path")
+
+    contentdom = parseString(epub.read(rootfilename))
+    #firstmetadom = contentdom.getElementsByTagName("metadata")[0]
+
+    ## Save the path to the .opf file--hrefs inside it are relative to it.
+    relpath = get_path_part(rootfilename)
+            
+    # spin through the manifest--only place there are item tags.
+    for item in contentdom.getElementsByTagName("item"):
+        # First, count the 'chapter' files.  FFDL uses file0000.xhtml,
+        # but can also update epubs downloaded from Twisting the
+        # Hellmouth, which uses chapter0.html.
+        #print("---- item:%s"%item)
+        if( item.getAttribute("media-type") == "application/xhtml+xml" ):
+            filehref=relpath+item.getAttribute("href")
+            soup = bs.BeautifulSoup(epub.read(filehref).decode("utf-8"))
+            for link in soup.findAll('a',href=re.compile(r'^http.*')):
+                ahref=link['href']
+                #print("href:(%s)"%ahref)
+                # hack for bad ficsaver ffnet URLs.
+                m = re.match(r"^http://www.fanfiction.net/s(?P<id>\d+)//$",ahref)
+                if m != None:
+                    ahref="http://www.fanfiction.net/s/%s/1/"%m.group('id')
+                if _is_good_url == None or _is_good_url(ahref):
+                    return ahref
+    return None