From 5072572f1c523dc38dc0c879b8ebd05d527f48a3 Mon Sep 17 00:00:00 2001 From: Jim Miller Date: Fri, 18 May 2012 12:11:16 -0500 Subject: [PATCH] Plugin only, optionally search epub text for story URL, option to GC only new. --- calibre-plugin/__init__.py | 2 +- calibre-plugin/config.py | 16 +++++++++++++++ calibre-plugin/ffdl_plugin.py | 14 ++++++++----- fanficdownloader/epubutils.py | 37 +++++++++++++++++++++++++++++++++++ 4 files changed, 63 insertions(+), 6 deletions(-) diff --git a/calibre-plugin/__init__.py b/calibre-plugin/__init__.py index 44a7dda..a9b76e3 100644 --- a/calibre-plugin/__init__.py +++ b/calibre-plugin/__init__.py @@ -27,7 +27,7 @@ class FanFictionDownLoaderBase(InterfaceActionBase): description = 'UI plugin to download FanFiction stories from various sites.' supported_platforms = ['windows', 'osx', 'linux'] author = 'Jim Miller' - version = (1, 5, 20) + version = (1, 5, 21) minimum_calibre_version = (0, 8, 30) #: This field defines the GUI plugin class that contains all the code diff --git a/calibre-plugin/config.py b/calibre-plugin/config.py index 336eb93..6f97fb3 100644 --- a/calibre-plugin/config.py +++ b/calibre-plugin/config.py @@ -47,6 +47,7 @@ all_prefs.defaults['collision'] = OVERWRITE all_prefs.defaults['deleteotherforms'] = False all_prefs.defaults['adddialogstaysontop'] = False all_prefs.defaults['includeimages'] = False +all_prefs.defaults['lookforurlinhtml'] = False all_prefs.defaults['send_lists'] = '' all_prefs.defaults['read_lists'] = '' @@ -54,6 +55,7 @@ all_prefs.defaults['addtolists'] = False all_prefs.defaults['addtoreadlists'] = False all_prefs.defaults['addtolistsonread'] = False +all_prefs.defaults['gcnewonly'] = False all_prefs.defaults['gc_site_settings'] = {} all_prefs.defaults['allow_gc_from_ini'] = True @@ -72,6 +74,8 @@ copylist = ['personal.ini', 'deleteotherforms', 'adddialogstaysontop', 'includeimages', + 'lookforurlinhtml', + 'gcnewonly', 'gc_site_settings', 'allow_gc_from_ini'] @@ -176,6 +180,7 @@ class ConfigWidget(QWidget): prefs['deleteotherforms'] = self.basic_tab.deleteotherforms.isChecked() prefs['adddialogstaysontop'] = self.basic_tab.adddialogstaysontop.isChecked() prefs['includeimages'] = self.basic_tab.includeimages.isChecked() + prefs['lookforurlinhtml'] = self.basic_tab.lookforurlinhtml.isChecked() if self.readinglist_tab: # lists @@ -196,6 +201,7 @@ class ConfigWidget(QWidget): prefs['personal.ini'] = get_resources('plugin-example.ini') # Generate Covers tab + prefs['gcnewonly'] = self.generatecover_tab.gcnewonly.isChecked() gc_site_settings = {} for (site,combo) in self.generatecover_tab.gc_dropdowns.iteritems(): val = unicode(combo.itemData(combo.currentIndex()).toString()) @@ -309,6 +315,11 @@ class BasicTab(QWidget): self.includeimages.setChecked(prefs['includeimages']) self.l.addWidget(self.includeimages) + self.lookforurlinhtml = QCheckBox("Search EPUB text for Story URL?",self) + self.lookforurlinhtml.setToolTip("Look for first valid story URL inside EPUB text if not found in metadata.\nSomewhat risky, could find wrong URL depending on EPUB content.\nAlso finds and corrects bad ffnet URLs from ficsaver.com files.") + self.lookforurlinhtml.setChecked(prefs['lookforurlinhtml']) + self.l.addWidget(self.lookforurlinhtml) + self.l.insertStretch(-1) def set_collisions(self): @@ -511,6 +522,11 @@ class GenerateCoverTab(QWidget): horz.addWidget(dropdown) self.sl.addLayout(horz) + self.gcnewonly = QCheckBox("Run Generate Cover Only on New Books",self) + self.gcnewonly.setToolTip("Default is to run GC any time the calibre metadata is updated.") + self.gcnewonly.setChecked(prefs['gcnewonly']) + self.l.addWidget(self.gcnewonly) + self.allow_gc_from_ini = QCheckBox('Allow generate_cover_settings from personal.ini to override.',self) self.allow_gc_from_ini.setToolTip("The INI parameter generate_cover_settings allows you to choose a GC setting based on metadata rather than site,\nbut it's much more complex. generate_cover_settings is ignored when this is off.") self.allow_gc_from_ini.setChecked(prefs['allow_gc_from_ini']) diff --git a/calibre-plugin/ffdl_plugin.py b/calibre-plugin/ffdl_plugin.py index a2d2e81..c142f0a 100644 --- a/calibre-plugin/ffdl_plugin.py +++ b/calibre-plugin/ffdl_plugin.py @@ -36,7 +36,7 @@ from calibre_plugins.fanfictiondownloader_plugin.common_utils import (set_plugin from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapters, writers, exceptions from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.htmlcleanup import stripHTML -from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.epubutils import get_dcsource, get_dcsource_chaptercount +from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.epubutils import get_dcsource, get_dcsource_chaptercount, get_story_url_from_html from calibre_plugins.fanfictiondownloader_plugin.config import (prefs, permitted_values) from calibre_plugins.fanfictiondownloader_plugin.dialogs import ( @@ -815,8 +815,8 @@ make_firstimage_cover:true db.commit() - if 'Generate Cover' in self.gui.iactions: - + print("book['added']:%s"%book['added']) + if 'Generate Cover' in self.gui.iactions and (book['added'] or not prefs['gcnewonly']): gc_plugin = self.gui.iactions['Generate Cover'] setting_name = None if prefs['allow_gc_from_ini']: @@ -1018,8 +1018,12 @@ make_firstimage_cover:true if 'url' in identifiers: #print("url from epub:"+identifiers['url'].replace('|',':')) return identifiers['url'].replace('|',':') - # look for dc:source - return get_dcsource(existingepub) + # look for dc:source first, then scan HTML if + link = get_dcsource(existingepub) + if link: + return link + elif prefs['lookforurlinhtml']: + return get_story_url_from_html(existingepub,self._is_good_downloader_url) return None def _is_good_downloader_url(self,url): diff --git a/fanficdownloader/epubutils.py b/fanficdownloader/epubutils.py index 8fcf30f..845f359 100644 --- a/fanficdownloader/epubutils.py +++ b/fanficdownloader/epubutils.py @@ -94,3 +94,40 @@ def get_path_part(n): if( len(relpath) > 0 ): relpath=relpath+"/" return relpath + +def get_story_url_from_html(inputio,_is_good_url=None): + + #print("get_story_url_from_html called") + epub = ZipFile(inputio, 'r') + + ## Find the .opf file. + container = epub.read("META-INF/container.xml") + containerdom = parseString(container) + rootfilenodelist = containerdom.getElementsByTagName("rootfile") + rootfilename = rootfilenodelist[0].getAttribute("full-path") + + contentdom = parseString(epub.read(rootfilename)) + #firstmetadom = contentdom.getElementsByTagName("metadata")[0] + + ## Save the path to the .opf file--hrefs inside it are relative to it. + relpath = get_path_part(rootfilename) + + # spin through the manifest--only place there are item tags. + for item in contentdom.getElementsByTagName("item"): + # First, count the 'chapter' files. FFDL uses file0000.xhtml, + # but can also update epubs downloaded from Twisting the + # Hellmouth, which uses chapter0.html. + #print("---- item:%s"%item) + if( item.getAttribute("media-type") == "application/xhtml+xml" ): + filehref=relpath+item.getAttribute("href") + soup = bs.BeautifulSoup(epub.read(filehref).decode("utf-8")) + for link in soup.findAll('a',href=re.compile(r'^http.*')): + ahref=link['href'] + #print("href:(%s)"%ahref) + # hack for bad ficsaver ffnet URLs. + m = re.match(r"^http://www.fanfiction.net/s(?P\d+)//$",ahref) + if m != None: + ahref="http://www.fanfiction.net/s/%s/1/"%m.group('id') + if _is_good_url == None or _is_good_url(ahref): + return ahref + return None