From eab986a07ab3cd559aa4996e9b3e4cc62105140a Mon Sep 17 00:00:00 2001 From: Jim Miller Date: Mon, 6 Aug 2012 21:22:18 -0500 Subject: [PATCH] Add Update Log feature (epub only). --- app.yaml | 2 +- calibre-plugin/__init__.py | 2 +- calibre-plugin/jobs.py | 6 +- defaults.ini | 17 +++- downloader.py | 3 +- fanficdownloader/adapters/__init__.py | 2 +- fanficdownloader/adapters/adapter_test1.py | 38 ++++---- fanficdownloader/adapters/base_adapter.py | 3 + fanficdownloader/epubutils.py | 8 +- fanficdownloader/story.py | 5 +- fanficdownloader/writers/writer_epub.py | 107 +++++++++++++++++++++ index.html | 9 +- plugin-defaults.ini | 17 +++- 13 files changed, 182 insertions(+), 37 deletions(-) diff --git a/app.yaml b/app.yaml index 5305bb9..f52edb4 100644 --- a/app.yaml +++ b/app.yaml @@ -1,6 +1,6 @@ # ffd-retief-hrd fanfictiondownloader application: fanfictiondownloader -version: 4-4-21 +version: 4-4-22 runtime: python27 api_version: 1 threadsafe: true diff --git a/calibre-plugin/__init__.py b/calibre-plugin/__init__.py index 21ce6fe..4721851 100644 --- a/calibre-plugin/__init__.py +++ b/calibre-plugin/__init__.py @@ -27,7 +27,7 @@ class FanFictionDownLoaderBase(InterfaceActionBase): description = 'UI plugin to download FanFiction stories from various sites.' supported_platforms = ['windows', 'osx', 'linux'] author = 'Jim Miller' - version = (1, 6, 2) + version = (1, 6, 3) minimum_calibre_version = (0, 8, 57) #: This field defines the GUI plugin class that contains all the code diff --git a/calibre-plugin/jobs.py b/calibre-plugin/jobs.py index 49ea863..6606a87 100644 --- a/calibre-plugin/jobs.py +++ b/calibre-plugin/jobs.py @@ -154,11 +154,13 @@ def do_download_for_worker(book,options): # update now handled by pre-populating the old images and # chapters in the adapter rather than merging epubs. urlchaptercount = int(story.getMetadata('numChapters')) - (url,chaptercount, + (url, + chaptercount, adapter.oldchapters, adapter.oldimgs, adapter.oldcover, - adapter.calibrebookmark) = get_update_data(book['epub_for_update']) + adapter.calibrebookmark, + adapter.logfile) = get_update_data(book['epub_for_update']) print("Do update - epub(%d) vs url(%d)" % (chaptercount, urlchaptercount)) print("write to %s"%outfile) diff --git a/defaults.ini b/defaults.ini index cc58a16..ff78d8f 100644 --- a/defaults.ini +++ b/defaults.ini @@ -197,9 +197,24 @@ windows_eol: true zip_output: false ## epub carries the TOC in metadata. -## mobi generated from epub will have a TOC at the end. +## mobi generated from epub by calibre will have a TOC at the end. include_tocpage: false +## include a Update Log page before the story text. If included, the +## log will be updated each time the epub is an all the metadata +## fields that have changed since the last update (typically +## dateUpdated,numChapters,numWords at a minimum) will be shown. +## Great for tracking when chapters came out and when the description, +## etc changed. +include_logpage: false + +## items to include in the log page Empty metadata entries, or those +## that haven't changed since the last update, will *not* appear, even +## if in the list. You can include extra text or HTML that will be +## included as-is in each log entry. Eg: logpage_entries: ...,
, +## summary,
,... +logpage_entries: dateCreated,datePublished,dateUpdated,numChapters,numWords,status,title,author,description,category,genre,rating,warnings + ## epub->mobi conversions typically don't like tables. titlepage_use_table: false diff --git a/downloader.py b/downloader.py index 58e607a..7b3e822 100644 --- a/downloader.py +++ b/downloader.py @@ -198,7 +198,8 @@ def main(): adapter.oldchapters, adapter.oldimgs, adapter.oldcover, - adapter.calibrebookmark) = get_update_data(args[0]) + adapter.calibrebookmark, + adapter.logfile) = get_update_data(args[0]) writeStory(config,adapter,"epub") diff --git a/fanficdownloader/adapters/__init__.py b/fanficdownloader/adapters/__init__.py index 92fb1df..aad3619 100644 --- a/fanficdownloader/adapters/__init__.py +++ b/fanficdownloader/adapters/__init__.py @@ -127,7 +127,7 @@ def getAdapter(config,url,fileform=None): if( domain != parsedUrl.netloc ): fixedurl = fixedurl.replace(parsedUrl.netloc,domain) - logging.debug("site:"+domain) + logging.debug("trying url:"+url) cls = getClassFor(domain) if not cls and domain.startswith("www."): domain = domain.replace("www.","") diff --git a/fanficdownloader/adapters/adapter_test1.py b/fanficdownloader/adapters/adapter_test1.py index 3067390..10d5420 100644 --- a/fanficdownloader/adapters/adapter_test1.py +++ b/fanficdownloader/adapters/adapter_test1.py @@ -140,25 +140,25 @@ Some more longer description. "I suck at summaries!" "Better than it sounds!" ('Chapter 3, Over Cinnabar',self.url+"&chapter=4"), ('Chapter 4',self.url+"&chapter=5"), ('Chapter 5',self.url+"&chapter=6"), - ('Chapter 6',self.url+"&chapter=6"), - # ('Chapter 7',self.url+"&chapter=6"), - # ('Chapter 8',self.url+"&chapter=6"), - # ('Chapter 9',self.url+"&chapter=6"), - # ('Chapter 0',self.url+"&chapter=6"), - # ('Chapter a',self.url+"&chapter=6"), - # ('Chapter b',self.url+"&chapter=6"), - # ('Chapter c',self.url+"&chapter=6"), - # ('Chapter d',self.url+"&chapter=6"), - # ('Chapter e',self.url+"&chapter=6"), - # ('Chapter f',self.url+"&chapter=6"), - # ('Chapter g',self.url+"&chapter=6"), - # ('Chapter h',self.url+"&chapter=6"), - # ('Chapter i',self.url+"&chapter=6"), - # ('Chapter j',self.url+"&chapter=6"), - # ('Chapter k',self.url+"&chapter=6"), - # ('Chapter l',self.url+"&chapter=6"), - # ('Chapter m',self.url+"&chapter=6"), - # ('Chapter n',self.url+"&chapter=6"), + #('Chapter 6',self.url+"&chapter=7"), + #('Chapter 7',self.url+"&chapter=8"), + #('Chapter 8',self.url+"&chapter=9"), + #('Chapter 9',self.url+"&chapter=0"), + #('Chapter 0',self.url+"&chapter=a"), + #('Chapter a',self.url+"&chapter=b"), + #('Chapter b',self.url+"&chapter=c"), + #('Chapter c',self.url+"&chapter=d"), + #('Chapter d',self.url+"&chapter=e"), + #('Chapter e',self.url+"&chapter=f"), + #('Chapter f',self.url+"&chapter=g"), + #('Chapter g',self.url+"&chapter=h"), + #('Chapter h',self.url+"&chapter=i"), + #('Chapter i',self.url+"&chapter=j"), + #('Chapter j',self.url+"&chapter=k"), + #('Chapter k',self.url+"&chapter=l"), + #('Chapter l',self.url+"&chapter=m"), + #('Chapter m',self.url+"&chapter=n"), + #('Chapter n',self.url+"&chapter=o"), ] self.story.setMetadata('numChapters',len(self.chapterUrls)) diff --git a/fanficdownloader/adapters/base_adapter.py b/fanficdownloader/adapters/base_adapter.py index 573e2ce..cbe938d 100644 --- a/fanficdownloader/adapters/base_adapter.py +++ b/fanficdownloader/adapters/base_adapter.py @@ -89,6 +89,7 @@ class BaseSiteAdapter(Configurable): self.oldimgs = None self.oldcover = None # (data of existing cover html, data of existing cover image) self.calibrebookmark = None + self.logfile = None ## order of preference for decoding. self.decode = ["utf8", "Windows-1252"] # 1252 is a superset of @@ -229,6 +230,8 @@ class BaseSiteAdapter(Configurable): # cheesy way to carry calibre bookmark file forward across update. if self.calibrebookmark: self.story.calibrebookmark = self.calibrebookmark + if self.logfile: + self.story.logfile = self.logfile return self.story diff --git a/fanficdownloader/epubutils.py b/fanficdownloader/epubutils.py index ebb74e9..b4c908c 100644 --- a/fanficdownloader/epubutils.py +++ b/fanficdownloader/epubutils.py @@ -42,6 +42,7 @@ def get_update_data(inputio, oldcover = None calibrebookmark = None + logfile = None # Looking for pre-existing cover. for item in contentdom.getElementsByTagName("reference"): if item.getAttribute("type") == "cover": @@ -96,6 +97,11 @@ def get_update_data(inputio, if( item.getAttribute("media-type") == "application/xhtml+xml" ): href=relpath+item.getAttribute("href") #print("---- item href:%s path part: %s"%(href,get_path_part(href))) + if re.match(r'.*/log_page\.x?html',href): + try: + logfile = epub.read(href).decode("utf-8") + except: + pass # corner case I bumped into while testing. if re.match(r'.*/(file|chapter)\d+\.x?html',href): if getsoups: soup = bs.BeautifulSoup(epub.read(href).decode("utf-8")) @@ -136,7 +142,7 @@ def get_update_data(inputio, for k in images.keys(): print("\tlongdesc:%s\n\tData len:%s\n"%(k,len(images[k]))) - return (source,filecount,soups,images,oldcover,calibrebookmark) + return (source,filecount,soups,images,oldcover,calibrebookmark,logfile) def get_path_part(n): relpath = os.path.dirname(n) diff --git a/fanficdownloader/story.py b/fanficdownloader/story.py index 3be4a95..666a017 100644 --- a/fanficdownloader/story.py +++ b/fanficdownloader/story.py @@ -201,6 +201,7 @@ class Story: self.cover=None # *href* of new cover image--need to create html. self.oldcover=None # (oldcoverhtmlhref,oldcoverhtmltype,oldcoverhtmldata,oldcoverimghref,oldcoverimgtype,oldcoverimgdata) self.calibrebookmark=None # cheesy way to carry calibre bookmark file forward across update. + self.logfile=None # cheesy way to carry log file forward across update. def setMetadata(self, key, value): ## still keeps < < and & @@ -230,6 +231,8 @@ class Story: if value: if key == "numWords": value = commaGroups(value) + if key == "numChapters": + value = commaGroups("%d"%value) if key == "dateCreated": value = value.strftime("%Y-%m-%d %H:%M:%S") if key == "datePublished" or key == "dateUpdated": @@ -279,7 +282,7 @@ class Story: # just for less clutter in adapters. def extendList(self,listname,l): for v in l: - self.addToList(listname,v) + self.addToList(listname,v.strip()) def addToList(self,listname,value): if value==None: diff --git a/fanficdownloader/writers/writer_epub.py b/fanficdownloader/writers/writer_epub.py index 39f33be..6db5ef5 100644 --- a/fanficdownloader/writers/writer_epub.py +++ b/fanficdownloader/writers/writer_epub.py @@ -140,6 +140,101 @@ ${value}
''') + self.EPUB_LOG_PAGE_START = string.Template(''' + + + +Update Log + + + +

Update Log

+''') + + self.EPUB_LOG_ENTRY = string.Template(''' +${label}: ${value} +''') + + self.EPUB_LOG_PAGE_END = string.Template(''' + + +''') + + def writeLogPage(self, out): + """ + XXX + + + Write the log page, but only include entries that there's + metadata for. START, ENTRY and END are expected to already by + string.Template(). START and END are expected to use the same + names as Story.metadata, but ENTRY should use id, label and value. + """ + if self.getConfig("include_logpage"): + + # if there's a self.story.logfile, there's an existing log + # to add to. + if self.story.logfile: + print("existing logfile found, appending") + print("existing data:%s"%self._getLastLogData(self.story.logfile)) + replace_string = "" # "" + self._write(out,self.story.logfile.replace(replace_string,self._makeLogEntry(self._getLastLogData(self.story.logfile))+replace_string)) + else: + # otherwise, write a new one. + self._write(out,self.EPUB_LOG_PAGE_START.substitute(self.story.getAllMetadata())) + self._write(out,self._makeLogEntry()) + self._write(out,self.EPUB_LOG_PAGE_END.substitute(self.story.getAllMetadata())) + + # self parsing instead of Soup because it should be simple and not + # worth the overhead. + def _getLastLogData(self,logfile): + """ + Make a dict() of the most recent(last) log entry for each piece of metadata. + Switch rindex to index to search from top instead of bottom. + """ + values = {} + for entry in self.getConfigList("logpage_entries"): + if entry in self.validEntries: + try: + # 1975-04-15 + span = ''%entry + idx = logfile.rindex(span)+len(span) + values[entry] = logfile[idx:logfile.index('',idx)] + except Exception, e: + #print("e:%s"%e) + pass + + return values + + def _makeLogEntry(self, oldvalues={}): + retval = "

" + + for entry in self.getConfigList("logpage_entries"): + if entry in self.validEntries: + val = self.story.getMetadata(entry) + if val and ( entry not in oldvalues or val != oldvalues[entry] ): + if self.hasConfig(entry+"_label"): + label=self.getConfig(entry+"_label") + else: + print("Using fallback label for %s_label"%entry) + label=self.titleLabels[entry] + + retval = retval + self.EPUB_LOG_ENTRY.substitute({'id':entry, + 'label':label, + 'value':val}) + else: + # could be useful for introducing extra text, but + # mostly it makes it easy to tell when you get the + # keyword wrong. + retval = retval + entry + + retval = retval + "

" + + if self.getConfig('replace_hr'): + retval = retval.replace("
","
* * *
") + + return retval + def writeStoryImpl(self, out): ## Python 2.5 ZipFile is rather more primative than later @@ -349,6 +444,11 @@ div { margin: 0pt; padding: 0pt; } if len(self.story.getChapters()) > 1 and self.getConfig("include_tocpage") and not self.metaonly : items.append(("toc_page","OEBPS/toc_page.xhtml","application/xhtml+xml","Table of Contents")) itemrefs.append("toc_page") + + if self.getConfig("include_logpage"): + items.append(("log_page","OEBPS/log_page.xhtml","application/xhtml+xml","Update Log")) + itemrefs.append("log_page") + for index, (title,html) in enumerate(self.story.getChapters()): if html: i=index+1 @@ -480,6 +580,13 @@ div { margin: 0pt; padding: 0pt; } outputepub.writestr("OEBPS/toc_page.xhtml",tocpageIO.getvalue()) tocpageIO.close() + # write log page. + logpageIO = StringIO.StringIO() + self.writeLogPage(logpageIO) + if logpageIO.getvalue(): # will be false if no log page. + outputepub.writestr("OEBPS/log_page.xhtml",logpageIO.getvalue()) + logpageIO.close() + for index, (title,html) in enumerate(self.story.getChapters()): if html: logging.debug('Writing chapter text for: %s' % title) diff --git a/index.html b/index.html index 7493425..4243e0a 100644 --- a/index.html +++ b/index.html @@ -54,13 +54,6 @@ much easier.

-

New Sites

-

- - New sites grangerenchanted.com, hlfiction.net and nha.magical-worlds.us. -
Thanks, Ida! - -

Questions? Check out our FAQs. @@ -69,7 +62,7 @@ If you have any problems with this application, please report them in the FanFictionDownLoader Google Group. The - Previous Version is also available for you to use if necessary. + Previous Version is also available for you to use if necessary.

{{ error_message }} diff --git a/plugin-defaults.ini b/plugin-defaults.ini index 83f4380..e2440fa 100644 --- a/plugin-defaults.ini +++ b/plugin-defaults.ini @@ -183,9 +183,24 @@ windows_eol: true [epub] ## epub carries the TOC in metadata. -## mobi generated from epub will have a TOC at the end. +## mobi generated from epub by calibre will have a TOC at the end. include_tocpage: false +## include a Update Log page before the story text. If included, the +## log will be updated each time the epub is an all the metadata +## fields that have changed since the last update (typically +## dateUpdated,numChapters,numWords at a minimum) will be shown. +## Great for tracking when chapters came out and when the description, +## etc changed. +include_logpage: false + +## items to include in the log page Empty metadata entries, or those +## that haven't changed since the last update, will *not* appear, even +## if in the list. You can include extra text or HTML that will be +## included as-is in each log entry. Eg: logpage_entries: ...,
, +## summary,
,... +logpage_entries: dateCreated,datePublished,dateUpdated,numChapters,numWords,status,title,author,description,category,genre,rating,warnings + ## epub->mobi conversions typically don't like tables. titlepage_use_table: false