wassname/FanFicFare
1
0
Fork 0
mirror of https://github.com/wassname/FanFicFare.git synced 2026-09-11 11:52:07 +08:00
Code Issues Packages Projects Releases Wiki Activity

Add Update Log feature (epub only).

Browse Source
This commit is contained in:
Jim Miller
2012-08-06 21:22:18 -05:00
parent 89b86545e0
commit eab986a07a
13 changed files with 182 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app.yaml
+1 -1
View File
@@ -1,6 +1,6 @@
# ffd-retief-hrd fanfictiondownloader
application: fanfictiondownloader
version: 4-4-21
version: 4-4-22
runtime: python27
api_version: 1
threadsafe: true
calibre-plugin/__init__.py
+1 -1
View File
@@ -27,7 +27,7 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
description = 'UI plugin to download FanFiction stories from various sites.'
supported_platforms = ['windows', 'osx', 'linux']
author = 'Jim Miller'
version = (1, 6, 2)
version = (1, 6, 3)
minimum_calibre_version = (0, 8, 57)
#: This field defines the GUI plugin class that contains all the code
calibre-plugin/jobs.py
+4 -2
View File
@@ -154,11 +154,13 @@ def do_download_for_worker(book,options):
# update now handled by pre-populating the old images and
# chapters in the adapter rather than merging epubs.
urlchaptercount = int(story.getMetadata('numChapters'))
(url,chaptercount,
(url,
chaptercount,
adapter.oldchapters,
adapter.oldimgs,
adapter.oldcover,
adapter.calibrebookmark) = get_update_data(book['epub_for_update'])
adapter.calibrebookmark,
adapter.logfile) = get_update_data(book['epub_for_update'])
print("Do update - epub(%d) vs url(%d)" % (chaptercount, urlchaptercount))
print("write to %s"%outfile)
defaults.ini
+16 -1
View File
@@ -197,9 +197,24 @@ windows_eol: true
zip_output: false
## epub carries the TOC in metadata.
## mobi generated from epub will have a TOC at the end.
## mobi generated from epub by calibre will have a TOC at the end.
include_tocpage: false
## include a Update Log page before the story text. If included, the
## log will be updated each time the epub is an all the metadata
## fields that have changed since the last update (typically
## dateUpdated,numChapters,numWords at a minimum) will be shown.
## Great for tracking when chapters came out and when the description,
## etc changed.
include_logpage: false
## items to include in the log page Empty metadata entries, or those
## that haven't changed since the last update, will *not* appear, even
## if in the list. You can include extra text or HTML that will be
## included as-is in each log entry. Eg: logpage_entries: ...,<br />,
## summary,<br />,...
logpage_entries: dateCreated,datePublished,dateUpdated,numChapters,numWords,status,title,author,description,category,genre,rating,warnings
## epub->mobi conversions typically don't like tables.
titlepage_use_table: false
downloader.py
+2 -1
View File
@@ -198,7 +198,8 @@ def main():
adapter.oldchapters,
adapter.oldimgs,
adapter.oldcover,
adapter.calibrebookmark) = get_update_data(args[0])
adapter.calibrebookmark,
adapter.logfile) = get_update_data(args[0])
writeStory(config,adapter,"epub")
fanficdownloader/adapters/__init__.py
+1 -1
View File
@@ -127,7 +127,7 @@ def getAdapter(config,url,fileform=None):
if( domain != parsedUrl.netloc ):
fixedurl = fixedurl.replace(parsedUrl.netloc,domain)
logging.debug("site:"+domain)
logging.debug("trying url:"+url)
cls = getClassFor(domain)
if not cls and domain.startswith("www."):
domain = domain.replace("www.","")
fanficdownloader/adapters/adapter_test1.py
+19 -19
View File
@@ -140,25 +140,25 @@ Some more longer description. "I suck at summaries!" "Better than it sounds!"
('Chapter 3, Over Cinnabar',self.url+"&chapter=4"),
('Chapter 4',self.url+"&chapter=5"),
('Chapter 5',self.url+"&chapter=6"),
('Chapter 6',self.url+"&chapter=6"),
# ('Chapter 7',self.url+"&chapter=6"),
# ('Chapter 8',self.url+"&chapter=6"),
# ('Chapter 9',self.url+"&chapter=6"),
# ('Chapter 0',self.url+"&chapter=6"),
# ('Chapter a',self.url+"&chapter=6"),
# ('Chapter b',self.url+"&chapter=6"),
# ('Chapter c',self.url+"&chapter=6"),
# ('Chapter d',self.url+"&chapter=6"),
# ('Chapter e',self.url+"&chapter=6"),
# ('Chapter f',self.url+"&chapter=6"),
# ('Chapter g',self.url+"&chapter=6"),
# ('Chapter h',self.url+"&chapter=6"),
# ('Chapter i',self.url+"&chapter=6"),
# ('Chapter j',self.url+"&chapter=6"),
# ('Chapter k',self.url+"&chapter=6"),
# ('Chapter l',self.url+"&chapter=6"),
# ('Chapter m',self.url+"&chapter=6"),
# ('Chapter n',self.url+"&chapter=6"),
#('Chapter 6',self.url+"&chapter=7"),
#('Chapter 7',self.url+"&chapter=8"),
#('Chapter 8',self.url+"&chapter=9"),
#('Chapter 9',self.url+"&chapter=0"),
#('Chapter 0',self.url+"&chapter=a"),
#('Chapter a',self.url+"&chapter=b"),
#('Chapter b',self.url+"&chapter=c"),
#('Chapter c',self.url+"&chapter=d"),
#('Chapter d',self.url+"&chapter=e"),
#('Chapter e',self.url+"&chapter=f"),
#('Chapter f',self.url+"&chapter=g"),
#('Chapter g',self.url+"&chapter=h"),
#('Chapter h',self.url+"&chapter=i"),
#('Chapter i',self.url+"&chapter=j"),
#('Chapter j',self.url+"&chapter=k"),
#('Chapter k',self.url+"&chapter=l"),
#('Chapter l',self.url+"&chapter=m"),
#('Chapter m',self.url+"&chapter=n"),
#('Chapter n',self.url+"&chapter=o"),
]
self.story.setMetadata('numChapters',len(self.chapterUrls))
fanficdownloader/adapters/base_adapter.py
+3
View File
@@ -89,6 +89,7 @@ class BaseSiteAdapter(Configurable):
self.oldimgs = None
self.oldcover = None # (data of existing cover html, data of existing cover image)
self.calibrebookmark = None
self.logfile = None
## order of preference for decoding.
self.decode = ["utf8",
"Windows-1252"] # 1252 is a superset of
@@ -229,6 +230,8 @@ class BaseSiteAdapter(Configurable):
# cheesy way to carry calibre bookmark file forward across update.
if self.calibrebookmark:
self.story.calibrebookmark = self.calibrebookmark
if self.logfile:
self.story.logfile = self.logfile
return self.story
fanficdownloader/epubutils.py
+7 -1
View File
@@ -42,6 +42,7 @@ def get_update_data(inputio,
oldcover = None
calibrebookmark = None
logfile = None
# Looking for pre-existing cover.
for item in contentdom.getElementsByTagName("reference"):
if item.getAttribute("type") == "cover":
@@ -96,6 +97,11 @@ def get_update_data(inputio,
if( item.getAttribute("media-type") == "application/xhtml+xml" ):
href=relpath+item.getAttribute("href")
#print("---- item href:%s path part: %s"%(href,get_path_part(href)))
if re.match(r'.*/log_page\.x?html',href):
try:
logfile = epub.read(href).decode("utf-8")
except:
pass # corner case I bumped into while testing.
if re.match(r'.*/(file|chapter)\d+\.x?html',href):
if getsoups:
soup = bs.BeautifulSoup(epub.read(href).decode("utf-8"))
@@ -136,7 +142,7 @@ def get_update_data(inputio,
for k in images.keys():
print("\tlongdesc:%s\n\tData len:%s\n"%(k,len(images[k])))
return (source,filecount,soups,images,oldcover,calibrebookmark)
return (source,filecount,soups,images,oldcover,calibrebookmark,logfile)
def get_path_part(n):
relpath = os.path.dirname(n)
fanficdownloader/story.py
+4 -1
View File
@@ -201,6 +201,7 @@ class Story:
self.cover=None # *href* of new cover image--need to create html.
self.oldcover=None # (oldcoverhtmlhref,oldcoverhtmltype,oldcoverhtmldata,oldcoverimghref,oldcoverimgtype,oldcoverimgdata)
self.calibrebookmark=None # cheesy way to carry calibre bookmark file forward across update.
self.logfile=None # cheesy way to carry log file forward across update.
def setMetadata(self, key, value):
## still keeps &lt; &lt; and &amp;
@@ -230,6 +231,8 @@ class Story:
if value:
if key == "numWords":
value = commaGroups(value)
if key == "numChapters":
value = commaGroups("%d"%value)
if key == "dateCreated":
value = value.strftime("%Y-%m-%d %H:%M:%S")
if key == "datePublished" or key == "dateUpdated":
@@ -279,7 +282,7 @@ class Story:
# just for less clutter in adapters.
def extendList(self,listname,l):
for v in l:
self.addToList(listname,v)
self.addToList(listname,v.strip())
def addToList(self,listname,value):
if value==None:
fanficdownloader/writers/writer_epub.py
+107
View File
@@ -140,6 +140,101 @@ ${value}<br />
</html>
''')
self.EPUB_LOG_PAGE_START = string.Template('''<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<title>Update Log</title>
<link href="stylesheet.css" type="text/css" charset="UTF-8" rel="stylesheet"/>
</head>
<body>
<h3>Update Log</h3>
''')
self.EPUB_LOG_ENTRY = string.Template('''
<b>${label}:</b> <span id="${id}">${value}</span>
''')
self.EPUB_LOG_PAGE_END = string.Template('''
</body>
</html>
''')
def writeLogPage(self, out):
"""
XXX
Write the log page, but only include entries that there's
metadata for. START, ENTRY and END are expected to already by
string.Template(). START and END are expected to use the same
names as Story.metadata, but ENTRY should use id, label and value.
"""
if self.getConfig("include_logpage"):
# if there's a self.story.logfile, there's an existing log
# to add to.
if self.story.logfile:
print("existing logfile found, appending")
print("existing data:%s"%self._getLastLogData(self.story.logfile))
replace_string = "</body>" # "</h3>"
self._write(out,self.story.logfile.replace(replace_string,self._makeLogEntry(self._getLastLogData(self.story.logfile))+replace_string))
else:
# otherwise, write a new one.
self._write(out,self.EPUB_LOG_PAGE_START.substitute(self.story.getAllMetadata()))
self._write(out,self._makeLogEntry())
self._write(out,self.EPUB_LOG_PAGE_END.substitute(self.story.getAllMetadata()))
# self parsing instead of Soup because it should be simple and not
# worth the overhead.
def _getLastLogData(self,logfile):
"""
Make a dict() of the most recent(last) log entry for each piece of metadata.
Switch rindex to index to search from top instead of bottom.
"""
values = {}
for entry in self.getConfigList("logpage_entries"):
if entry in self.validEntries:
try:
# <span id="dateUpdated">1975-04-15</span>
span = '<span id="%s">'%entry
idx = logfile.rindex(span)+len(span)
values[entry] = logfile[idx:logfile.index('</span>',idx)]
except Exception, e:
#print("e:%s"%e)
pass
return values
def _makeLogEntry(self, oldvalues={}):
retval = "<p class='log_entry'>"
for entry in self.getConfigList("logpage_entries"):
if entry in self.validEntries:
val = self.story.getMetadata(entry)
if val and ( entry not in oldvalues or val != oldvalues[entry] ):
if self.hasConfig(entry+"_label"):
label=self.getConfig(entry+"_label")
else:
print("Using fallback label for %s_label"%entry)
label=self.titleLabels[entry]
retval = retval + self.EPUB_LOG_ENTRY.substitute({'id':entry,
'label':label,
'value':val})
else:
# could be useful for introducing extra text, but
# mostly it makes it easy to tell when you get the
# keyword wrong.
retval = retval + entry
retval = retval + "</p><hr />"
if self.getConfig('replace_hr'):
retval = retval.replace("<hr />","<div class='center'>* * *</div>")
return retval
def writeStoryImpl(self, out):
## Python 2.5 ZipFile is rather more primative than later
@@ -349,6 +444,11 @@ div { margin: 0pt; padding: 0pt; }
if len(self.story.getChapters()) > 1 and self.getConfig("include_tocpage") and not self.metaonly :
items.append(("toc_page","OEBPS/toc_page.xhtml","application/xhtml+xml","Table of Contents"))
itemrefs.append("toc_page")
if self.getConfig("include_logpage"):
items.append(("log_page","OEBPS/log_page.xhtml","application/xhtml+xml","Update Log"))
itemrefs.append("log_page")
for index, (title,html) in enumerate(self.story.getChapters()):
if html:
i=index+1
@@ -480,6 +580,13 @@ div { margin: 0pt; padding: 0pt; }
outputepub.writestr("OEBPS/toc_page.xhtml",tocpageIO.getvalue())
tocpageIO.close()
# write log page.
logpageIO = StringIO.StringIO()
self.writeLogPage(logpageIO)
if logpageIO.getvalue(): # will be false if no log page.
outputepub.writestr("OEBPS/log_page.xhtml",logpageIO.getvalue())
logpageIO.close()
for index, (title,html) in enumerate(self.story.getChapters()):
if html:
logging.debug('Writing chapter text for: %s' % title)
index.html
+1 -8
View File
@@ -54,13 +54,6 @@
much easier. </p>
</div>
<!-- put announcements here, h3 is a good title size. -->
<h3>New Sites</h3>
<p>
New sites grangerenchanted.com, hlfiction.net and nha.magical-worlds.us.
<br />Thanks, Ida!
</p>
<p>
Questions? Check out our
<a href="http://code.google.com/p/fanficdownloader/wiki/FanFictionDownloaderFAQs">FAQs</a>.
@@ -69,7 +62,7 @@
If you have any problems with this application, please
report them in
the <a href="http://groups.google.com/group/fanfic-downloader">FanFictionDownLoader Google Group</a>. The
<a href="http://4-4-20.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
<a href="http://4-4-21.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
</p>
<div id='error'>
{{ error_message }}
plugin-defaults.ini
+16 -1
View File
@@ -183,9 +183,24 @@ windows_eol: true
[epub]
## epub carries the TOC in metadata.
## mobi generated from epub will have a TOC at the end.
## mobi generated from epub by calibre will have a TOC at the end.
include_tocpage: false
## include a Update Log page before the story text. If included, the
## log will be updated each time the epub is an all the metadata
## fields that have changed since the last update (typically
## dateUpdated,numChapters,numWords at a minimum) will be shown.
## Great for tracking when chapters came out and when the description,
## etc changed.
include_logpage: false
## items to include in the log page Empty metadata entries, or those
## that haven't changed since the last update, will *not* appear, even
## if in the list. You can include extra text or HTML that will be
## included as-is in each log entry. Eg: logpage_entries: ...,<br />,
## summary,<br />,...
logpage_entries: dateCreated,datePublished,dateUpdated,numChapters,numWords,status,title,author,description,category,genre,rating,warnings
## epub->mobi conversions typically don't like tables.
titlepage_use_table: false