mirror of
https://github.com/wassname/FanFicFare.git
synced 2026-09-11 11:52:07 +08:00
Improved (but not perfected) mobi file output. Reduce cron clean up time to 2 days.
This commit is contained in:
retiefjimm
@@ -29,17 +29,27 @@ MIMETYPE = '''application/epub+zip'''
|
||||
|
||||
TITLE_HEADER = '''<?xml version="1.0" encoding="utf-8"?><html xmlns="http://www.w3.org/1999/xhtml" xmlns:xlink="http://www.w3.org/1999/xlink"><head><meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>
|
||||
<title>%s - %s</title><link href="stylesheet.css" type="text/css" charset="UTF-8" rel="stylesheet"/></head><body>
|
||||
<p><h7 id="lnks"><b><a id="StoryLink" href="%s">%s</a></b> by <b><a id="AuthorLink" href="%s">%s</a></b></h7></p>
|
||||
<p><h3 id="lnks"><b><a id="StoryLink" href="%s">%s</a></b> by <b><a id="AuthorLink" href="%s">%s</a></b></h3></p>
|
||||
'''
|
||||
|
||||
TITLE_ENTRY = '''<b>%s</b> %s<br />
|
||||
'''
|
||||
|
||||
TITLE_FOOTER = '''
|
||||
<br /><b>Summary:</b><br />%s<br />
|
||||
</body></html>
|
||||
'''
|
||||
|
||||
TABLE_TITLE_HEADER = TITLE_HEADER + '''
|
||||
<table class="full">
|
||||
'''
|
||||
|
||||
TITLE_ENTRY = '''<tr><td><b>%s</b></td><td>%s</td></tr>
|
||||
TABLE_TITLE_ENTRY = '''<tr><td><b>%s</b></td><td>%s</td></tr>
|
||||
'''
|
||||
|
||||
TITLE_FOOTER = '''</table>
|
||||
<p><b>Summary:</b><br />%s</p>
|
||||
</body></html>
|
||||
'''
|
||||
TABLE_TITLE_FOOTER = '''
|
||||
</table>
|
||||
''' + TITLE_FOOTER
|
||||
|
||||
CONTAINER = '''<?xml version="1.0" encoding="utf-8"?>
|
||||
<container version="1.0" xmlns="urn:oasis:names:tc:opendocument:xmlns:container">
|
||||
|
||||
+24
-17
@@ -65,44 +65,50 @@ class Converter:
|
||||
return out.getvalue()
|
||||
|
||||
def ConvertFile(self, html_file, out_file):
|
||||
self._ConvertStringToFile(open(html_file).read(),
|
||||
open(out_file, 'w'))
|
||||
self._ConvertStringToFile(open(html_file,'rb').read(),
|
||||
open(out_file, 'wb'))
|
||||
|
||||
def ConvertFiles(self, html_files, out_file):
|
||||
html_strs = [open(f).read() for f in html_files]
|
||||
self._ConvertStringsToFile(html_strs, open(out_file, 'w'))
|
||||
html_strs = [open(f,'rb').read() for f in html_files]
|
||||
self._ConvertStringsToFile(html_strs, open(out_file, 'wb'))
|
||||
|
||||
def MakeOneHTML(self, html_strs):
|
||||
"""This takes a list of HTML strings and returns a big HTML file with
|
||||
all contents consolidated. It constructs a table of contents and adds
|
||||
anchors within the text
|
||||
"""
|
||||
title_html = []
|
||||
toc_html = []
|
||||
if self._refresh_url:
|
||||
toc_html.append('<a href="%s">Update Reading List</a><br>' %
|
||||
self._refresh_url)
|
||||
body_html = []
|
||||
titles = []
|
||||
|
||||
PAGE_BREAK = '<mdb;pagebreak>'
|
||||
for pos, html in enumerate(html_strs):
|
||||
PAGE_BREAK = '<mbp:pagebreak>'
|
||||
|
||||
# pull out the title page, assumed first html_strs.
|
||||
htmltitle = html_strs[0]
|
||||
entrytitle = _SubEntry(1, htmltitle)
|
||||
title_html.append(entrytitle.Body())
|
||||
|
||||
toc_html.append(PAGE_BREAK)
|
||||
toc_html.append('<h3>Table of Contents</h3><br />')
|
||||
|
||||
for pos, html in enumerate(html_strs[1:]):
|
||||
entry = _SubEntry(pos+1, html)
|
||||
titles.append(entry.title[:10])
|
||||
toc_html.append('%s<br>' % entry.TocLink())
|
||||
toc_html.append('%s<br />' % entry.TocLink())
|
||||
|
||||
# give some space between bodies of work.
|
||||
body_html.append(PAGE_BREAK)
|
||||
|
||||
body_html.append(entry.Anchor())
|
||||
|
||||
body_html.append('<h1>%s</h1>' % entry.title)
|
||||
body_html.append(entry.Body())
|
||||
|
||||
# TODO: this title can get way too long with RSS feeds. Not sure how to fix
|
||||
header = '<html><title>Bibliorize %s GMT</title><body>' % time.ctime(
|
||||
header = '<html><head><title>Bibliorize %s GMT</title></head><body>' % time.ctime(
|
||||
time.time())
|
||||
|
||||
footer = '</body></html>'
|
||||
all_html = header + '\n'.join(toc_html + body_html) + footer
|
||||
all_html = header + '\n'.join(title_html + toc_html + body_html) + footer
|
||||
#print "%s" % all_html.encode('utf8')
|
||||
return all_html
|
||||
|
||||
def _ConvertStringsToFile(self, html_strs, out_file):
|
||||
@@ -343,5 +349,6 @@ class Header:
|
||||
|
||||
if __name__ == '__main__':
|
||||
import sys
|
||||
m = Converter()
|
||||
m.ConvertFiles(sys.argv[1:], '/tmp/test.mobi')
|
||||
m = Converter(title='Testing Mobi', author='Mobi Author', publisher='mobi converter')
|
||||
m.ConvertFiles(sys.argv[1:], 'test.mobi')
|
||||
#m.ConvertFile(sys.argv[1], 'test.mobi')
|
||||
|
||||
+143
-27
@@ -83,7 +83,8 @@ class TextWriter(FanficWriter):
|
||||
|
||||
|
||||
class MobiWriter(FanficWriter):
|
||||
body = ''
|
||||
chapters = []
|
||||
files = {}
|
||||
|
||||
@staticmethod
|
||||
def getFormatName():
|
||||
@@ -104,6 +105,9 @@ class MobiWriter(FanficWriter):
|
||||
self.mobi = mobi
|
||||
self.inmemory = inmemory
|
||||
|
||||
self.files = {}
|
||||
self.chapters = []
|
||||
|
||||
if not self.inmemory and os.path.exists(self.fileName):
|
||||
os.remove(self.fileName)
|
||||
|
||||
@@ -122,27 +126,140 @@ class MobiWriter(FanficWriter):
|
||||
except:
|
||||
return text
|
||||
|
||||
def _writeFile(self, fileName, data):
|
||||
#logging.debug('_writeFile(`%s`, data)' % fileName)
|
||||
if fileName in self.files:
|
||||
try:
|
||||
d = data.decode('utf-8')
|
||||
except UnicodeEncodeError, e:
|
||||
d = data
|
||||
|
||||
self.files[fileName].write(d)
|
||||
else:
|
||||
self.files[fileName] = StringIO.StringIO()
|
||||
self._writeFile(fileName, data)
|
||||
|
||||
def _getFilesStrings(self):
|
||||
strings = []
|
||||
if "title_page.xhtml" in self.files:
|
||||
strings.append(self.files["title_page.xhtml"].getvalue())
|
||||
del(self.files["title_page.xhtml"])
|
||||
|
||||
keys = self.files.keys()
|
||||
keys.sort()
|
||||
|
||||
# Assumed all other files are chapter0000.xhtml.
|
||||
for fn in keys:
|
||||
strings.append(self.files[fn].getvalue())
|
||||
|
||||
return strings
|
||||
|
||||
def writeChapter(self, index, title, text):
|
||||
title = self._printableVersion(title) #title.decode('utf-8')
|
||||
text = self._printableVersion(text) #text.decode('utf-8')
|
||||
self.body = self.body + '\n' + self.chapterStartTemplate.substitute({'chapter' : title})
|
||||
self.body = self.body + '\n' + text
|
||||
title = removeEntities(title)
|
||||
logging.debug("Writing chapter: %s" % title)
|
||||
#title = self._printableVersion(title) #title.decode('utf-8')
|
||||
text = removeEntities(text)
|
||||
#text = self._printableVersion(text) #text.decode('utf-8')
|
||||
|
||||
# BeautifulStoneSoup doesn't have any selfClosingTags by default.
|
||||
# hr & br needs to be if they're going to work.
|
||||
# Some stories do use multiple br tags as their section breaks...
|
||||
self.soup = bs.BeautifulStoneSoup(text, selfClosingTags=('br','hr'))
|
||||
|
||||
allTags = self.soup.findAll(recursive=True)
|
||||
for t in allTags:
|
||||
for attr in t._getAttrMap().keys():
|
||||
if attr not in acceptable_attributes:
|
||||
del t[attr]
|
||||
# these are not acceptable strict XHTML. But we do already have
|
||||
# CSS classes of the same names defined in constants.py
|
||||
if t.name in ('u'):
|
||||
t['class']=t.name
|
||||
t.name='span'
|
||||
if t.name in ('center'):
|
||||
t['class']=t.name
|
||||
t.name='div'
|
||||
# removes paired, but empty tags.
|
||||
if t.string != None and len(t.string.strip()) == 0 :
|
||||
t.extract()
|
||||
|
||||
text = self.soup.__str__('utf8')
|
||||
|
||||
# ffnet(& maybe others) gives the whole chapter text
|
||||
# as one line. This causes problems for nook(at
|
||||
# least) when the chapter size starts getting big
|
||||
# (200k+) Using Soup's prettify() messes up italics
|
||||
# and such. Done after soup extract so <p> and <br>
|
||||
# tags are normalized. Doing it here seems less evil
|
||||
# than hacking BeautifulSoup, but it's debatable.
|
||||
text = text.replace('</p>','</p>\n').replace('<br />','<br />\n')
|
||||
|
||||
filename="chapter%04d.xhtml" % index
|
||||
self._writeFile(filename, XHTML_START % (title, title))
|
||||
self._writeFile(filename, text)
|
||||
self._writeFile(filename, XHTML_END)
|
||||
|
||||
#self.body = self.body + '\n' + self.chapterStartTemplate.substitute({'chapter' : title})
|
||||
#self.body = self.body + '\n' + text
|
||||
|
||||
def finalise(self):
|
||||
html = self.xhtmlTemplate.substitute({'title' : self.storyTitle, 'author' : self.authorName, 'body' : self.body})
|
||||
soup = bs.BeautifulSoup(html)
|
||||
result = soup.__str__('utf8')
|
||||
logging.debug("Finalising...")
|
||||
|
||||
# f = open(self.fileName, 'w')
|
||||
# f.write(result)
|
||||
# f.close()
|
||||
published = self.adapter.getStoryPublished().strftime("%Y-%m-%d")
|
||||
createda = self.adapter.getStoryCreated().strftime("%Y-%m-%d %H:%M:%S")
|
||||
created = self.adapter.getStoryCreated().strftime("%Y-%m-%d")
|
||||
updated = self.adapter.getStoryUpdated().strftime("%Y-%m-%d")
|
||||
updateyy = self.adapter.getStoryUpdated().strftime("%Y")
|
||||
updatemm = self.adapter.getStoryUpdated().strftime("%m")
|
||||
updatedd = self.adapter.getStoryUpdated().strftime("%d")
|
||||
calibre = self.adapter.getStoryUpdated().strftime("%Y-%m-%dT%H:%M:%S")
|
||||
|
||||
description = self.adapter.getStoryDescription()
|
||||
if hasattr(description, "text"):
|
||||
description = description.text
|
||||
prevalue=description
|
||||
try:
|
||||
description = unicode(description)
|
||||
except:
|
||||
description=prevalue
|
||||
|
||||
if description is not None and len(description) > 0:
|
||||
description = description.replace ('\\\'', '\'').replace('\\\"', '\"')
|
||||
description = removeEntities(description)
|
||||
else:
|
||||
description = ' '
|
||||
|
||||
c = mobi.Converter(title=self.storyTitle, author=self.authorName, publisher=self.publisher)
|
||||
mobidata = c.ConvertString(result)
|
||||
### writing content -- title page
|
||||
titleFilePath = "title_page.xhtml"
|
||||
self._writeFile(titleFilePath, TITLE_HEADER % (self.authorName, self.storyTitle, self.adapter.getStoryURL(), self.storyTitle, self.adapter.getAuthorURL(), self.authorName))
|
||||
self._writeFile(titleFilePath, TITLE_ENTRY % ('Category:', self.adapter.getCategory()))
|
||||
self._writeFile(titleFilePath, TITLE_ENTRY % ('Genre:', self.adapter.getGenre()))
|
||||
self._writeFile(titleFilePath, TITLE_ENTRY % ('Status:', self.adapter.getStoryStatus()))
|
||||
self._writeFile(titleFilePath, TITLE_ENTRY % ('Published:', published))
|
||||
self._writeFile(titleFilePath, TITLE_ENTRY % ('Updated:', updated))
|
||||
self._writeFile(titleFilePath, TITLE_ENTRY % ('Packaged:', createda))
|
||||
tmpstr = self.adapter.getStoryRating() + " / " + self.adapter.getStoryUserRating()
|
||||
self._writeFile(titleFilePath, TITLE_ENTRY % ('Rating Age/User:', tmpstr))
|
||||
tmpstr = unicode(self.adapter.getNumChapters()) + " / " + commaGroups(unicode(self.adapter.getNumWords()))
|
||||
self._writeFile(titleFilePath, TITLE_ENTRY % ('Chapters/Words:', tmpstr))
|
||||
self._writeFile(titleFilePath, TITLE_ENTRY % ('Publisher:', self.adapter.getHost()))
|
||||
self._writeFile(titleFilePath, TITLE_ENTRY % ('Story ID:', self.adapter.getStoryId()))
|
||||
self._writeFile(titleFilePath, TITLE_ENTRY % ('Author ID:', self.adapter.getAuthorId()))
|
||||
|
||||
self._writeFile(titleFilePath, TITLE_FOOTER % description )
|
||||
|
||||
|
||||
|
||||
c = mobi.Converter(title=self.storyTitle,
|
||||
author=self.authorName,
|
||||
publisher=self.publisher)
|
||||
mobidata = c.ConvertStrings(self._getFilesStrings())
|
||||
|
||||
self.output.write(mobidata)
|
||||
if not self.inmemory:
|
||||
self.output.close()
|
||||
# zipdir.toZip(filename, self.directory)
|
||||
|
||||
|
||||
|
||||
class HTMLWriter(FanficWriter):
|
||||
@@ -205,7 +322,6 @@ class HTMLWriter(FanficWriter):
|
||||
|
||||
class EPubFanficWriter(FanficWriter):
|
||||
chapters = []
|
||||
|
||||
files = {}
|
||||
|
||||
@staticmethod
|
||||
@@ -360,22 +476,22 @@ class EPubFanficWriter(FanficWriter):
|
||||
|
||||
### writing content -- title page
|
||||
titleFilePath = "OEBPS/title_page.xhtml"
|
||||
self._writeFile(titleFilePath, TITLE_HEADER % (self.authorName, self.storyTitle, self.adapter.getStoryURL(), self.storyTitle, self.adapter.getAuthorURL(), self.authorName))
|
||||
self._writeFile(titleFilePath, TITLE_ENTRY % ('Category:', self.adapter.getCategory()))
|
||||
self._writeFile(titleFilePath, TITLE_ENTRY % ('Genre:', self.adapter.getGenre()))
|
||||
self._writeFile(titleFilePath, TITLE_ENTRY % ('Status:', self.adapter.getStoryStatus()))
|
||||
self._writeFile(titleFilePath, TITLE_ENTRY % ('Published:', published))
|
||||
self._writeFile(titleFilePath, TITLE_ENTRY % ('Updated:', updated))
|
||||
self._writeFile(titleFilePath, TITLE_ENTRY % ('Packaged:', createda))
|
||||
self._writeFile(titleFilePath, TABLE_TITLE_HEADER % (self.authorName, self.storyTitle, self.adapter.getStoryURL(), self.storyTitle, self.adapter.getAuthorURL(), self.authorName))
|
||||
self._writeFile(titleFilePath, TABLE_TITLE_ENTRY % ('Category:', self.adapter.getCategory()))
|
||||
self._writeFile(titleFilePath, TABLE_TITLE_ENTRY % ('Genre:', self.adapter.getGenre()))
|
||||
self._writeFile(titleFilePath, TABLE_TITLE_ENTRY % ('Status:', self.adapter.getStoryStatus()))
|
||||
self._writeFile(titleFilePath, TABLE_TITLE_ENTRY % ('Published:', published))
|
||||
self._writeFile(titleFilePath, TABLE_TITLE_ENTRY % ('Updated:', updated))
|
||||
self._writeFile(titleFilePath, TABLE_TITLE_ENTRY % ('Packaged:', createda))
|
||||
tmpstr = self.adapter.getStoryRating() + " / " + self.adapter.getStoryUserRating()
|
||||
self._writeFile(titleFilePath, TITLE_ENTRY % ('Rating Age/User:', tmpstr))
|
||||
self._writeFile(titleFilePath, TABLE_TITLE_ENTRY % ('Rating Age/User:', tmpstr))
|
||||
tmpstr = unicode(self.adapter.getNumChapters()) + " / " + commaGroups(unicode(self.adapter.getNumWords()))
|
||||
self._writeFile(titleFilePath, TITLE_ENTRY % ('Chapters/Words:', tmpstr))
|
||||
self._writeFile(titleFilePath, TITLE_ENTRY % ('Publisher:', self.adapter.getHost()))
|
||||
self._writeFile(titleFilePath, TITLE_ENTRY % ('Story ID:', self.adapter.getStoryId()))
|
||||
self._writeFile(titleFilePath, TITLE_ENTRY % ('Author ID:', self.adapter.getAuthorId()))
|
||||
self._writeFile(titleFilePath, TABLE_TITLE_ENTRY % ('Chapters/Words:', tmpstr))
|
||||
self._writeFile(titleFilePath, TABLE_TITLE_ENTRY % ('Publisher:', self.adapter.getHost()))
|
||||
self._writeFile(titleFilePath, TABLE_TITLE_ENTRY % ('Story ID:', self.adapter.getStoryId()))
|
||||
self._writeFile(titleFilePath, TABLE_TITLE_ENTRY % ('Author ID:', self.adapter.getAuthorId()))
|
||||
|
||||
self._writeFile(titleFilePath, TITLE_FOOTER % description )
|
||||
self._writeFile(titleFilePath, TABLE_TITLE_FOOTER % description )
|
||||
|
||||
### writing content -- opf file
|
||||
opfFilePath = "OEBPS/content.opf"
|
||||
|
||||
+1
-1
@@ -21,7 +21,7 @@ class Remover(webapp.RequestHandler):
|
||||
logging.debug("Starting r3m0v3r")
|
||||
user = users.get_current_user()
|
||||
logging.debug("Working as user %s" % user)
|
||||
theDate = datetime.date.today() - datetime.timedelta(days=7)
|
||||
theDate = datetime.date.today() - datetime.timedelta(days=2)
|
||||
logging.debug("Will delete stuff older than %s" % theDate)
|
||||
|
||||
fics = DownloadMeta.all()
|
||||
|
||||
Reference in New Issue
Block a user