wassname/FanFicFare
1
0
Fork 0
mirror of https://github.com/wassname/FanFicFare.git synced 2026-09-10 11:40:24 +08:00
Code Issues Packages Projects Releases Wiki Activity

Accept-Encoding=gzip, Custom output_css for EPUB & HTML.

Browse Source 
Many(most?) of the sites ignore gzip because of our User-Agent. ffnet honors it.
Add progress bar while getting URLs or getting URLs for update.
This commit is contained in:
Jim Miller
2012-02-10 18:46:42 -06:00
parent b19385ada3
commit c6e06903c0
13 changed files with 250 additions and 109 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app.yaml
+1 -1
View File
@@ -1,6 +1,6 @@
# ffd-retief-hrd fanfictiondownloader
application: fanfictiondownloader
version: 4-3-1
version: 4-3-2
runtime: python27
api_version: 1
threadsafe: true
calibre-plugin/__init__.py
+1 -1
View File
@@ -27,7 +27,7 @@ class FanFictionDownLoaderBase(InterfaceActionBase):
description = 'UI plugin to download FanFiction stories from various sites.'
supported_platforms = ['windows', 'osx', 'linux']
author = 'Jim Miller'
version = (1, 3, 4)
version = (1, 4, 0)
minimum_calibre_version = (0, 8, 30)
#: This field defines the GUI plugin class that contains all the code
calibre-plugin/ffdl_plugin.py
+70 -30
View File
@@ -243,22 +243,35 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
def get_list_urls(self):
if len(self.gui.library_view.get_selected_ids()) > 0:
url_list = []
for book_id in self.gui.library_view.get_selected_ids():
url = self._get_story_url(self.gui.current_db, book_id)
if url != None:
url_list.append(url)
book_list = map( partial(self._convert_id_to_book, good=False), self.gui.library_view.get_selected_ids() )
if url_list:
d = ViewLog(_("List of URLs"),"\n".join(url_list),parent=self.gui)
d.setWindowIcon(get_icon('bookmarks.png'))
d.exec_()
else:
info_dialog(self.gui, _('List of URLs'),
_('No URLs found in selected books.'),
show=True,
show_copy_button=False)
LoopProgressDialog(self.gui,
book_list,
partial(self._get_story_url_for_list, db=self.gui.current_db),
self._finish_get_list_urls,
init_label="Collecting URLs for stories...",
win_title="Get URLs for stories",
status_prefix="URL retrieved")
def _get_story_url_for_list(self,book,db=None):
book['url'] = self._get_story_url(db,book['calibre_id'])
if book['url'] == None:
book['good']=False
else:
book['good']=True
def _finish_get_list_urls(self, book_list):
url_list = [ x['url'] for x in book_list if x['good'] ]
if url_list:
d = ViewLog(_("List of URLs"),"\n".join(url_list),parent=self.gui)
d.setWindowIcon(get_icon('bookmarks.png'))
d.exec_()
else:
info_dialog(self.gui, _('List of URLs'),
_('No URLs found in selected books.'),
show=True,
show_copy_button=False)
def add_dialog(self):
#print("add_dialog()")
@@ -294,17 +307,31 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
if len(self.gui.library_view.get_selected_ids()) == 0:
return
#print("update_existing()")
previous = self.gui.library_view.currentIndex()
db = self.gui.current_db
book_ids = self.gui.library_view.get_selected_ids()
books = self._convert_calibre_ids_to_books(db, book_ids)
book_list = map( partial(self._convert_id_to_book, good=False), self.gui.library_view.get_selected_ids() )
#book_ids = self.gui.library_view.get_selected_ids()
LoopProgressDialog(self.gui,
book_list,
partial(self._populate_book_from_calibre_id, db=self.gui.current_db),
self._update_existing_2,
init_label="Collecting stories for update...",
win_title="Get stories for updates",
status_prefix="URL retrieved")
#books = self._convert_calibre_ids_to_books(db, book_ids)
#print("update books:%s"%books)
## XXX split here.
def _update_existing_2(self,book_list):
d = UpdateExistingDialog(self.gui,
'Update Existing List',
prefs,
self.qaction.icon(),
books,
book_list,
)
d.exec_()
if d.result() != d.Accepted:
@@ -858,16 +885,30 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
self._set_book_url_and_comment(book,url)
return book
def _convert_calibre_ids_to_books(self, db, ids):
books = []
for book_id in ids:
books.append(self._convert_calibre_id_to_book(db,book_id))
return books
def _convert_calibre_id_to_book(self, db, book_id):
mi = db.get_metadata(book_id, index_is_id=True)
def _convert_id_to_book(self, idval, good=True):
book = {}
book['good'] = good
book['calibre_id'] = idval
book['title'] = 'Unknown'
book['author'] = 'Unknown'
book['author_sort'] = 'Unknown'
book['comment'] = ''
book['url'] = ''
book['added'] = False
return book
# def _convert_calibre_ids_to_books(self, db, ids):
# books = []
# for book_id in ids:
# books.append(self._convert_calibre_id_to_book(db,book_id))
# return books
def _populate_book_from_calibre_id(self, book, db=None):
mi = db.get_metadata(book['calibre_id'], index_is_id=True)
#book = {}
book['good'] = True
book['calibre_id'] = mi.id
book['title'] = mi.title
@@ -877,10 +918,9 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
book['url'] = ""
book['added'] = False
url = self._get_story_url(db,book_id)
url = self._get_story_url(db,book['calibre_id'])
self._set_book_url_and_comment(book,url)
return book
#return book
def _set_book_url_and_comment(self,book,url):
if not url:
defaults.ini
+58 -5
View File
@@ -128,10 +128,6 @@ extratags: FanFiction
## Primarily for commandline.
#slow_down_sleep_time:0.5
## output background color--only used by html and epub (and ignored in
## epub by many readers). Must be hex code, # will be added.
background_color: ffffff
## For use only with stand-alone CLI version--run a command on the
## generated file after it's produced. All of the titlepage_entries
## values are available, plus output_filename.
@@ -141,6 +137,30 @@ background_color: ffffff
## Each output format has a section that overrides [defaults]
[html]
## output background color--only used by html and epub (and ignored in
## epub by many readers). Included below in output_css--will be
## ignored if not in output_css.
background_color: ffffff
## Allow customization of CSS. Make sure to keep at least one space
## at the start of each line and to escape % to %%. Also need
## background_color to be in the same section, if included in CSS.
output_css:
body { background-color: #%(background_color)s; }
.CI {
text-align:center;
margin-top:0px;
margin-bottom:0px;
padding:0px;
}
.center {text-align: center;}
.cover {text-align: center;}
.full {width: 100%%; }
.quarter {width: 25%%; }
.smcap {font-variant: small-caps;}
.u {text-decoration: underline;}
.bold {font-weight: bold;}
[txt]
## Add URLs since there aren't links.
titlepage_entries: series,category,genre,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,storyUrl, authorUrl, description
@@ -163,11 +183,44 @@ titlepage_use_table: false
## When using tables, make these span both columns.
wide_titlepage_entries: description, storyUrl, author URL
## output background color--only used by html and epub (and ignored in
## epub by many readers). Included below in output_css--will be
## ignored if not in output_css.
background_color: ffffff
## Allow customization of CSS. Make sure to keep at least one space
## at the start of each line and to escape % to %%. Also need
## background_color to be in the same section, if included in CSS.
output_css:
body { background-color: #%(background_color)s;
text-align: justify;
margin: 2%%; }
pre { font-size: x-small; }
sml { font-size: small; }
h1 { text-align: center; }
h2 { text-align: center; }
h3 { text-align: center; }
h4 { text-align: center; }
h5 { text-align: center; }
h6 { text-align: center; }
.CI {
text-align:center;
margin-top:0px;
margin-bottom:0px;
padding:0px;
}
.center {text-align: center;}
.cover {text-align: center;}
.full {width: 100%%; }
.quarter {width: 25%%; }
.smcap {font-variant: small-caps;}
.u {text-decoration: underline;}
.bold {font-weight: bold;}
[mobi]
## mobi TOC cannot be turned off right now.
#include_tocpage: true
## Each site has a section that overrides [defaults] *and* the format
## sections test1.com specifically is not a real story site. Instead,
## it is a fake site for testing configuration and output. It uses
downloader.py
+3 -1
View File
@@ -25,6 +25,7 @@ from StringIO import StringIO
from optparse import OptionParser
import getpass
import string
import time
from subprocess import call
@@ -46,7 +47,6 @@ def writeStory(config,adapter,writeformat,metaonly=False,outstream=None):
return output_filename
def main():
# read in args, anything starting with -- will be treated as --<varible>=<value>
usage = "usage: %prog [options] storyurl"
parser = OptionParser(usage)
@@ -215,4 +215,6 @@ def main():
print us
if __name__ == "__main__":
start = time.time()
main()
print("Total time seconds:%f"%(time.time()-start))
fanficdownloader/adapters/base_adapter.py
+2 -1
View File
@@ -40,6 +40,7 @@ except:
#logging.info("Hook to make default deadline 10.0 NOT installed--not using appengine")
from ..story import Story
from ..gziphttp import GZipProcessor
from ..configurable import Configurable
from ..htmlcleanup import removeEntities, removeAllEntities, stripHTML
from ..exceptions import InvalidStoryURL
@@ -72,7 +73,7 @@ class BaseSiteAdapter(Configurable):
self.password = ""
self.is_adult=False
self.opener = u2.build_opener(u2.HTTPCookieProcessor())
self.opener = u2.build_opener(u2.HTTPCookieProcessor(),GZipProcessor())
self.storyDone = False
self.metadataDone = False
self.story = Story()
fanficdownloader/gziphttp.py
+38
View File
@@ -0,0 +1,38 @@
## Borrowed from http://techknack.net/python-urllib2-handlers/
import urllib2
from gzip import GzipFile
from StringIO import StringIO
class GZipProcessor(urllib2.BaseHandler):
"""A handler to add gzip capabilities to urllib2 requests
"""
def http_request(self, req):
req.add_header("Accept-Encoding", "gzip")
return req
https_request = http_request
def http_response(self, req, resp):
#print("Content-Encoding:%s"%resp.headers.get("Content-Encoding"))
if resp.headers.get("Content-Encoding") == "gzip":
gz = GzipFile(
fileobj=StringIO(resp.read()),
mode="r"
)
# resp.read = gz.read
# resp.readlines = gz.readlines
# resp.readline = gz.readline
# resp.next = gz.next
old_resp = resp
resp = urllib2.addinfourl(gz, old_resp.headers, old_resp.url, old_resp.code)
resp.msg = old_resp.msg
return resp
https_response = http_response
# brave new world - 1:30 w/o, 1:10 with? 40 chapters, so 20s from sleeps.
# with gzip, no sleep: 47.469
# w/o gzip, no sleep: 47.736
# I Am What I Am 67 chapters
# w/o gzip: 57.168
# w/ gzip: 40.692
fanficdownloader/writers/base_writer.py
+6
View File
@@ -194,6 +194,12 @@ class BaseStoryWriter(Configurable):
if outfilename == None:
outfilename=self.getOutputFileName()
# minor cheat, tucking css into metadata.
if self.getConfig("output_css"):
self.story.metadata["output_css"] = self.getConfig("output_css")
else:
self.story.metadata["output_css"] = ''
if not outstream:
close=True
logging.debug("Save directly to file: %s" % outfilename)
fanficdownloader/writers/writer_epub.py
+2 -27
View File
@@ -41,32 +41,7 @@ class EpubWriter(BaseStoryWriter):
def __init__(self, config, story):
BaseStoryWriter.__init__(self, config, story)
self.EPUB_CSS = string.Template('''
body { margin: 2%;
text-align: justify;
background-color: #${background_color}; }
pre { font-size: x-small; }
sml { font-size: small; }
h1 { text-align: center; }
h2 { text-align: center; }
h3 { text-align: center; }
h4 { text-align: center; }
h5 { text-align: center; }
h6 { text-align: center; }
.CI {
text-align:center;
margin-top:0px;
margin-bottom:0px;
padding:0px;
}
.center {text-align: center;}
.cover {text-align: center;}
.full {width: 100%; }
.quarter {width: 25%; }
.smcap {font-variant: small-caps;}
.u {text-decoration: underline;}
.bold {font-weight: bold;}
''')
self.EPUB_CSS = string.Template('''${output_css}''')
self.EPUB_TITLE_PAGE_START = string.Template('''<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
@@ -361,7 +336,7 @@ h6 { text-align: center; }
del tocncxdom
# write stylesheet.css file.
outputepub.writestr("OEBPS/stylesheet.css",self.EPUB_CSS.substitute({"background_color":self.getConfig("background_color")}))
outputepub.writestr("OEBPS/stylesheet.css",self.EPUB_CSS.substitute(self.story.metadata))
# write title page.
if self.getConfig("titlepage_use_table"):
fanficdownloader/writers/writer_html.py
+1 -17
View File
@@ -39,20 +39,7 @@ class HTMLWriter(BaseStoryWriter):
<head>
<title>${title} by ${author}</title>
<style type="text/css">
body { background-color: #${background_color}; }
.CI {
text-align:center;
margin-top:0px;
margin-bottom:0px;
padding:0px;
}
.center {text-align: center;}
.cover {text-align: center;}
.full {width: 100%; }
.quarter {width: 25%; }
.smcap {font-variant: small-caps;}
.u {text-decoration: underline;}
.bold {font-weight: bold;}
${output_css}
</style>
</head>
<body>
@@ -95,9 +82,6 @@ body { background-color: #${background_color}; }
def writeStoryImpl(self, out):
# minor cheat, tucking bg into metadata.
if self.getConfig("background_color"):
self.story.metadata["background_color"] = self.getConfig("background_color")
self._write(out,self.HTML_FILE_START.substitute(self.story.metadata))
self.writeTitlePage(out,
fanficdownloader/writers/writer_mobi.py
-4
View File
@@ -41,7 +41,6 @@ class MobiWriter(BaseStoryWriter):
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<title>${title} by ${author}</title>
<link href="stylesheet.css" type="text/css" charset="UTF-8" rel="stylesheet"/>
</head>
<body>
<h3><a href="${storyUrl}">${title}</a> by <a href="${authorUrl}">${author}</a></h3>
@@ -64,7 +63,6 @@ class MobiWriter(BaseStoryWriter):
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<title>${title} by ${author}</title>
<link href="stylesheet.css" type="text/css" charset="UTF-8" rel="stylesheet"/>
</head>
<body>
<h3><a href="${storyUrl}">${title}</a> by <a href="${authorUrl}">${author}</a></h3>
@@ -91,7 +89,6 @@ class MobiWriter(BaseStoryWriter):
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<title>${title} by ${author}</title>
<link href="stylesheet.css" type="text/css" charset="UTF-8" rel="stylesheet"/>
</head>
<body>
<div>
@@ -113,7 +110,6 @@ class MobiWriter(BaseStoryWriter):
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<title>${chapter}</title>
<link href="stylesheet.css" type="text/css" charset="UTF-8" rel="stylesheet"/>
</head>
<body>
<h3>${chapter}</h3>
index.html
+3 -22
View File
@@ -54,34 +54,15 @@
much easier. </p>
</div>
<!-- put announcements here, h3 is a good title size. -->
<h3>Support for 'Series'</h3>
<h3>Support for Custom CSS</h3>
<p>
We now collect 'Series' name and number for the sites:
harrypotterfanfiction.com,
potionsandsnitches.net,
adastrafanfic.com,
whofic.com,
fanfiction.tenhawkpresents.com,
castlefans.org,
tthfanfic.org,
www.siye.co.uk,
twilighted.net*,
twilighted.net* and
thewriterscoffeeshop.com*.
</p>
<p>
* The last three use series as reading lists and stories collections as much as true story series,
so they default to <i>not</i> collect series info. You can turn it on in your User Configuration if you want.
</p>
<h3>New Site: <a href="http://archiveofourown.org">archiveofourown.org</a></h3>
<p>
Thanks to Ida Leter for writing the code to support a new site: <a href="http://archiveofourown.org">archiveofourown.org</a>.
The CSS included in the HTML and EPUB output formats is now a customizable parameter.
</p>
<p>
If you have any problems with this application, please
report them in
the <a href="http://groups.google.com/group/fanfic-downloader">FanFictionDownLoader Google Group</a>. The
<a href="http://4-3-0.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
<a href="http://4-3-1.fanfictiondownloader.appspot.com">Previous Version</a> is also available for you to use if necessary.
</p>
<div id='error'>
{{ error_message }}
plugin-defaults.ini
+65
View File
@@ -114,6 +114,37 @@ background_color: ffffff
## Each output format has a section that overrides [defaults]
[html]
## output background color--only used by html and epub (and ignored in
## epub by many readers). Included below in output_css--will be
## ignored if not in output_css.
background_color: ffffff
## Allow customization of CSS. Make sure to keep at least one space
## at the start of each line and to escape % to %%. Also need
## background_color to be in the same section, if included in CSS.
output_css:
body { background-color: #%(background_color)s; }
.CI {
text-align:center;
margin-top:0px;
margin-bottom:0px;
padding:0px;
}
.center {text-align: center;}
.cover {text-align: center;}
.full {width: 100%%; }
.quarter {width: 25%%; }
.smcap {font-variant: small-caps;}
.u {text-decoration: underline;}
.bold {font-weight: bold;}
[txt]
## Add URLs since there aren't links.
titlepage_entries: series,category,genre,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,storyUrl, authorUrl, description
## use \r\n for line endings, the windows convention. text output only.
windows_eol: true
[txt]
## Add URLs since there aren't links.
titlepage_entries: series,category,genre,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,storyUrl, authorUrl, description
@@ -133,6 +164,40 @@ titlepage_use_table: false
## When using tables, make these span both columns.
wide_titlepage_entries: description, storyUrl, author URL
## output background color--only used by html and epub (and ignored in
## epub by many readers). Included below in output_css--will be
## ignored if not in output_css.
background_color: ffffff
## Allow customization of CSS. Make sure to keep at least one space
## at the start of each line and to escape % to %%. Also need
## background_color to be in the same section, if included in CSS.
output_css:
body { background-color: #%(background_color)s;
text-align: justify;
margin: 2%%; }
pre { font-size: x-small; }
sml { font-size: small; }
h1 { text-align: center; }
h2 { text-align: center; }
h3 { text-align: center; }
h4 { text-align: center; }
h5 { text-align: center; }
h6 { text-align: center; }
.CI {
text-align:center;
margin-top:0px;
margin-bottom:0px;
padding:0px;
}
.center {text-align: center;}
.cover {text-align: center;}
.full {width: 100%%; }
.quarter {width: 25%%; }
.smcap {font-variant: small-caps;}
.u {text-decoration: underline;}
.bold {font-weight: bold;}
[mobi]
## mobi TOC cannot be turned off right now.
#include_tocpage: true