From c6e06903c0ba36c72ce34e2ab121bbc759bc230c Mon Sep 17 00:00:00 2001 From: Jim Miller Date: Fri, 10 Feb 2012 18:46:42 -0600 Subject: [PATCH] Accept-Encoding=gzip, Custom output_css for EPUB & HTML. Many(most?) of the sites ignore gzip because of our User-Agent. ffnet honors it. Add progress bar while getting URLs or getting URLs for update. --- app.yaml | 2 +- calibre-plugin/__init__.py | 2 +- calibre-plugin/ffdl_plugin.py | 100 +++++++++++++++------- defaults.ini | 63 ++++++++++++-- downloader.py | 4 +- fanficdownloader/adapters/base_adapter.py | 3 +- fanficdownloader/gziphttp.py | 38 ++++++++ fanficdownloader/writers/base_writer.py | 6 ++ fanficdownloader/writers/writer_epub.py | 29 +------ fanficdownloader/writers/writer_html.py | 18 +--- fanficdownloader/writers/writer_mobi.py | 4 - index.html | 25 +----- plugin-defaults.ini | 65 ++++++++++++++ 13 files changed, 250 insertions(+), 109 deletions(-) create mode 100644 fanficdownloader/gziphttp.py diff --git a/app.yaml b/app.yaml index 652ca05..1fd9e75 100644 --- a/app.yaml +++ b/app.yaml @@ -1,6 +1,6 @@ # ffd-retief-hrd fanfictiondownloader application: fanfictiondownloader -version: 4-3-1 +version: 4-3-2 runtime: python27 api_version: 1 threadsafe: true diff --git a/calibre-plugin/__init__.py b/calibre-plugin/__init__.py index fd6fd1b..9659389 100644 --- a/calibre-plugin/__init__.py +++ b/calibre-plugin/__init__.py @@ -27,7 +27,7 @@ class FanFictionDownLoaderBase(InterfaceActionBase): description = 'UI plugin to download FanFiction stories from various sites.' supported_platforms = ['windows', 'osx', 'linux'] author = 'Jim Miller' - version = (1, 3, 4) + version = (1, 4, 0) minimum_calibre_version = (0, 8, 30) #: This field defines the GUI plugin class that contains all the code diff --git a/calibre-plugin/ffdl_plugin.py b/calibre-plugin/ffdl_plugin.py index 1a43c7d..fddf86e 100644 --- a/calibre-plugin/ffdl_plugin.py +++ b/calibre-plugin/ffdl_plugin.py @@ -243,22 +243,35 @@ class FanFictionDownLoaderPlugin(InterfaceAction): def get_list_urls(self): if len(self.gui.library_view.get_selected_ids()) > 0: - url_list = [] - for book_id in self.gui.library_view.get_selected_ids(): - url = self._get_story_url(self.gui.current_db, book_id) - if url != None: - url_list.append(url) + book_list = map( partial(self._convert_id_to_book, good=False), self.gui.library_view.get_selected_ids() ) - if url_list: - d = ViewLog(_("List of URLs"),"\n".join(url_list),parent=self.gui) - d.setWindowIcon(get_icon('bookmarks.png')) - d.exec_() - else: - info_dialog(self.gui, _('List of URLs'), - _('No URLs found in selected books.'), - show=True, - show_copy_button=False) + LoopProgressDialog(self.gui, + book_list, + partial(self._get_story_url_for_list, db=self.gui.current_db), + self._finish_get_list_urls, + init_label="Collecting URLs for stories...", + win_title="Get URLs for stories", + status_prefix="URL retrieved") + def _get_story_url_for_list(self,book,db=None): + book['url'] = self._get_story_url(db,book['calibre_id']) + if book['url'] == None: + book['good']=False + else: + book['good']=True + + def _finish_get_list_urls(self, book_list): + url_list = [ x['url'] for x in book_list if x['good'] ] + if url_list: + d = ViewLog(_("List of URLs"),"\n".join(url_list),parent=self.gui) + d.setWindowIcon(get_icon('bookmarks.png')) + d.exec_() + else: + info_dialog(self.gui, _('List of URLs'), + _('No URLs found in selected books.'), + show=True, + show_copy_button=False) + def add_dialog(self): #print("add_dialog()") @@ -294,17 +307,31 @@ class FanFictionDownLoaderPlugin(InterfaceAction): if len(self.gui.library_view.get_selected_ids()) == 0: return #print("update_existing()") - previous = self.gui.library_view.currentIndex() + db = self.gui.current_db - book_ids = self.gui.library_view.get_selected_ids() - books = self._convert_calibre_ids_to_books(db, book_ids) + book_list = map( partial(self._convert_id_to_book, good=False), self.gui.library_view.get_selected_ids() ) + #book_ids = self.gui.library_view.get_selected_ids() + + LoopProgressDialog(self.gui, + book_list, + partial(self._populate_book_from_calibre_id, db=self.gui.current_db), + self._update_existing_2, + init_label="Collecting stories for update...", + win_title="Get stories for updates", + status_prefix="URL retrieved") + + + #books = self._convert_calibre_ids_to_books(db, book_ids) #print("update books:%s"%books) + ## XXX split here. + def _update_existing_2(self,book_list): + d = UpdateExistingDialog(self.gui, 'Update Existing List', prefs, self.qaction.icon(), - books, + book_list, ) d.exec_() if d.result() != d.Accepted: @@ -858,16 +885,30 @@ class FanFictionDownLoaderPlugin(InterfaceAction): self._set_book_url_and_comment(book,url) return book - - def _convert_calibre_ids_to_books(self, db, ids): - books = [] - for book_id in ids: - books.append(self._convert_calibre_id_to_book(db,book_id)) - return books - - def _convert_calibre_id_to_book(self, db, book_id): - mi = db.get_metadata(book_id, index_is_id=True) + def _convert_id_to_book(self, idval, good=True): book = {} + book['good'] = good + book['calibre_id'] = idval + book['title'] = 'Unknown' + book['author'] = 'Unknown' + book['author_sort'] = 'Unknown' + + book['comment'] = '' + book['url'] = '' + book['added'] = False + + return book + + + # def _convert_calibre_ids_to_books(self, db, ids): + # books = [] + # for book_id in ids: + # books.append(self._convert_calibre_id_to_book(db,book_id)) + # return books + + def _populate_book_from_calibre_id(self, book, db=None): + mi = db.get_metadata(book['calibre_id'], index_is_id=True) + #book = {} book['good'] = True book['calibre_id'] = mi.id book['title'] = mi.title @@ -877,10 +918,9 @@ class FanFictionDownLoaderPlugin(InterfaceAction): book['url'] = "" book['added'] = False - url = self._get_story_url(db,book_id) + url = self._get_story_url(db,book['calibre_id']) self._set_book_url_and_comment(book,url) - - return book + #return book def _set_book_url_and_comment(self,book,url): if not url: diff --git a/defaults.ini b/defaults.ini index be36f33..a832393 100644 --- a/defaults.ini +++ b/defaults.ini @@ -128,10 +128,6 @@ extratags: FanFiction ## Primarily for commandline. #slow_down_sleep_time:0.5 -## output background color--only used by html and epub (and ignored in -## epub by many readers). Must be hex code, # will be added. -background_color: ffffff - ## For use only with stand-alone CLI version--run a command on the ## generated file after it's produced. All of the titlepage_entries ## values are available, plus output_filename. @@ -141,6 +137,30 @@ background_color: ffffff ## Each output format has a section that overrides [defaults] [html] +## output background color--only used by html and epub (and ignored in +## epub by many readers). Included below in output_css--will be +## ignored if not in output_css. +background_color: ffffff + +## Allow customization of CSS. Make sure to keep at least one space +## at the start of each line and to escape % to %%. Also need +## background_color to be in the same section, if included in CSS. +output_css: + body { background-color: #%(background_color)s; } + .CI { + text-align:center; + margin-top:0px; + margin-bottom:0px; + padding:0px; + } + .center {text-align: center;} + .cover {text-align: center;} + .full {width: 100%%; } + .quarter {width: 25%%; } + .smcap {font-variant: small-caps;} + .u {text-decoration: underline;} + .bold {font-weight: bold;} + [txt] ## Add URLs since there aren't links. titlepage_entries: series,category,genre,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,storyUrl, authorUrl, description @@ -163,11 +183,44 @@ titlepage_use_table: false ## When using tables, make these span both columns. wide_titlepage_entries: description, storyUrl, author URL +## output background color--only used by html and epub (and ignored in +## epub by many readers). Included below in output_css--will be +## ignored if not in output_css. +background_color: ffffff + +## Allow customization of CSS. Make sure to keep at least one space +## at the start of each line and to escape % to %%. Also need +## background_color to be in the same section, if included in CSS. +output_css: + body { background-color: #%(background_color)s; + text-align: justify; + margin: 2%%; } + pre { font-size: x-small; } + sml { font-size: small; } + h1 { text-align: center; } + h2 { text-align: center; } + h3 { text-align: center; } + h4 { text-align: center; } + h5 { text-align: center; } + h6 { text-align: center; } + .CI { + text-align:center; + margin-top:0px; + margin-bottom:0px; + padding:0px; + } + .center {text-align: center;} + .cover {text-align: center;} + .full {width: 100%%; } + .quarter {width: 25%%; } + .smcap {font-variant: small-caps;} + .u {text-decoration: underline;} + .bold {font-weight: bold;} + [mobi] ## mobi TOC cannot be turned off right now. #include_tocpage: true - ## Each site has a section that overrides [defaults] *and* the format ## sections test1.com specifically is not a real story site. Instead, ## it is a fake site for testing configuration and output. It uses diff --git a/downloader.py b/downloader.py index a6533a2..4bfcf79 100644 --- a/downloader.py +++ b/downloader.py @@ -25,6 +25,7 @@ from StringIO import StringIO from optparse import OptionParser import getpass import string +import time from subprocess import call @@ -46,7 +47,6 @@ def writeStory(config,adapter,writeformat,metaonly=False,outstream=None): return output_filename def main(): - # read in args, anything starting with -- will be treated as --= usage = "usage: %prog [options] storyurl" parser = OptionParser(usage) @@ -215,4 +215,6 @@ def main(): print us if __name__ == "__main__": + start = time.time() main() + print("Total time seconds:%f"%(time.time()-start)) diff --git a/fanficdownloader/adapters/base_adapter.py b/fanficdownloader/adapters/base_adapter.py index 7aaa3a1..a041975 100644 --- a/fanficdownloader/adapters/base_adapter.py +++ b/fanficdownloader/adapters/base_adapter.py @@ -40,6 +40,7 @@ except: #logging.info("Hook to make default deadline 10.0 NOT installed--not using appengine") from ..story import Story +from ..gziphttp import GZipProcessor from ..configurable import Configurable from ..htmlcleanup import removeEntities, removeAllEntities, stripHTML from ..exceptions import InvalidStoryURL @@ -72,7 +73,7 @@ class BaseSiteAdapter(Configurable): self.password = "" self.is_adult=False - self.opener = u2.build_opener(u2.HTTPCookieProcessor()) + self.opener = u2.build_opener(u2.HTTPCookieProcessor(),GZipProcessor()) self.storyDone = False self.metadataDone = False self.story = Story() diff --git a/fanficdownloader/gziphttp.py b/fanficdownloader/gziphttp.py new file mode 100644 index 0000000..76049ee --- /dev/null +++ b/fanficdownloader/gziphttp.py @@ -0,0 +1,38 @@ +## Borrowed from http://techknack.net/python-urllib2-handlers/ + +import urllib2 +from gzip import GzipFile +from StringIO import StringIO + +class GZipProcessor(urllib2.BaseHandler): + """A handler to add gzip capabilities to urllib2 requests + """ + def http_request(self, req): + req.add_header("Accept-Encoding", "gzip") + return req + https_request = http_request + + def http_response(self, req, resp): + #print("Content-Encoding:%s"%resp.headers.get("Content-Encoding")) + if resp.headers.get("Content-Encoding") == "gzip": + gz = GzipFile( + fileobj=StringIO(resp.read()), + mode="r" + ) +# resp.read = gz.read +# resp.readlines = gz.readlines +# resp.readline = gz.readline +# resp.next = gz.next + old_resp = resp + resp = urllib2.addinfourl(gz, old_resp.headers, old_resp.url, old_resp.code) + resp.msg = old_resp.msg + return resp + https_response = http_response + +# brave new world - 1:30 w/o, 1:10 with? 40 chapters, so 20s from sleeps. +# with gzip, no sleep: 47.469 +# w/o gzip, no sleep: 47.736 + +# I Am What I Am 67 chapters +# w/o gzip: 57.168 +# w/ gzip: 40.692 diff --git a/fanficdownloader/writers/base_writer.py b/fanficdownloader/writers/base_writer.py index de97690..121f943 100644 --- a/fanficdownloader/writers/base_writer.py +++ b/fanficdownloader/writers/base_writer.py @@ -194,6 +194,12 @@ class BaseStoryWriter(Configurable): if outfilename == None: outfilename=self.getOutputFileName() + # minor cheat, tucking css into metadata. + if self.getConfig("output_css"): + self.story.metadata["output_css"] = self.getConfig("output_css") + else: + self.story.metadata["output_css"] = '' + if not outstream: close=True logging.debug("Save directly to file: %s" % outfilename) diff --git a/fanficdownloader/writers/writer_epub.py b/fanficdownloader/writers/writer_epub.py index aa091bf..acd0dcd 100644 --- a/fanficdownloader/writers/writer_epub.py +++ b/fanficdownloader/writers/writer_epub.py @@ -41,32 +41,7 @@ class EpubWriter(BaseStoryWriter): def __init__(self, config, story): BaseStoryWriter.__init__(self, config, story) - self.EPUB_CSS = string.Template(''' -body { margin: 2%; - text-align: justify; - background-color: #${background_color}; } -pre { font-size: x-small; } -sml { font-size: small; } -h1 { text-align: center; } -h2 { text-align: center; } -h3 { text-align: center; } -h4 { text-align: center; } -h5 { text-align: center; } -h6 { text-align: center; } -.CI { - text-align:center; - margin-top:0px; - margin-bottom:0px; - padding:0px; - } -.center {text-align: center;} -.cover {text-align: center;} -.full {width: 100%; } -.quarter {width: 25%; } -.smcap {font-variant: small-caps;} -.u {text-decoration: underline;} -.bold {font-weight: bold;} -''') + self.EPUB_CSS = string.Template('''${output_css}''') self.EPUB_TITLE_PAGE_START = string.Template(''' @@ -361,7 +336,7 @@ h6 { text-align: center; } del tocncxdom # write stylesheet.css file. - outputepub.writestr("OEBPS/stylesheet.css",self.EPUB_CSS.substitute({"background_color":self.getConfig("background_color")})) + outputepub.writestr("OEBPS/stylesheet.css",self.EPUB_CSS.substitute(self.story.metadata)) # write title page. if self.getConfig("titlepage_use_table"): diff --git a/fanficdownloader/writers/writer_html.py b/fanficdownloader/writers/writer_html.py index cec520a..5b0b1ea 100644 --- a/fanficdownloader/writers/writer_html.py +++ b/fanficdownloader/writers/writer_html.py @@ -39,20 +39,7 @@ class HTMLWriter(BaseStoryWriter): ${title} by ${author} @@ -95,9 +82,6 @@ body { background-color: #${background_color}; } def writeStoryImpl(self, out): - # minor cheat, tucking bg into metadata. - if self.getConfig("background_color"): - self.story.metadata["background_color"] = self.getConfig("background_color") self._write(out,self.HTML_FILE_START.substitute(self.story.metadata)) self.writeTitlePage(out, diff --git a/fanficdownloader/writers/writer_mobi.py b/fanficdownloader/writers/writer_mobi.py index bb141c6..d6ced53 100644 --- a/fanficdownloader/writers/writer_mobi.py +++ b/fanficdownloader/writers/writer_mobi.py @@ -41,7 +41,6 @@ class MobiWriter(BaseStoryWriter): ${title} by ${author} -

${title} by ${author}

@@ -64,7 +63,6 @@ class MobiWriter(BaseStoryWriter): ${title} by ${author} -

${title} by ${author}

@@ -91,7 +89,6 @@ class MobiWriter(BaseStoryWriter): ${title} by ${author} -
@@ -113,7 +110,6 @@ class MobiWriter(BaseStoryWriter): ${chapter} -

${chapter}

diff --git a/index.html b/index.html index a498875..d96a0ac 100644 --- a/index.html +++ b/index.html @@ -54,34 +54,15 @@ much easier.

-

Support for 'Series'

+

Support for Custom CSS

- We now collect 'Series' name and number for the sites: - harrypotterfanfiction.com, - potionsandsnitches.net, - adastrafanfic.com, - whofic.com, - fanfiction.tenhawkpresents.com, - castlefans.org, - tthfanfic.org, - www.siye.co.uk, - twilighted.net*, - twilighted.net* and - thewriterscoffeeshop.com*. -

-

- * The last three use series as reading lists and stories collections as much as true story series, - so they default to not collect series info. You can turn it on in your User Configuration if you want. -

-

New Site: archiveofourown.org

-

- Thanks to Ida Leter for writing the code to support a new site: archiveofourown.org. + The CSS included in the HTML and EPUB output formats is now a customizable parameter.

If you have any problems with this application, please report them in the FanFictionDownLoader Google Group. The - Previous Version is also available for you to use if necessary. + Previous Version is also available for you to use if necessary.

{{ error_message }} diff --git a/plugin-defaults.ini b/plugin-defaults.ini index 3cc6ab1..251ce3b 100644 --- a/plugin-defaults.ini +++ b/plugin-defaults.ini @@ -114,6 +114,37 @@ background_color: ffffff ## Each output format has a section that overrides [defaults] [html] +## output background color--only used by html and epub (and ignored in +## epub by many readers). Included below in output_css--will be +## ignored if not in output_css. +background_color: ffffff + +## Allow customization of CSS. Make sure to keep at least one space +## at the start of each line and to escape % to %%. Also need +## background_color to be in the same section, if included in CSS. +output_css: + body { background-color: #%(background_color)s; } + .CI { + text-align:center; + margin-top:0px; + margin-bottom:0px; + padding:0px; + } + .center {text-align: center;} + .cover {text-align: center;} + .full {width: 100%%; } + .quarter {width: 25%%; } + .smcap {font-variant: small-caps;} + .u {text-decoration: underline;} + .bold {font-weight: bold;} + +[txt] +## Add URLs since there aren't links. +titlepage_entries: series,category,genre,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,storyUrl, authorUrl, description + +## use \r\n for line endings, the windows convention. text output only. +windows_eol: true + [txt] ## Add URLs since there aren't links. titlepage_entries: series,category,genre,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,storyUrl, authorUrl, description @@ -133,6 +164,40 @@ titlepage_use_table: false ## When using tables, make these span both columns. wide_titlepage_entries: description, storyUrl, author URL +## output background color--only used by html and epub (and ignored in +## epub by many readers). Included below in output_css--will be +## ignored if not in output_css. +background_color: ffffff + +## Allow customization of CSS. Make sure to keep at least one space +## at the start of each line and to escape % to %%. Also need +## background_color to be in the same section, if included in CSS. +output_css: + body { background-color: #%(background_color)s; + text-align: justify; + margin: 2%%; } + pre { font-size: x-small; } + sml { font-size: small; } + h1 { text-align: center; } + h2 { text-align: center; } + h3 { text-align: center; } + h4 { text-align: center; } + h5 { text-align: center; } + h6 { text-align: center; } + .CI { + text-align:center; + margin-top:0px; + margin-bottom:0px; + padding:0px; + } + .center {text-align: center;} + .cover {text-align: center;} + .full {width: 100%%; } + .quarter {width: 25%%; } + .smcap {font-variant: small-caps;} + .u {text-decoration: underline;} + .bold {font-weight: bold;} + [mobi] ## mobi TOC cannot be turned off right now. #include_tocpage: true