From baaf151fa26c5c1f3f77e306e37a47b704fd436b Mon Sep 17 00:00:00 2001 From: Jim Miller Date: Fri, 24 Feb 2012 22:24:55 -0600 Subject: [PATCH] Hardcode not to d/l images in appengine. Better no-img lib support for non-jpg. Pass formtype into adapter in CLI & Web. Center fimfic cover image in summary. --- downloader.py | 2 +- .../adapters/adapter_fimfictionnet.py | 2 +- fanficdownloader/story.py | 63 +++++++++++++------ fanficdownloader/writers/writer_epub.py | 15 ++--- main.py | 4 +- 5 files changed, 51 insertions(+), 35 deletions(-) diff --git a/downloader.py b/downloader.py index 70262c4..6a6c07b 100644 --- a/downloader.py +++ b/downloader.py @@ -128,7 +128,7 @@ def main(): else: url = args[0] - adapter = adapters.getAdapter(config,url) + adapter = adapters.getAdapter(config,url,options.format) ## three tries, that's enough if both user/pass & is_adult needed, ## or a couple tries of one or the other diff --git a/fanficdownloader/adapters/adapter_fimfictionnet.py b/fanficdownloader/adapters/adapter_fimfictionnet.py index ae3328d..f4cd135 100644 --- a/fanficdownloader/adapters/adapter_fimfictionnet.py +++ b/fanficdownloader/adapters/adapter_fimfictionnet.py @@ -146,7 +146,7 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter): if self.getConfig('keep_summary_html') and \ self.getConfig('include_images') and \ story_img: - self.setDescription(self.url,"%s
%s"%(story_img,description_soup.text)) + self.setDescription(self.url,"

%s

%s"%(story_img,description_soup.text)) else: self.setDescription(self.url,description_soup.text) #self.story.setMetadata('description', description_soup.text) diff --git a/fanficdownloader/story.py b/fanficdownloader/story.py index c618e7c..6d9f776 100644 --- a/fanficdownloader/story.py +++ b/fanficdownloader/story.py @@ -25,16 +25,33 @@ from htmlcleanup import conditionalRemoveEntities, removeAllEntities try: from calibre.utils.magick.draw import minify_image - def convert_image(data,sizes,grayscale): + def convert_image(url,data,sizes,grayscale): img = minify_image(data, minify_to=sizes) if grayscale: img.type = "GrayscaleType" - return img.export('JPG') + return (img.export('JPG'),'jpg','image/jpeg') except: + imagetypes = { + 'jpg':'image/jpeg', + 'jpeg':'image/jpeg', + 'png':'image/png', + 'gif':'image/gif', + 'svg':'image/svg+xml', + } + # Problem: writer_epub assumes image is jpg. - def convert_image(data,sizes,grayscale): - return data - + def convert_image(url,data,sizes,grayscale): + ext=url[url.rfind('.')+1:].lower() + return (data,ext,imagetypes[ext]) + +try: + # doesn't really matter what, just checking for appengine. + from google.appengine.api import apiproxy_stub_map + + is_appengine = True +except: + is_appengine = False + # The list comes from ffnet, the only multi-language site we support # at the time of writing. Values are taken largely from pycountry, @@ -90,7 +107,7 @@ class Story: self.replacements = [] self.chapters = [] # chapters will be tuples of (title,html) self.imgurls = [] - self.imgurldata = [] + self.imgtuples = [] self.listables = {} # some items (extratags, category, warnings & genres) are also kept as lists. def setMetadata(self, key, value): @@ -176,6 +193,12 @@ class Story: # pass fetch in from adapter in case we need the cookies collected # as well as it's a base_story class method. def addImgUrl(self,configurable,parenturl,url,fetch): + + # appengine (web version) isn't allowed to do images--just + # gets too big too fast and breaks things. + if is_appengine: + return + if url.startswith("http") : imgurl = url elif parenturl != None: @@ -220,21 +243,21 @@ class Story: prefix=self.getMetadataRaw('dateCreated').strftime("%Y%m%d%H%M%S") if imgurl not in self.imgurls: - self.imgurls.append(imgurl) parsedUrl = urlparse.urlparse(imgurl) - newsrc = "images/%s-%s.jpg"%( - prefix, - self.imgurls.index(imgurl)) sizes = [ int(x) for x in configurable.getConfigList('image_max_size') ] - data = convert_image(fetch(imgurl), - sizes, - configurable.getConfig('grayscale_images')) - print("\nimgurl:%s\nnewsrc:%s\nimage size:%d\n"%(imgurl,newsrc,len(data))) - self.imgurldata.append((newsrc,data)) - else: - newsrc = "images/%s-%s.jpg"%( + (data,ext,mime) = convert_image(imgurl, + fetch(imgurl), + sizes, + configurable.getConfig('grayscale_images')) + self.imgurls.append(imgurl) + newsrc = "images/%s-%s.%s"%( prefix, - self.imgurls.index(imgurl)) + self.imgurls.index(imgurl), + ext) + self.imgtuples.append({'newsrc':newsrc,'mime':mime,'data':data}) + print("\nimgurl:%s\nnewsrc:%s\nimage size:%d\n"%(imgurl,newsrc,len(data))) + else: + newsrc = self.imgtuples[self.imgurls.index(imgurl)]['newsrc'] #print("===============\n%s\nimg url:%s\n============"%(newsrc,self.imgurls[-1])) @@ -243,8 +266,8 @@ class Story: def getImgUrls(self): retlist = [] for i, url in enumerate(self.imgurls): - parsedUrl = urlparse.urlparse(url) - retlist.append(self.imgurldata[i]) + #parsedUrl = urlparse.urlparse(url) + retlist.append(self.imgtuples[i]) return retlist def __str__(self): diff --git a/fanficdownloader/writers/writer_epub.py b/fanficdownloader/writers/writer_epub.py index eabd8b1..49740fb 100644 --- a/fanficdownloader/writers/writer_epub.py +++ b/fanficdownloader/writers/writer_epub.py @@ -278,20 +278,13 @@ ${value}
itemrefs.append("file%04d"%i) if self.getConfig('include_images'): - #from calibre.utils.magick.draw import minify_image - imgcount=0 - sizes = [ int(x) for x in self.getConfigList('image_max_size') ] - for (newsrc,data) in self.story.getImgUrls(): - imgfile = "OEBPS/"+newsrc - # saveimg = minify_image(data, minify_to=sizes) - # if self.getConfig('grayscale_images'): - # saveimg.type = "GrayscaleType" - # outputepub.writestr(imgfile,saveimg.export('JPG')) - outputepub.writestr(imgfile,data) + for imgmap in self.story.getImgUrls(): + imgfile = "OEBPS/"+imgmap['newsrc'] + outputepub.writestr(imgfile,imgmap['data']) items.append(("image%04d"%imgcount, imgfile, - "image/jpeg", + imgmap['mime'], None)) imgcount+=1 diff --git a/main.py b/main.py index 50da665..e2d4ab4 100644 --- a/main.py +++ b/main.py @@ -339,7 +339,7 @@ class FanfictionDownloader(UserConfigServer): self.redirect("/?error=custom&errtext=%s"%urlEscape("There's an error in your User Configuration: "+str(e))) return - adapter = adapters.getAdapter(config,url) + adapter = adapters.getAdapter(config,url,format) logging.info('Created an adaper: %s' % adapter) if len(login) > 1: @@ -442,7 +442,7 @@ class FanfictionDownloaderTask(UserConfigServer): try: config = self.getUserConfig(user) - adapter = adapters.getAdapter(config,url) + adapter = adapters.getAdapter(config,url,format) logging.info('Created an adapter: %s' % adapter)