From 17a631dc79e0f9ba98eb02a342ea82e0a388b0be Mon Sep 17 00:00:00 2001 From: Jim Miller Date: Tue, 21 Apr 2015 10:44:49 -0500 Subject: [PATCH 1/8] More fix for storiesonline.net change --- fanficfare/adapters/adapter_storiesonlinenet.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/fanficfare/adapters/adapter_storiesonlinenet.py b/fanficfare/adapters/adapter_storiesonlinenet.py index b3d2346..a8da7c2 100644 --- a/fanficfare/adapters/adapter_storiesonlinenet.py +++ b/fanficfare/adapters/adapter_storiesonlinenet.py @@ -321,7 +321,10 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter): soup = bs.BeautifulSoup(self._fetchUrl(url), selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags. - div = soup.find('article', {'id' : 'story'}) + div = soup.find('div', {'id' : 'story'}) + if not div: + logger.debug("div id=story not found, try article") + div = soup.find('article', {'id' : 'story'}) # some big chapters are split over several pages pager = div.find('span', {'class' : 'pager'}) @@ -343,7 +346,10 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter): soup = bs.BeautifulSoup(self._fetchUrl("http://"+self.getSiteDomain()+ur['href']), selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags. - div1 = soup.find('article', {'id' : 'story'}) + div1 = soup.find('div', {'id' : 'story'}) + if not div1: + logger.debug("div id=story not found, try article") + div1 = soup.find('article', {'id' : 'story'}) # Find the "Continues" marker on the current page and remove everything after that. continues = div.find('span', {'class' : 'conTag'}) @@ -401,14 +407,17 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter): # For a story or the last chapter, remove voting form and the in library box a = div.find('div', {'id' : 'vote-form'}) + if a != None: + a.extract() + a = div.find('div', {'id' : 'top-header'}) if a != None: a.extract() a = div.find('div', {'id' : 'b-man-div'}) if a != None: a.extract() - # Kill the "The End" header and everything after it. - a = div.find(['h2', 'h3'], {'class' : 'end'}) + # Kill the vote form and everything after it. + a = div.find('div', {'class' : 'vform'}) logger.debug("Chapter end= '{0}'".format(a)) while a != None: b = a.nextSibling From b16aee27a19bc6a06a3288c3dbbb398d248336d5 Mon Sep 17 00:00:00 2001 From: Jim Miller Date: Tue, 21 Apr 2015 13:28:20 -0500 Subject: [PATCH 2/8] More fix for storiesonline.net change --- .../adapters/adapter_storiesonlinenet.py | 33 ++++++++++++------- 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/fanficfare/adapters/adapter_storiesonlinenet.py b/fanficfare/adapters/adapter_storiesonlinenet.py index a8da7c2..571b392 100644 --- a/fanficfare/adapters/adapter_storiesonlinenet.py +++ b/fanficfare/adapters/adapter_storiesonlinenet.py @@ -21,7 +21,7 @@ logger = logging.getLogger(__name__) import re import urllib2 -from .. import BeautifulSoup as bs +#from .. import BeautifulSoup as bs from ..htmlcleanup import stripHTML from .. import exceptions as exceptions @@ -144,7 +144,7 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter): raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Error! The story you're trying to access is being filtered by your choice of contents filtering.") # use BeautifulSoup HTML parser to make everything easier to find. - soup = bs.BeautifulSoup(data) + soup = self.make_soup(data) #print data # Now go hunting for all the meta data and the chapter list. @@ -178,7 +178,7 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter): page=0 i=0 while i == 0: - asoup = bs.BeautifulSoup(self._fetchUrl(self.story.getList('authorUrl')[0]+"/"+str(page))) + asoup = self.make_soup(self._fetchUrl(self.story.getList('authorUrl')[0]+"/"+str(page))) a = asoup.findAll('td', {'class' : 'lc2'}) for lc2 in a: @@ -212,7 +212,7 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter): self.story.setMetadata('seriesUrl',seriesUrl) series_name = stripHTML(a) logger.debug("Series name= %s" % series_name) - series_soup = bs.BeautifulSoup(self._fetchUrl(seriesUrl)) + series_soup = self.make_soup(self._fetchUrl(seriesUrl)) if series_soup: logger.debug("Retrieving Series - looking for name") series_name = series_soup.find('span', {'id' : 'ptitle'}).text.partition(' — ')[0] @@ -221,7 +221,7 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter): desc = lc4.contents[2] # Check if series is in a universe universe_url = self.story.getList('authorUrl')[0] + "&type=uni" - universes_soup = bs.BeautifulSoup(self._fetchUrl(universe_url) ) + universes_soup = self.make_soup(self._fetchUrl(universe_url) ) logger.debug("Universe url='{0}'".format(universe_url)) if universes_soup: universes = universes_soup.findAll('div', {'class' : 'ser-box'}) @@ -255,7 +255,7 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter): universe_name = stripHTML(a) universeUrl = 'http://'+self.host+a['href'] logger.debug("Retrieving Universe - about to get page - universeUrl='{0}".format(universeUrl)) - universe_soup = bs.BeautifulSoup(self._fetchUrl(universeUrl)) + universe_soup = self.make_soup(self._fetchUrl(universeUrl)) logger.debug("Retrieving Universe - have page") if universe_soup: logger.debug("Retrieving Universe - looking for name") @@ -318,8 +318,7 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter): logger.debug('Getting chapter text from: %s' % url) - soup = bs.BeautifulSoup(self._fetchUrl(url), - selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags. + soup = self.make_soup(self._fetchUrl(url)) div = soup.find('div', {'id' : 'story'}) if not div: @@ -343,8 +342,7 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter): # logger.debug(div) for ur in urls: - soup = bs.BeautifulSoup(self._fetchUrl("http://"+self.getSiteDomain()+ur['href']), - selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags. + soup = self.make_soup(self._fetchUrl("http://"+self.getSiteDomain()+ur['href'])) div1 = soup.find('div', {'id' : 'story'}) if not div1: @@ -374,7 +372,7 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter): if endPager != None: b = endPager.nextSibling while endPager != None: - logger.debug("removing end: {0}".format(endPager)) +# logger.debug("removing end: {0}".format(endPager)) b = endPager.nextSibling endPager.extract() endPager = b @@ -418,13 +416,24 @@ class StoriesOnlineNetAdapter(BaseSiteAdapter): # Kill the vote form and everything after it. a = div.find('div', {'class' : 'vform'}) - logger.debug("Chapter end= '{0}'".format(a)) +# logger.debug("Chapter end= '{0}'".format(a)) while a != None: b = a.nextSibling a.extract() a=b + # Kill the vote form and everything after it. + a = div.find('h3', {'class' : 'end'}) +# logger.debug("Chapter end= '{0}'".format(a)) + while a != None: + b = a.nextSibling + a.extract() + a=b + foot = div.find('footer') + if foot != None: + foot.extract() + if None == div: raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) From e07bd0653b30d7b31bbf80e5d6525c8f543a8215 Mon Sep 17 00:00:00 2001 From: Jim Miller Date: Wed, 22 Apr 2015 14:03:58 -0500 Subject: [PATCH 3/8] Add to FimF cover_exclusion_regexp and smaller cover when href not found. --- calibre-plugin/plugin-defaults.ini | 6 ++++++ fanficfare/adapters/adapter_fimfictionnet.py | 5 ++++- fanficfare/adapters/base_adapter.py | 4 ++-- fanficfare/defaults.ini | 4 ++++ fanficfare/story.py | 2 +- 5 files changed, 17 insertions(+), 4 deletions(-) diff --git a/calibre-plugin/plugin-defaults.ini b/calibre-plugin/plugin-defaults.ini index d49a1c6..2104e94 100644 --- a/calibre-plugin/plugin-defaults.ini +++ b/calibre-plugin/plugin-defaults.ini @@ -1471,6 +1471,8 @@ extra_titlepage_entries:readings,awards awards_label:Awards readings_label:Readings +## some sites include images that we don't ever want becoming the +## cover image. This lets you exclude them. cover_exclusion_regexp:art/.*Awards.jpg [voracity2.e-fic.com] @@ -1667,6 +1669,10 @@ dateUpdated_format:%%Y-%%m-%%d %%H:%%M:%%S ## to on, but can be switched off if it is found to cause problems. fix_fimf_blockquotes:true +## some sites include images that we don't ever want becoming the +## cover image. This lets you exclude them. +cover_exclusion_regexp:/images/emoticons/ + ## Site dedicated to these categories/characters/ships extracategories:My Little Pony: Friendship is Magic diff --git a/fanficfare/adapters/adapter_fimfictionnet.py b/fanficfare/adapters/adapter_fimfictionnet.py index fd55ceb..cd42ebc 100644 --- a/fanficfare/adapters/adapter_fimfictionnet.py +++ b/fanficfare/adapters/adapter_fimfictionnet.py @@ -176,7 +176,10 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter): if coverurl.startswith('//'): # fix for img urls missing 'http:' coverurl = "http:"+coverurl if get_cover: - self.setCoverImage(self.url,coverurl) + # try setting from href, if fails, try using the img src + if self.setCoverImage(self.url,coverurl)[0] == "failedtoload": + coverurl = storyImage.find('img')['src'] + self.setCoverImage(self.url,coverurl) coverSource = storyImage.find('a', {'class':'source'}) if coverSource: diff --git a/fanficfare/adapters/base_adapter.py b/fanficfare/adapters/base_adapter.py index e7ae20b..1dbb3df 100644 --- a/fanficfare/adapters/base_adapter.py +++ b/fanficfare/adapters/base_adapter.py @@ -479,8 +479,8 @@ class BaseSiteAdapter(Configurable): def setCoverImage(self,storyurl,imgurl): if self.getConfig('include_images'): - self.story.addImgUrl(storyurl,imgurl,self._fetchUrlRaw,cover=True, - coverexclusion=self.getConfig('cover_exclusion_regexp')) + return self.story.addImgUrl(storyurl,imgurl,self._fetchUrlRaw,cover=True, + coverexclusion=self.getConfig('cover_exclusion_regexp')) # bs3 & bs4 are different here. # will move to a bs3 vs bs4 block if there's lots of changes. diff --git a/fanficfare/defaults.ini b/fanficfare/defaults.ini index dbd26a7..b6294dc 100644 --- a/fanficfare/defaults.ini +++ b/fanficfare/defaults.ini @@ -1656,6 +1656,10 @@ dateUpdated_format:%%Y-%%m-%%d %%H:%%M:%%S ## to on, but can be switched off if it is found to cause problems. fix_fimf_blockquotes:true +## some sites include images that we don't ever want becoming the +## cover image. This lets you exclude them. +cover_exclusion_regexp:/images/emoticons/ + ## Site dedicated to these categories/characters/ships extracategories:My Little Pony: Friendship is Magic diff --git a/fanficfare/story.py b/fanficfare/story.py index 9d78fd8..a0674d4 100644 --- a/fanficfare/story.py +++ b/fanficfare/story.py @@ -51,7 +51,7 @@ try: export = False img = Image() img.load(data) - + owidth, oheight = img.size nwidth, nheight = sizes scaled, nwidth, nheight = fit_image(owidth, oheight, nwidth, nheight) From f51b21cfeb06adad906c63026e2c85e079bd476f Mon Sep 17 00:00:00 2001 From: Jim Miller Date: Wed, 22 Apr 2015 14:04:25 -0500 Subject: [PATCH 4/8] Add to FimF cover_exclusion_regexp and smaller cover when href not found. --- fanficfare/story.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fanficfare/story.py b/fanficfare/story.py index a0674d4..9d78fd8 100644 --- a/fanficfare/story.py +++ b/fanficfare/story.py @@ -51,7 +51,7 @@ try: export = False img = Image() img.load(data) - + owidth, oheight = img.size nwidth, nheight = sizes scaled, nwidth, nheight = fit_image(owidth, oheight, nwidth, nheight) From dda836004465aa77f07941f7fd7cc2218d68dd8c Mon Sep 17 00:00:00 2001 From: Jim Miller Date: Wed, 22 Apr 2015 15:40:46 -0500 Subject: [PATCH 5/8] Improve error reporting when only one URL vs list. --- fanficfare/cli.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/fanficfare/cli.py b/fanficfare/cli.py index 5613e21..b4cb450 100644 --- a/fanficfare/cli.py +++ b/fanficfare/cli.py @@ -144,14 +144,20 @@ def main(argv=None, parser=None, passed_defaultsini=None, passed_personalini=Non else: urls = args - for url in urls: - try: - do_download(url, - options, - passed_defaultsini, - passed_personalini) - except Exception, e: - print "URL(%s) Failed: Exception (%s). Run URL individually for more detail."%(url,e) + if len(urls) > 1: + for url in urls: + try: + do_download(url, + options, + passed_defaultsini, + passed_personalini) + except Exception, e: + print "URL(%s) Failed: Exception (%s). Run URL individually for more detail."%(url,e) + else: + do_download(urls[0], + options, + passed_defaultsini, + passed_personalini) # make rest a function and loop on it. def do_download(arg, From 53ca251a49d05795e25fa5d9e2531536dbed29c0 Mon Sep 17 00:00:00 2001 From: Jim Miller Date: Wed, 22 Apr 2015 15:48:24 -0500 Subject: [PATCH 6/8] Add to FimF cover_exclusion_regexp and smaller cover when href not found. --- fanficfare/adapters/base_adapter.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fanficfare/adapters/base_adapter.py b/fanficfare/adapters/base_adapter.py index 1dbb3df..46c3a62 100644 --- a/fanficfare/adapters/base_adapter.py +++ b/fanficfare/adapters/base_adapter.py @@ -481,6 +481,8 @@ class BaseSiteAdapter(Configurable): if self.getConfig('include_images'): return self.story.addImgUrl(storyurl,imgurl,self._fetchUrlRaw,cover=True, coverexclusion=self.getConfig('cover_exclusion_regexp')) + else: + return (None,None) # bs3 & bs4 are different here. # will move to a bs3 vs bs4 block if there's lots of changes. From ca6bc7f88edbc4fb5993827589ece23a3d94469d Mon Sep 17 00:00:00 2001 From: Jim Miller Date: Wed, 22 Apr 2015 16:50:56 -0500 Subject: [PATCH 7/8] Detect and reject animated gifs in Calibre because they come out so poorly. --- fanficfare/exceptions.py | 7 + fanficfare/story.py | 32 ++-- included_dependencies/gif.py | 283 +++++++++++++++++++++++++++++++++++ makeplugin.py | 2 +- 4 files changed, 310 insertions(+), 14 deletions(-) create mode 100644 included_dependencies/gif.py diff --git a/fanficfare/exceptions.py b/fanficfare/exceptions.py index 876711d..c68b22a 100644 --- a/fanficfare/exceptions.py +++ b/fanficfare/exceptions.py @@ -24,6 +24,13 @@ class FailedToDownload(Exception): def __str__(self): return self.error +class RejectImage(Exception): + def __init__(self,error): + self.error=error + + def __str__(self): + return self.error + class InvalidStoryURL(Exception): def __init__(self,url,domain,example): self.url=url diff --git a/fanficfare/story.py b/fanficfare/story.py index 9d78fd8..84f3589 100644 --- a/fanficfare/story.py +++ b/fanficfare/story.py @@ -44,6 +44,8 @@ imagetypes = { try: from calibre.utils.magick import Image + from StringIO import StringIO + from gif import GifInfo, CHECK_IS_ANIMATED convtype = {'jpg':'JPG', 'png':'PNG'} def convert_image(url,data,sizes,grayscale, @@ -55,6 +57,10 @@ try: owidth, oheight = img.size nwidth, nheight = sizes scaled, nwidth, nheight = fit_image(owidth, oheight, nwidth, nheight) + + if normalize_format_name(img.format)=="gif" and GifInfo(StringIO(data),CHECK_IS_ANIMATED).frameCount > 1: + raise exceptions.RejectImage("Animated gifs come out purely--not going to use it.") + if scaled: img.size = (nwidth, nheight) export = True @@ -224,7 +230,7 @@ langs = { "Devanagari":"hi", ## These are from/for AO3: - + u'العربية':'ar', u'беларуская':'be', u'Български език':'bg', @@ -325,7 +331,7 @@ class InExMatch: else: retval = self.match == value #print(">>>>>>>>>>>>>%s==%s r: %s,%s=%s"%(self.match,value,self.negate,retval, self.negate != retval)) - + return self.negate != retval def __str__(self): @@ -338,7 +344,7 @@ class InExMatch: else: s='=' return u'InExMatch(%s %s%s %s)'%(self.keys,f,s,self.match) - + ## metakey[,metakey]=~pattern ## metakey[,metakey]==string ## *for* part lines. Effect only when trailing conditional key=~regexp matches @@ -358,7 +364,7 @@ def set_in_ex_clude(setting): match = InExMatch(line) dest.append([match,condmatch]) return dest - + ## Two or three part lines. Two part effect everything. ## Three part effect only those key(s) lists. ## pattern=>replacement @@ -433,7 +439,7 @@ class Story(Configurable): def join_list(self, key, vallist): return self.getConfig("join_string_"+key,u", ").replace(SPACE_REPLACE,' ').join(map(unicode, vallist)) - + def setMetadata(self, key, value, condremoveentities=True): # keep as list type, but set as only value. @@ -445,20 +451,20 @@ class Story(Configurable): self.metadata[key]=conditionalRemoveEntities(value) else: self.metadata[key]=value - + if key == "language": try: # getMetadata not just self.metadata[] to do replace_metadata. self.setMetadata('langcode',langs[self.getMetadata(key)]) except: self.setMetadata('langcode','en') - + if key == 'dateUpdated' and value: # Last Update tags for Bill. self.addToList('lastupdate',value.strftime("Last Update Year/Month: %Y/%m")) self.addToList('lastupdate',value.strftime("Last Update: %Y/%m/%d")) - + def do_in_ex_clude(self,which,value,key): if value and which in self.in_ex_cludes: include = 'include' in which @@ -487,7 +493,7 @@ class Story(Configurable): if include and keyfound and not found: value = None return value - + def doReplacements(self,value,key,return_list=False,seen_list=[]): value = self.do_in_ex_clude('include_metadata_pre',value,key) @@ -526,7 +532,7 @@ class Story(Configurable): # print("replacement,value:%s,%s->%s"%(replacement,value,regexp.sub(replacement,value))) value = regexp.sub(replacement,value) retlist = [value] - + for val in retlist: retlist = map(partial(self.do_in_ex_clude,'include_metadata_post',key=key),retlist) retlist = map(partial(self.do_in_ex_clude,'exclude_metadata_post',key=key),retlist) @@ -610,7 +616,7 @@ class Story(Configurable): self.getMetadata('author', removeallentities, doreplacements))) self.extendList("extratags",self.getConfigList("extratags")) - + if self.getMetadataRaw('seriesUrl'): self.setMetadata('seriesHTML',linkhtml%('series',self.getMetadata('seriesUrl', removeallentities, doreplacements), self.getMetadata('series', removeallentities, doreplacements))) @@ -703,10 +709,10 @@ class Story(Configurable): for val in retlist: newretlist.extend(self.doReplacements(val,listname,return_list=True)) retlist = newretlist - + if removeallentities: retlist = map(removeAllEntities,retlist) - + retlist = filter( lambda x : x!=None and x!='' ,retlist) # reorder ships so b/a and c/b/a become a/b and a/b/c. Only on '/', diff --git a/included_dependencies/gif.py b/included_dependencies/gif.py new file mode 100644 index 0000000..406bab2 --- /dev/null +++ b/included_dependencies/gif.py @@ -0,0 +1,283 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +A pure Python GIF metadata extractor. +Supports adjustable detail to fine-tune performance. + +Example code and full epydoc docstrings included. + +Uses: + - Identifying whether a GIF is static or animated. + - Extracting the dimensions, pixel aspect ratio, number of frames, loop count, + global palette or palette size, and background color. + - Extracting comments and other plaintext. + - Testing for various structural errors. + +TODO: + - Provide basic support for XMP Metadata extraction + - http://en.wikipedia.org/wiki/Extensible_Metadata_Platform#Location_in_file_types + - http://www.matthewflickinger.com/lab/whatsinagif/bits_and_bytes.asp + - Generate test GIF with http://code.google.com/p/python-xmp-toolkit/ + +Changelog: + - 0.2.2: Audited the code and made some corrections. + - 0.2.1: 40% speed improvement (went from 15 to 9 seconds for 1000 images) + - 0.2.0: Feature-complete + - 0.1.0: Initial release +""" + +__appname__ = "gif.py" +__author__ = "Stephan Sokolow (deitarion/SSokolow)" +__version__ = "0.2.2" +__license__ = "PSF License 2.4 or higher (The Python License)" + +#{ Check Types (enum, numerical ordering is significant) +CHECK_IS_GIF_FILE = 0 #: Just check for a valid GIF header. +CHECK_IS_ANIMATED = 1 #: Check whether the file has more than one frame. +CHECK_COUNT_FRAMES = 2 #: Count the number of frames in the file. +CHECK_PARSE_PALETTE = 3 #: Parse the palette and resolve the background color. +CHECK_READ_COMMENTS = 4 #: Load comments (can sometimes be large) into the L{GifInfo} object. +CHECK_READ_ALL_TEXT = 5 #: Also load the contents of Plain Text extension blocks. + +CHECK_ALL = CHECK_READ_ALL_TEXT #: alias to allow for future modifications + +#{ Warning Codes (bitfield) +WARN_NONE = 0 #: No warnings +WARN_BAD_IMG = 1 #: Corruption (of the [sub]block size field(s)) or truncation detected in an image block +WARN_BAD_EXT = 2 #: Corruption (of the [sub]block size field(s)) or truncation detected in an extension block +WARN_BAD_SIZE = 4 #: An image block specifies dimensions exceeding the global canvas size +WARN_BAD_BGCOLOR = 8 #: The background color index specified is greater than the palette size +WARN_EOF = 16 #: File is missing it's trailer. (Corrupt elsewhere, truncated, or breaking spec by using EOF as the terminator.) +WARN_TRUNC = 32 #: File is definitively either truncated or corrupt. (An EOF was encountered part-way through a structure.) +WARN_LOOP_POS = 64 #: Netscape Application Extension block (animation-control) was present but not first in the file. +#} + +import struct + +#{ Structures used by GifInfo +gifHeaderStruct = struct.Struct('= CHECK_PARSE_PALETTE: + self.palette = [] + for pos in range(0, self.paletteSize): + self.palette.append(gifColorTripleStruct.unpack_from(rawPalette, pos * 3)) + + if self.paletteSize and bgColor > self.paletteSize: + self.warnFlags = self.warnFlags | WARN_BAD_BGCOLOR + elif self.palette: + self.bgColor = self.palette[bgColor] + + # Iterate blocks + self.firstBlock = True + blocktype = self._read(fh, 1) + while not blocktype == chr(0x3B) and not self.warnFlags & WARN_EOF: + self._blockHandlers.get(blocktype, lambda x, y:'')(self, fh) + if self.checkLevel <= CHECK_IS_ANIMATED and self.frameCount > 1: + return + + self.firstBlock = False + blocktype = self._read(fh, 1) + + del self.firstBlock + + def _handleImageBlock(self, fh): + """""" + self.frameCount += 1 + try: + x, y, w, h, LCTF_Byte = gifImageStruct.unpack(self._read(fh, gifImageStruct.size)) + except: + self.warnFlags = self.warnFlags | WARN_EOF | WARN_TRUNC + return + + if x + w > self.width or y + h > self.height: + self.warnFlags = self.warnFlags | WARN_BAD_SIZE + + self._getPalette(fh, LCTF_Byte) # Skip the local color table if present + fh.read(1) # Skip the LZW minimum code size. + + # Skip content and test for the block terminator + if not self._skipSubBlocks(fh): # For example, if it's a zero-length string like EOF would return. + self.warnFlags = self.warnFlags | WARN_BAD_IMG + + def _handleGenericExtensionBlock(self, fh): + """@todo: Rewrite this so extension block types have method handlers.""" + try: + extType, blkSize = gifExtenStruct.unpack(self._read(fh, gifExtenStruct.size)) + except: + self.warnFlags = self.warnFlags | WARN_EOF | WARN_TRUNC + return + startOffset = fh.tell() + + if extType == 0x01 and self.checkLevel >= CHECK_READ_ALL_TEXT: # Plain Text Block + self._read(fh, gifPlaintextStruct.size) + self.otherText = self.otherText or [] + blkSize = self._read(fh, 1) + while blkSize and blkSize != '\x00': + self.otherText.append(self._read(fh, ord(blkSize))) + blkSize = self._read(fh, 1) + elif extType == 0xFE and self.checkLevel >= CHECK_READ_COMMENTS: # Comment Block + self.comments = self.comments or [] + blkSize = self._read(fh, 1) + while blkSize and blkSize != '\x00': + self.comments.append(self._read(fh, ord(blkSize))) + blkSize = self._read(fh, 1) + elif extType == 0xFF: # Application Block + if blkSize == 0x0B and self._read(fh, blkSize) == "NETSCAPE2.0": + try: + a, b, self.loopCount = gifNetscapeStruct.unpack(self._read(fh, gifNetscapeStruct.size)) + except: + self.warnFlags = self.warnFlags | WARN_EOF | WARN_TRUNC + return + + if a != 3 and b != 1: + self.warnFlags = self.warnFlags | WARN_BAD_EXT + + if not self.firstBlock: + self.warnFlags = self.warnFlags | WARN_LOOP_POS + else: + fh.seek( startOffset + blkSize ) # Skip the contents + + # Test for the block terminator + if not self._skipSubBlocks(fh): + self.warnFlags = self.warnFlags | WARN_BAD_EXT + + def _getPalette(self, handle, bitfield): + """Using the size value from C{bitfield}, + load the palette at C{handle}'s current file pointer position.""" + if bitfield & int("10000000", 2): + nBits = bitfield & int("00000111", 2) + tableSize = 3 * 2**( nBits + 1 ) + return handle.read(tableSize) + else: + return '' + + def _read(self, handle, size): + """Attempt to read the specified number of bytes. Set L{WARN_EOF} if + fewer are received.""" + content = handle.read(size) + if len(content) < size: + self.warnFlags = self.warnFlags | WARN_EOF + return content + + def _skipSubBlocks(self, handle): + """Skip sub-blocks beginning at the current file pointer position + using fseek.""" + offset = handle.tell() + blkSize = handle.read(1) + while blkSize and blkSize != '\x00': + offset += ord(blkSize) + 1 + handle.seek(offset) + blkSize = handle.read(1) + return blkSize + + _blockHandlers = { + chr(0x2C) : _handleImageBlock, + chr(0x21) : _handleGenericExtensionBlock, + } + +def gif_is_animated(path): + """A simple convenience function for testing whether a GIF is animated. + @rtype: C{bool} + """ + return GifInfo(file(path,'rb'), CHECK_IS_ANIMATED).frameCount > 1 + +if __name__ == '__main__': + from optparse import OptionParser + parser = OptionParser(description=__doc__.split('\n\n')[0], + version="%%prog v%s" % __version__, usage="%prog ...") + + opts, args = parser.parse_args() + + if args: + for fpath in args: + try: + info = GifInfo(fpath, CHECK_COUNT_FRAMES) + warnFlags = ( + (info.warnFlags & WARN_BAD_IMG and 'I' or ' ') + + (info.warnFlags & WARN_BAD_EXT and 'X' or ' ') + + (info.warnFlags & WARN_BAD_SIZE and 'C' or ' ') + + (info.warnFlags & WARN_BAD_BGCOLOR and 'B' or ' ') + + (info.warnFlags & WARN_EOF and 'E' or ' ') + + (info.warnFlags & WARN_TRUNC and 'T' or ' ') + + (info.warnFlags & WARN_LOOP_POS and 'L' or ' ') + ) + print "[%s](%3s Frames): %s" % (warnFlags, info.frameCount, info.path) + except BadHeaderException, err: + print "%s: %s" % (str(err), fpath) + print "\nWarning Flags:" + print " I = Image Chunk Corruption/Truncation" + print " X = Extension Chunk Corruption/Truncation" + print " C = Image Chunk Dimensions Exceed Global Canvas" + print " B = Bad Background Color (Index Exceeds Palette Size)" + print " E = Unexpected EOF Encountered (Missing Image Terminator)" + print " T = EOF Encountered Within A Block Header (Corrupt or Truncated File)" + print " L = Loop-control block misplaced within the file" + print + print "Note: A nearly-threefold speed-up can be had by using CHECK_IS_ANIMATED rather than CHECK_COUNT_FRAMES" diff --git a/makeplugin.py b/makeplugin.py index 56c8efb..9350d6a 100644 --- a/makeplugin.py +++ b/makeplugin.py @@ -36,7 +36,7 @@ if __name__=="__main__": os.chdir('../included_dependencies') # 'a' for append - files=['six.py','bs4','html5lib','chardet'] + files=['gif.py','six.py','bs4','html5lib','chardet'] createZipFile("../"+filename,"a", files, exclude=exclude) From 4980dd802795bc3aa23023ee997645ddcd34e663 Mon Sep 17 00:00:00 2001 From: Jim Miller Date: Wed, 22 Apr 2015 17:23:54 -0500 Subject: [PATCH 8/8] Fix Spanish(es) %(pini)s--fixed in Transifex --- calibre-plugin/translations/es.po | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/calibre-plugin/translations/es.po b/calibre-plugin/translations/es.po index c61f17f..57bd5ce 100644 --- a/calibre-plugin/translations/es.po +++ b/calibre-plugin/translations/es.po @@ -5,12 +5,13 @@ # Adolfo Jayme Barrientos, 2014 # dario hereñu , 2015 # Jellby , 2014-2015 +# JimmXinu , 2015 msgid "" msgstr "" "Project-Id-Version: calibre-plugins\n" "POT-Creation-Date: 2015-04-11 11:17+Central Daylight Time\n" -"PO-Revision-Date: 2015-04-17 13:02+0000\n" -"Last-Translator: Jellby \n" +"PO-Revision-Date: 2015-04-22 22:08+0000\n" +"Last-Translator: JimmXinu \n" "Language-Team: Spanish (http://www.transifex.com/projects/p/calibre-plugins/language/es/)\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" @@ -558,7 +559,7 @@ msgid "" "The %(pini)s parameter %(gcset)s allows you to choose a GC setting based on " "metadata rather than site, but it's much more complex.
%(gcset)s is " "ignored when this is off." -msgstr "El parámetro %(gcset)s de %(pini) le permite elegir una configuración de GC según los metadatos en vez del sitio, pero es mucho más complejo.
%(gcset)s no se tiene en cuenta si esta opción está desactivada." +msgstr "El parámetro %(gcset)s de %(pini)s le permite elegir una configuración de GC según los metadatos en vez del sitio, pero es mucho más complejo.
%(gcset)s no se tiene en cuenta si esta opción está desactivada." #: config.py:811 msgid "Use calibre's Polish feature to inject/update the cover" @@ -794,7 +795,7 @@ msgid "" "The %(pini)s parameter %(ccset)s allows you to set custom columns to site " "specific values that aren't common to all sites.
%(ccset)s is ignored " "when this is off." -msgstr "El parámetro %(ccset)s de %(pini) le permite asignar a las columnas personalizadas valores específicos para cada sitio.
%(ccset)s no se tiene en cuenta si esta opción está desactivada." +msgstr "El parámetro %(ccset)s de %(pini)s le permite asignar a las columnas personalizadas valores específicos para cada sitio.
%(ccset)s no se tiene en cuenta si esta opción está desactivada." #: config.py:1065 msgid "Special column:"