From 30bafd4e53ba19e0987a4b6fc9a4a11469bff553 Mon Sep 17 00:00:00 2001 From: Jim Miller Date: Mon, 29 Feb 2016 22:49:28 -0600 Subject: [PATCH] Add processed metadata caching, more base_xenforoforum ini features. --- calibre-plugin/plugin-defaults.ini | 27 ++++++++ fanficfare/adapters/base_adapter.py | 3 - .../adapters/base_xenforoforum_adapter.py | 42 +++++++----- fanficfare/configurable.py | 66 ++++++++++++------- fanficfare/defaults.ini | 27 ++++++++ fanficfare/story.py | 32 +++++++-- 6 files changed, 149 insertions(+), 48 deletions(-) diff --git a/calibre-plugin/plugin-defaults.ini b/calibre-plugin/plugin-defaults.ini index 37aa332..466dd82 100644 --- a/calibre-plugin/plugin-defaults.ini +++ b/calibre-plugin/plugin-defaults.ini @@ -488,6 +488,33 @@ description_limit:500 ## in the ebook for that chapter. continue_on_chapter_error:false +## When given a thread URL, use threadmarks as chapter links when +## there are at least this many threadmarks. A number of older +## threads have a single threadmark to an 'index' post. Set to 1 to +## use threadmarks whenever they exist. +minimum_threadmarks:2 + +## When 'first post' (or post URL) is being added as a chapter, give +## the chapter this title. +first_post_title:First Post + +## In normal operation, if given a post URL or a thread URL with less +## than minimum_threadmarks, the given post or the first post of the +## thread will be included as the first chapter (with chapter title +## from first_post_title) unless that post is explicitly linked to in +## the collected chapter list. First post is not included when using +## thread marks. +## +## If always_include_first_post:true, then the given or first post +## will be included as above even if it is a link in the post or even +## if threadmarks are used. Can result in a duplicated chapter. +always_include_first_post:false + +## In normal operation, forumtags will only be populated when +## threadmarks are used for chapters (see minimum_threadmarks above). +## When always_use_forumtags:true, always populate forumtags. +always_use_forumtags:false + ## Each output format has a section that overrides [defaults] [html] diff --git a/fanficfare/adapters/base_adapter.py b/fanficfare/adapters/base_adapter.py index 70dc1d5..a1dc29e 100644 --- a/fanficfare/adapters/base_adapter.py +++ b/fanficfare/adapters/base_adapter.py @@ -188,9 +188,6 @@ class BaseSiteAdapter(Configurable): self.host = self.parsedUrl.netloc self.path = self.parsedUrl.path self.story.setMetadata('storyUrl',self.url,condremoveentities=False) - self.addUrlConfigSection(url) # self.story shares the same configuration. - # ignored inside if config is_lightweight() - self.story.config_prepped = False ## website encoding(s)--in theory, each website reports the character ## encoding they use for each page. In practice, some sites report it diff --git a/fanficfare/adapters/base_xenforoforum_adapter.py b/fanficfare/adapters/base_xenforoforum_adapter.py index 63efe56..fe38a80 100644 --- a/fanficfare/adapters/base_xenforoforum_adapter.py +++ b/fanficfare/adapters/base_xenforoforum_adapter.py @@ -49,9 +49,9 @@ class BaseXenForoForumAdapter(BaseSiteAdapter): m = re.match(self.getSiteURLPattern(),url) if m: #logger.debug("groupdict:%s"%m.groupdict()) - if m.group('post'): - self.story.setMetadata('storyId',m.group('post')) - self._setURL(self.getURLPrefix() + '/posts/'+m.group('post')+'/') + if m.group('anchorpost'): + self.story.setMetadata('storyId',m.group('anchorpost')) + self._setURL(self.getURLPrefix() + '/posts/'+m.group('anchorpost')+'/') else: self.story.setMetadata('storyId',m.group('id')) # normalized story URL. @@ -83,7 +83,7 @@ class BaseXenForoForumAdapter(BaseSiteAdapter): return cls.getURLPrefix()+"/threads/some-story-name.123456/ "+cls.getURLPrefix()+"/posts/123456/" def getSiteURLPattern(self): - return r"https?://"+re.escape(self.getSiteDomain())+r"/(?Pthreads|posts)/(.+\.)?(?P\d+)/?[^#]*?(#post-(?P\d+))?$" + return r"https?://"+re.escape(self.getSiteDomain())+r"/(?Pthreads|posts)/(.+\.)?(?P\d+)/?[^#]*?(#post-(?P\d+))?$" def use_pagecache(self): ''' @@ -150,7 +150,7 @@ class BaseXenForoForumAdapter(BaseSiteAdapter): raise # use BeautifulSoup HTML parser to make everything easier to find. - soup = self.make_soup(data) + topsoup = soup = self.make_soup(data) a = soup.find('h3',{'class':'userText'}).find('a') self.story.addToList('authorId',a['href'].split('/')[1]) @@ -160,6 +160,9 @@ class BaseXenForoForumAdapter(BaseSiteAdapter): h1 = soup.find('div',{'class':'titleBar'}).h1 self.story.setMetadata('title',stripHTML(h1)) + first_post_title = self.getConfig('first_post_title','First Post') + + threadmark_chaps = False if '#' in useurl: anchorid = useurl.split('#')[1] soup = soup.find('li',id=anchorid) @@ -176,7 +179,11 @@ class BaseXenForoForumAdapter(BaseSiteAdapter): ## SV changed their threadmarks. Not isolated to ## SV only incase SB or QQ make the same change. markas = soupmarks.find('div',{'class':'threadmarks'}).find_all('a',{'class':'PreviewTooltip'}) - if len(markas) > 1: + if len(markas) >= int(self.getConfig('minimum_threadmarks',2)): + threadmark_chaps = True + if self.getConfig('always_include_first_post'): + self.chapterUrls.append((first_post_title,useurl)) + for (atag,url,name) in [ (x,x['href'],stripHTML(x)) for x in markas ]: date = self.make_date(atag.find_next_sibling('div',{'class':'extra'})) if not self.story.getMetadataRaw('datePublished') or date < self.story.getMetadataRaw('datePublished'): @@ -186,16 +193,16 @@ class BaseXenForoForumAdapter(BaseSiteAdapter): self.chapterUrls.append((name,self.getURLPrefix()+'/'+url)) - ## only use tags if threadmarks for chapters. - ## a bit arbitrary, but likely. - for tag in soup.findAll('a',{'class':'tag'}): - tstr = stripHTML(tag) - if self.getConfig('capitalize_forumtags'): - tstr = tstr.title() - self.story.addToList('forumtags',tstr) - soup = soup.find('li',{'class':'message'}) # limit first post for date stuff below. ('#' posts above) + if threadmark_chaps or self.getConfig('always_use_forumtags'): + ## only use tags if threadmarks for chapters or + for tag in topsoup.findAll('a',{'class':'tag'}): + tstr = stripHTML(tag) + if self.getConfig('capitalize_forumtags'): + tstr = tstr.title() + self.story.addToList('forumtags',tstr) + # Now go hunting for the 'chapter list'. bq = soup.find('blockquote') # assume first posting contains TOC urls. @@ -211,8 +218,9 @@ class BaseXenForoForumAdapter(BaseSiteAdapter): # otherwise, use first post links--include first post since # that's often also the first chapter. + if not self.chapterUrls: - self.chapterUrls.append(("First Post",useurl)) + self.chapterUrls.append((first_post_title,useurl)) for (url,name) in [ (x['href'],stripHTML(x)) for x in bq.find_all('a') ]: #logger.debug("found chapurl:%s"%url) if not url.startswith('http'): @@ -230,9 +238,9 @@ class BaseXenForoForumAdapter(BaseSiteAdapter): logger.debug("(ch:%s)used chapurl:%s"%(len(self.chapterUrls)+1,url)) self.chapterUrls.append((name,url)) - if url == useurl and 'First Post' == self.chapterUrls[0][0]: + if url == useurl and first_post_title == self.chapterUrls[0][0] \ + and not self.getConfig('always_include_first_post',False): # remove "First Post" if included in list. - logger.debug("delete dup 'First Post' chapter: %s %s"%self.chapterUrls[0]) del self.chapterUrls[0] # Didn't use threadmarks, so take created/updated dates diff --git a/fanficfare/configurable.py b/fanficfare/configurable.py index ca25375..d684b38 100644 --- a/fanficfare/configurable.py +++ b/fanficfare/configurable.py @@ -111,8 +111,6 @@ def get_valid_list_entries(): 'authorId', 'authorUrl', 'lastupdate', - 'keep_html_attrs', - 'replace_tags_with_spans', ]) boollist=['true','false'] @@ -125,6 +123,10 @@ def get_valid_set_options(): ''' dict() of names of boolean options, but as a tuple with valid sites, valid formats and valid values (None==all) + + This is to further restrict keywords to certain sections and/or + values. get_valid_keywords() below is the list of allowed + keywords. Any keyword listed here must also be listed there. ''' valdict = {'collect_series':(None,None,boollist), @@ -177,7 +179,13 @@ def get_valid_set_options(): 'grayscale_images':(None,['epub','html'],boollist), 'no_image_processing':(None,['epub','html'],boollist), + 'capitalize_forumtags':(base_xenforo_list,None,boollist), 'continue_on_chapter_error':(base_xenforo_list,None,boollist), + 'minimum_threadmarks':(base_xenforo_list,None,None), + 'first_post_title':(base_xenforo_list,None,None), + 'always_include_first_post':(base_xenforo_list,None,boollist), + '':(base_xenforo_list,None,boollist), + '':(base_xenforo_list,None,boollist), '':(base_xenforo_list,None,boollist), } @@ -339,8 +347,12 @@ def get_valid_keywords(): 'wrap_width', 'zip_filename', 'zip_output', - 'continue_on_chapter_error', 'capitalize_forumtags', + 'continue_on_chapter_error', + 'minimum_threadmarks', + 'first_post_title', + 'always_include_first_post', + '', ]) # *known* entry keywords -- or rather regexps for them. @@ -405,9 +417,16 @@ class Configuration(ConfigParser.SafeConfigParser): self.validEntries = get_valid_entries() + self.url_config_set = False + def addUrlConfigSection(self,url): if not self.lightweight: # don't need when just checking for normalized URL. - self.addConfigSection(url,'overrides') + # replace if already set once. + if self.url_config_set: + self.sectionslist[self.sectionslist.index('overrides')+1]=url + else: + self.addConfigSection(url,'overrides') + self.url_config_set=True def addConfigSection(self,section,before=None): if section not in self.sectionslist: # don't add if already present. @@ -616,7 +635,10 @@ class Configuration(ConfigParser.SafeConfigParser): def test_config(self): errors=[] - allowedsections_re = re.compile(r'^(teststory:(defaults|[0-9]+)|https?://.*)$') + ## too complicated right now to enforce + ## get_valid_set_options() warnings on teststory and + ## [storyUrl] sections. + allow_all_sections_re = re.compile(r'^(teststory:(defaults|[0-9]+)|https?://.*)$') allowedsections = get_valid_sections() clude_metadata_re = re.compile(r'(add_to_)?(in|ex)clude_metadata_(pre|post)') @@ -626,12 +648,13 @@ class Configuration(ConfigParser.SafeConfigParser): custom_columns_settings_re = re.compile(r'(add_to_)?custom_columns_settings') - generate_cover_settings_re = re.compile(r'(add_to_)?generate_cover_settings') - + generate_cover_settings_re = re.compile(r'(add_to_)?generate_cover_settings') + valdict = get_valid_set_options() for section in self.sections(): - if section not in allowedsections and not allowedsections_re.match(section): + allow_all_section = allow_all_sections_re.match(section) + if section not in allowedsections and not allow_all_section: errors.append((self.get_lineno(section),"Bad Section Name: [%s]"%section)) else: sitename = section.replace('www.','') @@ -669,26 +692,25 @@ class Configuration(ConfigParser.SafeConfigParser): # timeline=>#ccolumn,n # "FanFiction"=>#collection - def make_sections(x): - return '['+'], ['.join(x)+']' - if keyword in valdict: - (valsites,valformats,vals)=valdict[keyword] - if valsites != None and sitename != None and sitename not in valsites: - errors.append((self.get_lineno(section,keyword),"%s not valid in section [%s] -- only valid in %s sections."%(keyword,section,make_sections(valsites)))) - if valformats != None and formatname != None and formatname not in valformats: - errors.append((self.get_lineno(section,keyword),"%s not valid in section [%s] -- only valid in %s sections."%(keyword,section,make_sections(valformats)))) - if value not in vals: - errors.append((self.get_lineno(section,keyword),"%s not a valid value for %s"%(value,keyword))) + if not allow_all_section: + def make_sections(x): + return '['+'], ['.join(x)+']' + if keyword in valdict: + (valsites,valformats,vals)=valdict[keyword] + if valsites != None and sitename != None and sitename not in valsites: + errors.append((self.get_lineno(section,keyword),"%s not valid in section [%s] -- only valid in %s sections."%(keyword,section,make_sections(valsites)))) + if valformats != None and formatname != None and formatname not in valformats: + errors.append((self.get_lineno(section,keyword),"%s not valid in section [%s] -- only valid in %s sections."%(keyword,section,make_sections(valformats)))) + if vals != None and value not in vals: + errors.append((self.get_lineno(section,keyword),"%s not a valid value for %s"%(value,keyword))) - ## skipping output_filename_safepattern ## regex--not used with plugin and this isn't ## used with CLI/web yet. except Exception as e: - errors.append((self.get_lineno(section,keyword),"Error:%s in (%s:%s)"%(e,keyword,value))) - - + errors.append((self.get_lineno(section,keyword),"Error:%s in (%s:%s)"%(e,keyword,value))) + return errors # extended by adapter, writer and story for ease of calling configuration. diff --git a/fanficfare/defaults.ini b/fanficfare/defaults.ini index 5105473..48da55a 100644 --- a/fanficfare/defaults.ini +++ b/fanficfare/defaults.ini @@ -487,6 +487,33 @@ description_limit:500 ## in the ebook for that chapter. continue_on_chapter_error:false +## When given a thread URL, use threadmarks as chapter links when +## there are at least this many threadmarks. A number of older +## threads have a single threadmark to an 'index' post. Set to 1 to +## use threadmarks whenever they exist. +minimum_threadmarks:2 + +## When 'first post' (or post URL) is being added as a chapter, give +## the chapter this title. +first_post_title:First Post + +## In normal operation, if given a post URL or a thread URL with less +## than minimum_threadmarks, the given post or the first post of the +## thread will be included as the first chapter (with chapter title +## from first_post_title) unless that post is explicitly linked to in +## the collected chapter list. First post is not included when using +## thread marks. +## +## If always_include_first_post:true, then the given or first post +## will be included as above even if it is a link in the post or even +## if threadmarks are used. Can result in a duplicated chapter. +always_include_first_post:false + +## In normal operation, forumtags will only be populated when +## threadmarks are used for chapters (see minimum_threadmarks above). +## When always_use_forumtags:true, always populate forumtags. +always_use_forumtags:false + ## Each output format has a section that overrides [defaults] [html] diff --git a/fanficfare/story.py b/fanficfare/story.py index a639bfb..fd65560 100644 --- a/fanficfare/story.py +++ b/fanficfare/story.py @@ -426,6 +426,9 @@ class Story(Configurable): self.chapter_last = None self.imgurls = [] self.imgtuples = [] + # save processed metadata, dicts keyed by 'key', then (removeentities,dorepl) + # {'key':{(removeentities,dorepl):"value",(...):"value"},'key':... } + self.processed_metadata_cache = {} self.cover=None # *href* of new cover image--need to create html. self.oldcover=None # (oldcoverhtmlhref,oldcoverhtmltype,oldcoverhtmldata,oldcoverimghref,oldcoverimgtype,oldcoverimgdata) @@ -470,6 +473,9 @@ class Story(Configurable): def setMetadata(self, key, value, condremoveentities=True): + # delete + if key in self.processed_metadata_cache: + del self.processed_metadata_cache[key] # keep as list type, but set as only value. if self.isList(key): self.addToList(key,value,condremoveentities=condremoveentities,clear=True) @@ -492,6 +498,12 @@ class Story(Configurable): self.addToList('lastupdate',value.strftime("Last Update Year/Month: %Y/%m"),clear=True) self.addToList('lastupdate',value.strftime("Last Update: %Y/%m/%d")) + if key == 'storyUrl' and value: + self.addUrlConfigSection(value) # adapter/writer share the + # same configuration. + # ignored if config + # is_lightweight() + self.replacements_prepped = False def do_in_ex_clude(self,which,value,key): # sets self.replacements and self.in_ex_cludes if needed @@ -659,13 +671,16 @@ class Story(Configurable): if not self.isValidMetaEntry(key): return value - if self.isList(key): + # check for a cached value to speed processing + if key in self.processed_metadata_cache \ + and (removeallentities,doreplacements) in self.processed_metadata_cache[key]: + return self.processed_metadata_cache[key][(removeallentities,doreplacements)] + elif self.isList(key): # join_string = self.getConfig("join_string_"+key,u", ").replace(SPACE_REPLACE,' ') # value = join_string.join(self.getList(key, removeallentities, doreplacements=True)) value = self.join_list(key,self.getList(key, removeallentities, doreplacements=True)) if doreplacements: value = self.doReplacements(value,key+"_LIST") - return value elif self.metadata.has_key(key): value = self.metadata[key] if value: @@ -690,11 +705,16 @@ class Story(Configurable): if doreplacements: value=self.doReplacements(value,key) if removeallentities and value != None: - return removeAllEntities(value) - else: - return value + value = removeAllEntities(value) else: #if self.getConfig("default_value_"+key): - return self.getConfig("default_value_"+key) + value = self.getConfig("default_value_"+key) + + # save a cached value to speed processing + if key not in self.processed_metadata_cache: + self.processed_metadata_cache[key] = {} + self.processed_metadata_cache[key][(removeallentities,doreplacements)] = value + + return value def getAllMetadata(self, removeallentities=False,