wassname/FanFicFare
1
0
Fork 0
mirror of https://github.com/wassname/FanFicFare.git synced 2026-09-09 11:14:08 +08:00
Code Issues Packages Projects Releases Wiki Activity

Add processed metadata caching, more base_xenforoforum ini features.

Browse Source
This commit is contained in:
Jim Miller
2016-02-29 22:49:28 -06:00
parent c14c52f670
commit 30bafd4e53
6 changed files with 149 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files
calibre-plugin/plugin-defaults.ini
+27
View File
@@ -488,6 +488,33 @@ description_limit:500
## in the ebook for that chapter.
continue_on_chapter_error:false
## When given a thread URL, use threadmarks as chapter links when
## there are at least this many threadmarks. A number of older
## threads have a single threadmark to an 'index' post. Set to 1 to
## use threadmarks whenever they exist.
minimum_threadmarks:2
## When 'first post' (or post URL) is being added as a chapter, give
## the chapter this title.
first_post_title:First Post
## In normal operation, if given a post URL or a thread URL with less
## than minimum_threadmarks, the given post or the first post of the
## thread will be included as the first chapter (with chapter title
## from first_post_title) unless that post is explicitly linked to in
## the collected chapter list. First post is not included when using
## thread marks.
##
## If always_include_first_post:true, then the given or first post
## will be included as above even if it is a link in the post or even
## if threadmarks are used. Can result in a duplicated chapter.
always_include_first_post:false
## In normal operation, forumtags will only be populated when
## threadmarks are used for chapters (see minimum_threadmarks above).
## When always_use_forumtags:true, always populate forumtags.
always_use_forumtags:false
## Each output format has a section that overrides [defaults]
[html]
fanficfare/adapters/base_adapter.py
-3
View File
@@ -188,9 +188,6 @@ class BaseSiteAdapter(Configurable):
self.host = self.parsedUrl.netloc
self.path = self.parsedUrl.path
self.story.setMetadata('storyUrl',self.url,condremoveentities=False)
self.addUrlConfigSection(url) # self.story shares the same configuration.
# ignored inside if config is_lightweight()
self.story.config_prepped = False
## website encoding(s)--in theory, each website reports the character
## encoding they use for each page. In practice, some sites report it
fanficfare/adapters/base_xenforoforum_adapter.py
+25 -17
View File
@@ -49,9 +49,9 @@ class BaseXenForoForumAdapter(BaseSiteAdapter):
m = re.match(self.getSiteURLPattern(),url)
if m:
#logger.debug("groupdict:%s"%m.groupdict())
if m.group('post'):
self.story.setMetadata('storyId',m.group('post'))
self._setURL(self.getURLPrefix() + '/posts/'+m.group('post')+'/')
if m.group('anchorpost'):
self.story.setMetadata('storyId',m.group('anchorpost'))
self._setURL(self.getURLPrefix() + '/posts/'+m.group('anchorpost')+'/')
else:
self.story.setMetadata('storyId',m.group('id'))
# normalized story URL.
@@ -83,7 +83,7 @@ class BaseXenForoForumAdapter(BaseSiteAdapter):
return cls.getURLPrefix()+"/threads/some-story-name.123456/ "+cls.getURLPrefix()+"/posts/123456/"
def getSiteURLPattern(self):
return r"https?://"+re.escape(self.getSiteDomain())+r"/(?P<tp>threads|posts)/(.+\.)?(?P<id>\d+)/?[^#]*?(#post-(?P<post>\d+))?$"
return r"https?://"+re.escape(self.getSiteDomain())+r"/(?P<tp>threads|posts)/(.+\.)?(?P<id>\d+)/?[^#]*?(#post-(?P<anchorpost>\d+))?$"
def use_pagecache(self):
'''
@@ -150,7 +150,7 @@ class BaseXenForoForumAdapter(BaseSiteAdapter):
raise
# use BeautifulSoup HTML parser to make everything easier to find.
soup = self.make_soup(data)
topsoup = soup = self.make_soup(data)
a = soup.find('h3',{'class':'userText'}).find('a')
self.story.addToList('authorId',a['href'].split('/')[1])
@@ -160,6 +160,9 @@ class BaseXenForoForumAdapter(BaseSiteAdapter):
h1 = soup.find('div',{'class':'titleBar'}).h1
self.story.setMetadata('title',stripHTML(h1))
first_post_title = self.getConfig('first_post_title','First Post')
threadmark_chaps = False
if '#' in useurl:
anchorid = useurl.split('#')[1]
soup = soup.find('li',id=anchorid)
@@ -176,7 +179,11 @@ class BaseXenForoForumAdapter(BaseSiteAdapter):
## SV changed their threadmarks. Not isolated to
## SV only incase SB or QQ make the same change.
markas = soupmarks.find('div',{'class':'threadmarks'}).find_all('a',{'class':'PreviewTooltip'})
if len(markas) > 1:
if len(markas) >= int(self.getConfig('minimum_threadmarks',2)):
threadmark_chaps = True
if self.getConfig('always_include_first_post'):
self.chapterUrls.append((first_post_title,useurl))
for (atag,url,name) in [ (x,x['href'],stripHTML(x)) for x in markas ]:
date = self.make_date(atag.find_next_sibling('div',{'class':'extra'}))
if not self.story.getMetadataRaw('datePublished') or date < self.story.getMetadataRaw('datePublished'):
@@ -186,16 +193,16 @@ class BaseXenForoForumAdapter(BaseSiteAdapter):
self.chapterUrls.append((name,self.getURLPrefix()+'/'+url))
## only use tags if threadmarks for chapters.
## a bit arbitrary, but likely.
for tag in soup.findAll('a',{'class':'tag'}):
tstr = stripHTML(tag)
if self.getConfig('capitalize_forumtags'):
tstr = tstr.title()
self.story.addToList('forumtags',tstr)
soup = soup.find('li',{'class':'message'}) # limit first post for date stuff below. ('#' posts above)
if threadmark_chaps or self.getConfig('always_use_forumtags'):
## only use tags if threadmarks for chapters or
for tag in topsoup.findAll('a',{'class':'tag'}):
tstr = stripHTML(tag)
if self.getConfig('capitalize_forumtags'):
tstr = tstr.title()
self.story.addToList('forumtags',tstr)
# Now go hunting for the 'chapter list'.
bq = soup.find('blockquote') # assume first posting contains TOC urls.
@@ -211,8 +218,9 @@ class BaseXenForoForumAdapter(BaseSiteAdapter):
# otherwise, use first post links--include first post since
# that's often also the first chapter.
if not self.chapterUrls:
self.chapterUrls.append(("First Post",useurl))
self.chapterUrls.append((first_post_title,useurl))
for (url,name) in [ (x['href'],stripHTML(x)) for x in bq.find_all('a') ]:
#logger.debug("found chapurl:%s"%url)
if not url.startswith('http'):
@@ -230,9 +238,9 @@ class BaseXenForoForumAdapter(BaseSiteAdapter):
logger.debug("(ch:%s)used chapurl:%s"%(len(self.chapterUrls)+1,url))
self.chapterUrls.append((name,url))
if url == useurl and 'First Post' == self.chapterUrls[0][0]:
if url == useurl and first_post_title == self.chapterUrls[0][0] \
and not self.getConfig('always_include_first_post',False):
# remove "First Post" if included in list.
logger.debug("delete dup 'First Post' chapter: %s %s"%self.chapterUrls[0])
del self.chapterUrls[0]
# Didn't use threadmarks, so take created/updated dates
fanficfare/configurable.py
+44 -22
View File
@@ -111,8 +111,6 @@ def get_valid_list_entries():
'authorId',
'authorUrl',
'lastupdate',
'keep_html_attrs',
'replace_tags_with_spans',
])
boollist=['true','false']
@@ -125,6 +123,10 @@ def get_valid_set_options():
'''
dict() of names of boolean options, but as a tuple with
valid sites, valid formats and valid values (None==all)
This is to further restrict keywords to certain sections and/or
values. get_valid_keywords() below is the list of allowed
keywords. Any keyword listed here must also be listed there.
'''
valdict = {'collect_series':(None,None,boollist),
@@ -177,7 +179,13 @@ def get_valid_set_options():
'grayscale_images':(None,['epub','html'],boollist),
'no_image_processing':(None,['epub','html'],boollist),
'capitalize_forumtags':(base_xenforo_list,None,boollist),
'continue_on_chapter_error':(base_xenforo_list,None,boollist),
'minimum_threadmarks':(base_xenforo_list,None,None),
'first_post_title':(base_xenforo_list,None,None),
'always_include_first_post':(base_xenforo_list,None,boollist),
'':(base_xenforo_list,None,boollist),
'':(base_xenforo_list,None,boollist),
'':(base_xenforo_list,None,boollist),
}
@@ -339,8 +347,12 @@ def get_valid_keywords():
'wrap_width',
'zip_filename',
'zip_output',
'continue_on_chapter_error',
'capitalize_forumtags',
'continue_on_chapter_error',
'minimum_threadmarks',
'first_post_title',
'always_include_first_post',
'',
])
# *known* entry keywords -- or rather regexps for them.
@@ -405,9 +417,16 @@ class Configuration(ConfigParser.SafeConfigParser):
self.validEntries = get_valid_entries()
self.url_config_set = False
def addUrlConfigSection(self,url):
if not self.lightweight: # don't need when just checking for normalized URL.
self.addConfigSection(url,'overrides')
# replace if already set once.
if self.url_config_set:
self.sectionslist[self.sectionslist.index('overrides')+1]=url
else:
self.addConfigSection(url,'overrides')
self.url_config_set=True
def addConfigSection(self,section,before=None):
if section not in self.sectionslist: # don't add if already present.
@@ -616,7 +635,10 @@ class Configuration(ConfigParser.SafeConfigParser):
def test_config(self):
errors=[]
allowedsections_re = re.compile(r'^(teststory:(defaults|[0-9]+)|https?://.*)$')
## too complicated right now to enforce
## get_valid_set_options() warnings on teststory and
## [storyUrl] sections.
allow_all_sections_re = re.compile(r'^(teststory:(defaults|[0-9]+)|https?://.*)$')
allowedsections = get_valid_sections()
clude_metadata_re = re.compile(r'(add_to_)?(in|ex)clude_metadata_(pre|post)')
@@ -626,12 +648,13 @@ class Configuration(ConfigParser.SafeConfigParser):
custom_columns_settings_re = re.compile(r'(add_to_)?custom_columns_settings')
generate_cover_settings_re = re.compile(r'(add_to_)?generate_cover_settings')
generate_cover_settings_re = re.compile(r'(add_to_)?generate_cover_settings')
valdict = get_valid_set_options()
for section in self.sections():
if section not in allowedsections and not allowedsections_re.match(section):
allow_all_section = allow_all_sections_re.match(section)
if section not in allowedsections and not allow_all_section:
errors.append((self.get_lineno(section),"Bad Section Name: [%s]"%section))
else:
sitename = section.replace('www.','')
@@ -669,26 +692,25 @@ class Configuration(ConfigParser.SafeConfigParser):
# timeline=>#ccolumn,n
# "FanFiction"=>#collection
def make_sections(x):
return '['+'], ['.join(x)+']'
if keyword in valdict:
(valsites,valformats,vals)=valdict[keyword]
if valsites != None and sitename != None and sitename not in valsites:
errors.append((self.get_lineno(section,keyword),"%s not valid in section [%s] -- only valid in %s sections."%(keyword,section,make_sections(valsites))))
if valformats != None and formatname != None and formatname not in valformats:
errors.append((self.get_lineno(section,keyword),"%s not valid in section [%s] -- only valid in %s sections."%(keyword,section,make_sections(valformats))))
if value not in vals:
errors.append((self.get_lineno(section,keyword),"%s not a valid value for %s"%(value,keyword)))
if not allow_all_section:
def make_sections(x):
return '['+'], ['.join(x)+']'
if keyword in valdict:
(valsites,valformats,vals)=valdict[keyword]
if valsites != None and sitename != None and sitename not in valsites:
errors.append((self.get_lineno(section,keyword),"%s not valid in section [%s] -- only valid in %s sections."%(keyword,section,make_sections(valsites))))
if valformats != None and formatname != None and formatname not in valformats:
errors.append((self.get_lineno(section,keyword),"%s not valid in section [%s] -- only valid in %s sections."%(keyword,section,make_sections(valformats))))
if vals != None and value not in vals:
errors.append((self.get_lineno(section,keyword),"%s not a valid value for %s"%(value,keyword)))
## skipping output_filename_safepattern
## regex--not used with plugin and this isn't
## used with CLI/web yet.
except Exception as e:
errors.append((self.get_lineno(section,keyword),"Error:%s in (%s:%s)"%(e,keyword,value)))
errors.append((self.get_lineno(section,keyword),"Error:%s in (%s:%s)"%(e,keyword,value)))
return errors
# extended by adapter, writer and story for ease of calling configuration.
fanficfare/defaults.ini
+27
View File
@@ -487,6 +487,33 @@ description_limit:500
## in the ebook for that chapter.
continue_on_chapter_error:false
## When given a thread URL, use threadmarks as chapter links when
## there are at least this many threadmarks. A number of older
## threads have a single threadmark to an 'index' post. Set to 1 to
## use threadmarks whenever they exist.
minimum_threadmarks:2
## When 'first post' (or post URL) is being added as a chapter, give
## the chapter this title.
first_post_title:First Post
## In normal operation, if given a post URL or a thread URL with less
## than minimum_threadmarks, the given post or the first post of the
## thread will be included as the first chapter (with chapter title
## from first_post_title) unless that post is explicitly linked to in
## the collected chapter list. First post is not included when using
## thread marks.
##
## If always_include_first_post:true, then the given or first post
## will be included as above even if it is a link in the post or even
## if threadmarks are used. Can result in a duplicated chapter.
always_include_first_post:false
## In normal operation, forumtags will only be populated when
## threadmarks are used for chapters (see minimum_threadmarks above).
## When always_use_forumtags:true, always populate forumtags.
always_use_forumtags:false
## Each output format has a section that overrides [defaults]
[html]
fanficfare/story.py
+26 -6
View File
@@ -426,6 +426,9 @@ class Story(Configurable):
self.chapter_last = None
self.imgurls = []
self.imgtuples = []
# save processed metadata, dicts keyed by 'key', then (removeentities,dorepl)
# {'key':{(removeentities,dorepl):"value",(...):"value"},'key':... }
self.processed_metadata_cache = {}
self.cover=None # *href* of new cover image--need to create html.
self.oldcover=None # (oldcoverhtmlhref,oldcoverhtmltype,oldcoverhtmldata,oldcoverimghref,oldcoverimgtype,oldcoverimgdata)
@@ -470,6 +473,9 @@ class Story(Configurable):
def setMetadata(self, key, value, condremoveentities=True):
# delete
if key in self.processed_metadata_cache:
del self.processed_metadata_cache[key]
# keep as list type, but set as only value.
if self.isList(key):
self.addToList(key,value,condremoveentities=condremoveentities,clear=True)
@@ -492,6 +498,12 @@ class Story(Configurable):
self.addToList('lastupdate',value.strftime("Last Update Year/Month: %Y/%m"),clear=True)
self.addToList('lastupdate',value.strftime("Last Update: %Y/%m/%d"))
if key == 'storyUrl' and value:
self.addUrlConfigSection(value) # adapter/writer share the
# same configuration.
# ignored if config
# is_lightweight()
self.replacements_prepped = False
def do_in_ex_clude(self,which,value,key):
# sets self.replacements and self.in_ex_cludes if needed
@@ -659,13 +671,16 @@ class Story(Configurable):
if not self.isValidMetaEntry(key):
return value
if self.isList(key):
# check for a cached value to speed processing
if key in self.processed_metadata_cache \
and (removeallentities,doreplacements) in self.processed_metadata_cache[key]:
return self.processed_metadata_cache[key][(removeallentities,doreplacements)]
elif self.isList(key):
# join_string = self.getConfig("join_string_"+key,u", ").replace(SPACE_REPLACE,' ')
# value = join_string.join(self.getList(key, removeallentities, doreplacements=True))
value = self.join_list(key,self.getList(key, removeallentities, doreplacements=True))
if doreplacements:
value = self.doReplacements(value,key+"_LIST")
return value
elif self.metadata.has_key(key):
value = self.metadata[key]
if value:
@@ -690,11 +705,16 @@ class Story(Configurable):
if doreplacements:
value=self.doReplacements(value,key)
if removeallentities and value != None:
return removeAllEntities(value)
else:
return value
value = removeAllEntities(value)
else: #if self.getConfig("default_value_"+key):
return self.getConfig("default_value_"+key)
value = self.getConfig("default_value_"+key)
# save a cached value to speed processing
if key not in self.processed_metadata_cache:
self.processed_metadata_cache[key] = {}
self.processed_metadata_cache[key][(removeallentities,doreplacements)] = value
return value
def getAllMetadata(self,
removeallentities=False,